diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh index f47a1b9da..58f0c3439 100644 --- a/scripts/fork-validate.sh +++ b/scripts/fork-validate.sh @@ -21,6 +21,10 @@ # and every shutdown path (mid-release, mid-wait, mid-generation, cold, warm, kill -9 of # a waiter) exits cleanly and hands the GPU token on. These are the past production # incidents (stranded requests, wedged token, exit 134 on cold teardown). +# Then a-handoff-latency: ordinary handoff latency of --build vs --ref in the same +# session (fresh server pairs, blocks ref,cand,cand,ref x 16 handoffs) -> FAIL if the +# candidate median exceeds the reference median by more than max(25%, 200 ms). Two +# production deploys were lost to arbiter changes that only showed up here. # --perf llama-bench, candidate vs reference, alternating ref,cand,cand,ref, on the 35B MoE # -> no prefill regression > 5% at depth 0/16k/32k (FAIL), decode tg256@d4096 # reported; peak VRAM per arm (FAIL if > 95% of the card). Compares MEDIANS of every @@ -582,6 +586,64 @@ run_arbiter() { local bad_lines; bad_lines=$(scan_logs "$S/A.log" "$S/B.log") check arbiter/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) \ "$([ -z "$bad_lines" ] && echo 'no asserts / Vulkan errors in A/B logs' || echo "$bad_lines" | head -3 | tr '\n' ' ')" + + run_handoff_ab +} + +# handoff_block BUILD TAG N : a fresh pair of BUILD on the scratch arena (A with mmproj, B text-only, +# the same config as case a), one warm-up request on A, then N+1 alternating requests starting with +# B. Request 0 is not a handoff (B holds the token after its load); the latency of requests 1..N is +# appended to $S/lat-TAG.txt. HB_BAD counts setup failures, non-200s and answers without the token. +handoff_block() { + local build=$1 tag=$2 n=$3 a b i port pid code t0 t1 + HB_BAD=0 + : > "$S/lat-$tag.txt" + start_server "L$tag-A" "$build" "$PA" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --mmproj "$MMPROJ"; a=$SPID + if ! wait_health "$PA" "$a" 180; then HB_BAD=1; stop_server "$a"; return 1; fi + complete "$PA" "$S/L$tag-warm.json" "$(greedy_body 8 false)" > /dev/null + start_server "L$tag-B" "$build" "$PB" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --no-mmproj; b=$SPID + if ! wait_health "$PB" "$b" 180; then HB_BAD=1; stop_server "$b"; stop_server "$a"; return 1; fi + for i in $(seq 0 "$n"); do + if [ $((i % 2)) -eq 0 ]; then port=$PB pid=$b; else port=$PA pid=$a; fi + t0=$(now_ms) + code=$(complete "$port" "$S/L$tag-$i.json" "$(greedy_body 48 false)" 60) + t1=$(now_ms) + if [ "$code" != 200 ] || [ "$(token_holder)" != "$pid" ]; then HB_BAD=$((HB_BAD + 1)); continue; fi + [ "$i" -gt 0 ] && echo $((t1 - t0)) >> "$S/lat-$tag.txt" + done + stop_server "$b"; stop_server "$a" +} + +# ---- (a') ordinary handoff latency, candidate vs reference, in one session +# Two production deploys were lost to arbiter changes that passed the pathological cases but made +# every ordinary handoff slower. Blocks run ref, cand, cand, ref (a fixed order once gave the later +# slot about 9%); each build's median is over all its handoffs. FAIL if the candidate median exceeds +# the reference median by more than max(25%, 200 ms): small absolute numbers are noisy. +run_handoff_ab() { + local n=16 blk k=0 bad_r=0 bad_c=0 tag + log "arbiter: handoff latency A/B, blocks ref,cand,cand,ref x $n handoffs" + : > "$S/lat-ref.txt"; : > "$S/lat-cand.txt" + for blk in ref cand cand ref; do + k=$((k + 1)); tag=$k-$blk + local b; [ $blk = ref ] && b=$REF || b=$BUILD + handoff_block "$b" "$tag" "$n" + if [ $blk = ref ]; then bad_r=$((bad_r + HB_BAD)); else bad_c=$((bad_c + HB_BAD)); fi + cat "$S/lat-$tag.txt" >> "$S/lat-$blk.txt" + log "arbiter: block $tag median $(median < "$S/lat-$tag.txt") ms over $(wc -l < "$S/lat-$tag.txt") handoffs, $HB_BAD bad" + done + local mr mc nr nc maxr maxc limit + mr=$(median < "$S/lat-ref.txt"); mc=$(median < "$S/lat-cand.txt") + nr=$(wc -l < "$S/lat-ref.txt"); nc=$(wc -l < "$S/lat-cand.txt") + maxr=$(sort -n "$S/lat-ref.txt" | tail -1); maxc=$(sort -n "$S/lat-cand.txt" | tail -1) + if [ -z "$mr" ] || [ -z "$mc" ] || [ $bad_r -gt 0 ] || [ $bad_c -gt 0 ]; then + fail arbiter/a-handoff-latency "blocks incomplete: reference $nr handoffs ($bad_r bad), candidate $nc ($bad_c bad); see $S/L*-*.log" + return + fi + limit=$(awk -v r="$mr" 'BEGIN {d = r * 0.25; if (d < 200) d = 200; printf "%.0f", r + d}') + check arbiter/a-handoff-latency "$(awk -v c="$mc" -v l="$limit" 'BEGIN {print (c > l) ? 1 : 0}')" \ + "median handoff latency candidate ${mc} ms (n $nc, max $maxc) vs reference ${mr} ms (n $nr, max $maxr); limit ${limit} ms = ref + max(25%, 200 ms)" + local bad_lines; bad_lines=$(scan_logs "$S"/L*-A.log "$S"/L*-B.log) + [ -z "$bad_lines" ] || fail arbiter/a-handoff-latency-logs "$(echo "$bad_lines" | head -3 | tr '\n' ' ')" } # ================================================================ 4. perf