--arbiter only reported the candidate's handoff latency, while the fork's process rule (two production deploys lost) is to A/B the ordinary handoff, not only the pathological cases. After the arbiter cases, a-handoff-latency starts fresh A/B server pairs of each build on the scratch arena and runs blocks ref, cand, cand, ref of 16 alternating handoffs each. It FAILs if the candidate median exceeds the reference median by more than max(25%, 200 ms) and reports both medians, maxima and per-block medians. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
783 lines
45 KiB
Bash
783 lines
45 KiB
Bash
#!/usr/bin/env bash
|
|
# fork-validate.sh - validate a candidate build of the RX 580 llama.cpp fork against a reference build.
|
|
#
|
|
# Runs ON THE POD (Vulkan / RADV, RX 580). Prints PASS/FAIL/SKIP per check plus a summary; exits
|
|
# nonzero if any check FAILs. Only touches its own scratch dir under /tmp, its own scratch ports and
|
|
# processes it started itself (killed by explicit PID). Build dirs are only read.
|
|
#
|
|
# fork-validate.sh --build /root/arms/X/build --ref /root/llama.cpp/build [checks] [options]
|
|
#
|
|
# Checks (no check flag = --ops --concat --arbiter):
|
|
# --ops test-backend-ops on Vulkan0 for CONCAT, FLASH_ATTN_EXT, MUL_MAT, MUL_MAT_ID, GET_ROWS
|
|
# -> the fork's kernels and upstream's agree with the CPU reference; asserts the case count
|
|
# is > 0 because "-b Vulkan" (no index) silently runs nothing and still prints OK.
|
|
# --concat Vulkan concat tiled-transpose fast path (ggml_vk_concat, gate src_transposed_2d)
|
|
# -> it fires at prefill and NOT at decode on a real hybrid (delta-net) model; it is the
|
|
# fork's biggest prefill win, and taking it at decode once cost -17% decode.
|
|
# Needs a build carrying the GGML_VK_CONCAT_FASTPATH knob (=0 off, =log report);
|
|
# either --build has it or pass --concat-build DIR (candidate + the knob patch).
|
|
# --arbiter on-demand VRAM sharing (LLAMA_SLEEP_VRAM_ONLY ...) with two real servers on a scratch
|
|
# arena -> handoffs never strand, release/restore is bit-exact, multimodal survives it,
|
|
# and every shutdown path (mid-release, mid-wait, mid-generation, cold, warm, kill -9 of
|
|
# a waiter) exits cleanly and hands the GPU token on. These are the past production
|
|
# incidents (stranded requests, wedged token, exit 134 on cold teardown).
|
|
# Then a-handoff-latency: ordinary handoff latency of --build vs --ref in the same
|
|
# session (fresh server pairs, blocks ref,cand,cand,ref x 16 handoffs) -> FAIL if the
|
|
# candidate median exceeds the reference median by more than max(25%, 200 ms). Two
|
|
# production deploys were lost to arbiter changes that only showed up here.
|
|
# --perf llama-bench, candidate vs reference, alternating ref,cand,cand,ref, on the 35B MoE
|
|
# -> no prefill regression > 5% at depth 0/16k/32k (FAIL), decode tg256@d4096
|
|
# reported; peak VRAM per arm (FAIL if > 95% of the card). Compares MEDIANS of every
|
|
# repetition sample pooled over a build's two arms (4 per prefill cell, 6 for decode),
|
|
# never means: CPU contention on the node produces single catastrophic samples.
|
|
# Needs llama-swap STOPPED (run it under /root/bench/moe/swapdown.sh); refuses to
|
|
# run if any other llama process is up. Samples are kept in <scratch>/perf.tsv.
|
|
# --moecache MODEL
|
|
# one arbiter server with --moe-cache-mib N on a MoE model, forced cold/warm twice
|
|
# -> greedy output identical before/after (the expert-cache SLRU is re-initialised on
|
|
# release; a stale SLRU maps experts to empty banks and produces garbage).
|
|
#
|
|
# Options:
|
|
# --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build)
|
|
# --small MODEL small hybrid model for --concat/--arbiter (default: Qwen3.5-0.8B UD-Q4_K_XL)
|
|
# --mmproj FILE its multimodal projector (default: mmproj-F16 next to it)
|
|
# --perf-model MODEL model for --perf (default: Qwen3.6-35B-A3B UD-Q4_K_XL)
|
|
# --moecache-args S extra server args for --moecache (default: "-ngl 99 --n-cpu-moe 30 -c 4096")
|
|
# --moecache-mib N cache size for --moecache (default 2048; 256 never hits on a 26B and proves nothing)
|
|
# --port-base N scratch ports N..N+2 (default 8091)
|
|
# --keep keep the scratch dir even when everything passed (it is kept on any FAIL)
|
|
# --perf-report TSV only re-aggregate a saved <scratch>/perf.tsv of an earlier --perf run and
|
|
# apply the same gates (no GPU, no --build/--ref needed); runs nothing else
|
|
#
|
|
# Requirements: bash, coreutils, awk, sed, grep, curl, flock(1); gdb optional (makes the
|
|
# SIGTERM-during-release case deterministic). No python/jq.
|
|
|
|
set -u
|
|
|
|
# ---------------------------------------------------------------- defaults
|
|
HF=/root/.cache/huggingface/hub
|
|
SMALL_DIR=$HF/models--unsloth--Qwen3.5-0.8B-GGUF/snapshots/6ab461498e2023f6e3c1baea90a8f0fe38ab64d0
|
|
SMALL=$SMALL_DIR/Qwen3.5-0.8B-UD-Q4_K_XL.gguf
|
|
MMPROJ=
|
|
PERF_MODEL=$HF/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
|
MOECACHE_MODEL=
|
|
MOECACHE_ARGS="-ngl 99 --n-cpu-moe 30 -c 4096"
|
|
MOECACHE_MIB=2048
|
|
BUILD= ; REF= ; CONCAT_BUILD=
|
|
PORT_BASE=8091
|
|
KEEP=0
|
|
DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0
|
|
PERF_REPORT=
|
|
|
|
usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; }
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--build) BUILD=$2; shift 2 ;;
|
|
--ref) REF=$2; shift 2 ;;
|
|
--ops) DO_OPS=1; shift ;;
|
|
--concat) DO_CONCAT=1; shift ;;
|
|
--arbiter) DO_ARB=1; shift ;;
|
|
--perf) DO_PERF=1; shift ;;
|
|
--moecache) DO_MOEC=1; MOECACHE_MODEL=$2; shift 2 ;;
|
|
--moecache-args) MOECACHE_ARGS=$2; shift 2 ;;
|
|
--moecache-mib) MOECACHE_MIB=$2; shift 2 ;;
|
|
--concat-build) CONCAT_BUILD=$2; shift 2 ;;
|
|
--small) SMALL=$2; shift 2 ;;
|
|
--mmproj) MMPROJ=$2; shift 2 ;;
|
|
--perf-model) PERF_MODEL=$2; shift 2 ;;
|
|
--port-base) PORT_BASE=$2; shift 2 ;;
|
|
--keep) KEEP=1; shift ;;
|
|
--perf-report) PERF_REPORT=$(readlink -f "$2"); shift 2 ;;
|
|
-h|--help) usage ;;
|
|
*) echo "unknown argument: $1" >&2; usage ;;
|
|
esac
|
|
done
|
|
[ -n "$PERF_REPORT" ] || { [ -n "$BUILD" ] && [ -n "$REF" ]; } || usage
|
|
if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC)) -eq 0 ]; then
|
|
DO_OPS=1 DO_CONCAT=1 DO_ARB=1
|
|
fi
|
|
[ -n "$MMPROJ" ] || MMPROJ=$(dirname "$SMALL")/mmproj-F16.gguf
|
|
PA=$PORT_BASE PB=$((PORT_BASE + 1)) PC=$((PORT_BASE + 2))
|
|
|
|
# ---------------------------------------------------------------- reporting
|
|
RESULTS=()
|
|
NFAIL=0
|
|
ts() { date +%H:%M:%S; }
|
|
log() { echo "[$(ts)] $*"; }
|
|
pass() { RESULTS+=("PASS $1: $2"); echo "[$(ts)] PASS $1: $2"; }
|
|
fail() { RESULTS+=("FAIL $1: $2"); echo "[$(ts)] FAIL $1: $2"; NFAIL=$((NFAIL + 1)); }
|
|
skip() { RESULTS+=("SKIP $1: $2"); echo "[$(ts)] SKIP $1: $2"; }
|
|
# check NAME CONDITION-RESULT(0/1) MESSAGE
|
|
check() { if [ "$2" -eq 0 ]; then pass "$1" "$3"; else fail "$1" "$3"; fi; }
|
|
now_ms() { date +%s%3N; }
|
|
|
|
# ---------------------------------------------------------------- scratch + cleanup
|
|
S=$(mktemp -d /tmp/fork-validate.XXXXXX)
|
|
ARENA=$S/arena
|
|
mkdir -p "$ARENA"
|
|
cd "$S" || exit 2 # anything written to a relative path lands in the scratch dir
|
|
PIDS=() # "pid:starttime" of every background process we start (starttime guards PID reuse)
|
|
|
|
pid_alive() { # alive and not a zombie
|
|
[ -r "/proc/$1/stat" ] || return 1
|
|
local st; st=$(sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | awk '{print $1}') || return 1
|
|
[ -n "$st" ] && [ "$st" != Z ]
|
|
}
|
|
starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | awk '{print $20}'; }
|
|
track() { PIDS+=("$1:$(starttime "$1")"); }
|
|
ours() { # "pid:starttime" -> pid if that exact process is still alive
|
|
local p=${1%%:*} t=${1#*:}
|
|
pid_alive "$p" && [ "$(starttime "$p")" = "$t" ] && echo "$p"
|
|
}
|
|
|
|
cleanup() {
|
|
local e p
|
|
for e in "${PIDS[@]}"; do p=$(ours "$e") && kill -TERM "$p" 2>/dev/null; done
|
|
local i=0
|
|
while [ $i -lt 50 ]; do
|
|
local any=0
|
|
for e in "${PIDS[@]}"; do ours "$e" > /dev/null && any=1; done
|
|
[ $any -eq 0 ] && break
|
|
sleep 0.1; i=$((i + 1))
|
|
done
|
|
for e in "${PIDS[@]}"; do p=$(ours "$e") && kill -KILL "$p" 2>/dev/null; done
|
|
wait 2>/dev/null
|
|
}
|
|
finish() {
|
|
local rc=$?
|
|
trap - EXIT INT TERM HUP
|
|
cleanup
|
|
echo
|
|
echo "==================== fork-validate summary ===================="
|
|
echo "candidate: $BUILD ($(git -C "$BUILD/.." rev-parse --short HEAD 2>/dev/null || echo '?'))"
|
|
echo "reference: $REF ($(git -C "$REF/.." rev-parse --short HEAD 2>/dev/null || echo '?'))"
|
|
local r; for r in "${RESULTS[@]}"; do echo "$r"; done
|
|
local np ns
|
|
np=$(printf '%s\n' "${RESULTS[@]}" | grep -c '^PASS')
|
|
ns=$(printf '%s\n' "${RESULTS[@]}" | grep -c '^SKIP')
|
|
echo "total: $np PASS, $NFAIL FAIL, $ns SKIP"
|
|
if [ $NFAIL -gt 0 ] || [ $KEEP -eq 1 ] || [ $rc -ne 0 ]; then
|
|
echo "scratch kept: $S"
|
|
else
|
|
rm -rf "$S"
|
|
fi
|
|
[ $NFAIL -gt 0 ] && exit 1
|
|
exit $rc
|
|
}
|
|
trap finish EXIT
|
|
trap 'echo "interrupted"; NFAIL=$((NFAIL + 1)); exit 130' INT TERM HUP
|
|
|
|
# ---------------------------------------------------------------- environment sanity
|
|
for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\)[A-Za-z0-9_]*\)=.*/\1/p'); do
|
|
log "unsetting inherited $v"; unset "$v"
|
|
done
|
|
[ -n "$PERF_REPORT" ] || for d in "$BUILD" "$REF"; do
|
|
[ -x "$d/bin/llama-server" ] || { echo "no llama-server in $d/bin" >&2; exit 2; }
|
|
done
|
|
case "$ARENA" in /dev/shm/llama-vram*) echo "refusing the production arena" >&2; exit 2 ;; esac
|
|
VRAM_USED=$(ls /sys/class/drm/card*/device/mem_info_vram_used 2>/dev/null | head -1)
|
|
VRAM_TOTAL=$(ls /sys/class/drm/card*/device/mem_info_vram_total 2>/dev/null | head -1)
|
|
vram_mb() { echo $(( $(cat "$VRAM_USED") / 1048576 )); }
|
|
port_busy() { # LISTEN socket on the port (v4 or v6)
|
|
local hex; hex=$(printf '%04X' "$1")
|
|
awk -v h=":$hex" '$4=="0A" && substr($2, length($2)-4)==h {f=1} END {exit !f}' /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
|
}
|
|
[ -n "$PERF_REPORT" ] || for p in $PA $PB $PC; do
|
|
port_busy "$p" && { echo "scratch port $p is in use" >&2; exit 2; }
|
|
done
|
|
log "scratch dir $S, ports $PA/$PB/$PC, VRAM used $(vram_mb) MiB"
|
|
|
|
# ---------------------------------------------------------------- helpers: servers
|
|
# start_server NAME BUILD PORT ENV-ASSIGNMENTS... -- SERVER-ARGS... (sets SPID, log at $S/NAME.log)
|
|
start_server() {
|
|
local name=$1 build=$2 port=$3; shift 3
|
|
local envs=()
|
|
while [ $# -gt 0 ] && [ "$1" != -- ]; do envs+=("$1"); shift; done
|
|
shift
|
|
env LD_LIBRARY_PATH="$build/bin" "${envs[@]}" "$build/bin/llama-server" \
|
|
--host 127.0.0.1 --port "$port" "$@" >> "$S/$name.log" 2>&1 &
|
|
SPID=$!
|
|
track "$SPID"
|
|
echo "=== start pid $SPID $(date +%T.%N)" >> "$S/$name.log.events"
|
|
}
|
|
# wait_health PORT PID TIMEOUT_S
|
|
wait_health() {
|
|
local t0; t0=$(now_ms)
|
|
while [ $(( $(now_ms) - t0 )) -lt $(( $3 * 1000 )) ]; do
|
|
pid_alive "$2" || return 1
|
|
[ "$(curl -s -m 2 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$1/health")" = 200 ] && return 0
|
|
sleep 0.2
|
|
done
|
|
return 1
|
|
}
|
|
# wait_exit PID TIMEOUT_MS -> prints elapsed ms, returns 1 on timeout. Then reap with `wait PID`.
|
|
wait_exit() {
|
|
local t0; t0=$(now_ms)
|
|
while pid_alive "$1"; do
|
|
[ $(( $(now_ms) - t0 )) -ge "$2" ] && { echo $(( $(now_ms) - t0 )); return 1; }
|
|
sleep 0.02
|
|
done
|
|
echo $(( $(now_ms) - t0 ))
|
|
}
|
|
# stop_server PID -> SIGTERM, wait up to 20 s, sets RC and EXIT_MS
|
|
stop_server() {
|
|
kill -TERM "$1" 2>/dev/null
|
|
EXIT_MS=$(wait_exit "$1" 20000) || { kill -KILL "$1" 2>/dev/null; }
|
|
wait "$1" 2>/dev/null; RC=$?
|
|
}
|
|
|
|
# ---------------------------------------------------------------- helpers: locks (/proc/locks)
|
|
lock_id() { # maj:min:inode as /proc/locks prints it (hex dev, decimal inode)
|
|
local d; d=$(( 16#$(stat -c %D "$1") ))
|
|
printf '%02x:%02x:%s' $(( (d >> 8) & 0xfff )) $(( (d & 0xff) | ((d >> 12) & 0xfff00) )) "$(stat -c %i "$1")"
|
|
}
|
|
lock_pids() { # FILE WRITE|READ -> PIDs holding a flock on it (granted, not blocked)
|
|
[ -e "$1" ] || return 0
|
|
awk -v id="$(lock_id "$1")" -v m="$2" '$2=="FLOCK" && $4==m && $6==id {print $5}' /proc/locks
|
|
}
|
|
token_holder() { lock_pids "$ARENA/token.lock" WRITE | head -1; }
|
|
holder_num() { local h; h=$(token_holder); echo "${h:-0}"; } # for arithmetic: 0 = nobody
|
|
# wait_holder PID TIMEOUT_MS : until PID holds token.lock ("" = until nobody does)
|
|
wait_holder() {
|
|
local t0; t0=$(now_ms)
|
|
while [ "$(token_holder)" != "$1" ]; do
|
|
[ $(( $(now_ms) - t0 )) -ge "$2" ] && return 1
|
|
sleep 0.01
|
|
done
|
|
echo $(( $(now_ms) - t0 ))
|
|
}
|
|
wait_want() { # PID TIMEOUT_MS : until PID holds want.lock shared (registered waiter)
|
|
local t0; t0=$(now_ms)
|
|
until lock_pids "$ARENA/want.lock" READ | grep -qx "$1"; do
|
|
[ $(( $(now_ms) - t0 )) -ge "$2" ] && return 1
|
|
sleep 0.01
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------- helpers: requests
|
|
json_str() { sed -n 's/.*"'"$1"'":"\(\([^"\\]\|\\.\)*\)".*/\1/p' "$2" | head -1; }
|
|
json_num() { grep -o "\"$1\":[0-9]*" "$2" | head -1 | cut -d: -f2; }
|
|
# complete PORT OUTFILE JSON-BODY [TIMEOUT_S] -> prints HTTP code ("000" on timeout/refused)
|
|
complete() {
|
|
curl -sS -m "${4:-60}" -o "$2" -w '%{http_code}' -H 'Content-Type: application/json' \
|
|
-d "$3" "http://127.0.0.1:$1/completion" 2>>"$S/curl.err" || true
|
|
}
|
|
PROMPT='The history of the printing press began in'
|
|
GREEDY="\"temperature\":0,\"top_k\":1,\"seed\":42"
|
|
greedy_body() { # N_PREDICT CACHE_PROMPT [PROMPT]
|
|
echo "{\"prompt\":\"${3:-$PROMPT}\",\"n_predict\":$1,$GREEDY,\"cache_prompt\":$2}"
|
|
}
|
|
LONG_BODY="{\"prompt\":\"Write a very long story about a lighthouse keeper.\",\"n_predict\":4000,\"ignore_eos\":true,$GREEDY,\"stream\":true}"
|
|
# stream_bg PORT OUTFILE BODY -> starts a streaming request, sets CPID
|
|
stream_bg() {
|
|
curl -sS -N -m 300 -H 'Content-Type: application/json' -d "$3" \
|
|
"http://127.0.0.1:$1/completion" > "$2" 2>&1 &
|
|
CPID=$!; track "$CPID"
|
|
}
|
|
wait_tokens() { # FILE N TIMEOUT_MS : until N SSE data lines arrived
|
|
local t0; t0=$(now_ms)
|
|
while [ "$(grep -c '^data: ' "$1" 2>/dev/null)" -lt "$2" ]; do
|
|
[ $(( $(now_ms) - t0 )) -ge "$3" ] && return 1
|
|
sleep 0.05
|
|
done
|
|
}
|
|
scan_logs() { # NAME-PATTERN... -> prints offending lines
|
|
grep -h -n -E 'GGML_ASSERT|Assertion|terminate called|Segmentation fault|core dumped|ErrorDeviceLost|ErrorOutOfDeviceMemory|DEVICE_LOST|vk::.*Error|GGML_ABORT|fatal error' "$@" 2>/dev/null | head -20
|
|
}
|
|
|
|
# ================================================================ 1. ops
|
|
run_ops() {
|
|
local op out passed total
|
|
for op in CONCAT FLASH_ATTN_EXT MUL_MAT MUL_MAT_ID GET_ROWS; do
|
|
out=$S/ops-$op.log
|
|
log "ops: test-backend-ops -b Vulkan0 -o $op"
|
|
( cd "$S" && LD_LIBRARY_PATH="$BUILD/bin" timeout 1800 "$BUILD/bin/test-backend-ops" -b Vulkan0 -o "$op" ) \
|
|
> "$out" 2>&1
|
|
local rc=$?
|
|
# strip ANSI colours; "N/M tests passed" is printed once per tested backend
|
|
read -r passed total < <(sed 's/\x1b\[[0-9;]*m//g' "$out" | sed -n 's/^ *\([0-9]*\)\/\([0-9]*\) tests passed.*/\1 \2/p' | head -1)
|
|
passed=${passed:-0}; total=${total:-0}
|
|
if [ $rc -eq 0 ] && [ "$total" -gt 0 ] && [ "$passed" -eq "$total" ] \
|
|
&& sed 's/\x1b\[[0-9;]*m//g' "$out" | grep -q 'Backend Vulkan0: OK'; then
|
|
pass "ops/$op" "$passed/$total cases on Vulkan0"
|
|
else
|
|
fail "ops/$op" "rc=$rc, $passed/$total passed (0 cases = wrong backend name); see $out"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ================================================================ 2. concat fast path
|
|
# Method: llama-bench on the small hybrid model (qwen35: delta-net layers concat the conv state with
|
|
# the transposed qkv in build_conv_state). (1) with the knob =log, count fast-path dispatches in a
|
|
# prefill-only run (must be > 0, all with >= 32 tokens) and a decode-only run (must be 0, while the
|
|
# perf logger proves CONCAT nodes did execute); (2) GGML_VK_PERF_LOGGER CONCAT time at prefill: the
|
|
# unmodified candidate must match knob-on and be >= 2x faster than knob-off, which ties the
|
|
# candidate binary itself (no knob) to the fast path.
|
|
concat_us() { # LOG -> "count avg_us" over all perf-logger CONCAT lines
|
|
awk '/^CONCAT: / {n += $2; t += $(NF-1)} END {if (n) printf "%d %.2f\n", n, t/n; else print "0 0"}' "$1"
|
|
}
|
|
run_concat() {
|
|
local tb=$CONCAT_BUILD
|
|
[ -n "$tb" ] || tb=$BUILD
|
|
if ! grep -qa GGML_VK_CONCAT_FASTPATH "$tb/bin/libggml-vulkan.so" 2>/dev/null; then
|
|
fail concat "no GGML_VK_CONCAT_FASTPATH knob in $tb (build the candidate + knob patch, pass --concat-build)"
|
|
return
|
|
fi
|
|
[ -f "$SMALL" ] || { fail concat "small model missing: $SMALL"; return; }
|
|
local common=(-m "$SMALL" -ngl 99 -fa 1 -lm none -t 6)
|
|
local pp=(-p 512 -n 0 -b 512 -ub 512 -r 3)
|
|
local tg=(-p 0 -n 32 -r 1)
|
|
lb() { # BUILD OUT ENV... -- ARGS
|
|
local b=$1 out=$2; shift 2
|
|
local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift
|
|
env LD_LIBRARY_PATH="$b/bin" GGML_VK_PERF_LOGGER=1 "${envs[@]}" "$b/bin/llama-bench" "${common[@]}" "$@" \
|
|
> "$out" 2>&1
|
|
}
|
|
log "concat: llama-bench prefill/decode arms (perf logger on)"
|
|
lb "$tb" "$S/cc-pp-log.log" GGML_VK_CONCAT_FASTPATH=log -- "${pp[@]}"
|
|
lb "$tb" "$S/cc-tg-log.log" GGML_VK_CONCAT_FASTPATH=log -- "${tg[@]}"
|
|
lb "$tb" "$S/cc-pp-off.log" GGML_VK_CONCAT_FASTPATH=0 -- "${pp[@]}"
|
|
lb "$BUILD" "$S/cc-pp-cand.log" -- "${pp[@]}"
|
|
lb "$tb" "$S/cc-pp-off2.log" GGML_VK_CONCAT_FASTPATH=0 -- "${pp[@]}"
|
|
lb "$BUILD" "$S/cc-pp-cand2.log" -- "${pp[@]}"
|
|
cat "$S/cc-pp-off2.log" >> "$S/cc-pp-off.log"; cat "$S/cc-pp-cand2.log" >> "$S/cc-pp-cand.log"
|
|
|
|
local n_pp n_pp_small n_tg n_tg_concat
|
|
n_pp=$(grep -c 'transpose fast path' "$S/cc-pp-log.log")
|
|
n_pp_small=$(sed -n 's/.*transpose fast path, transposed ne0=\([0-9]*\).*/\1/p' "$S/cc-pp-log.log" | awk '$1 < 32' | wc -l)
|
|
n_tg=$(grep -c 'transpose fast path' "$S/cc-tg-log.log")
|
|
read -r n_tg_concat _ < <(concat_us "$S/cc-tg-log.log")
|
|
local ne0s; ne0s=$(sed -n 's/.*transposed ne0=\([0-9]*\).*/\1/p' "$S/cc-pp-log.log" | sort -n | uniq -c | awk '{printf "%sx%s ", $1, $2}')
|
|
check concat/prefill-fires $(( n_pp > 0 && n_pp_small == 0 ? 0 : 1 )) \
|
|
"$n_pp fast-path dispatches in the prefill run (transposed ne0: ${ne0s:-none})"
|
|
check concat/decode-skips $(( n_tg == 0 && n_tg_concat > 0 ? 0 : 1 )) \
|
|
"$n_tg fast-path dispatches over $n_tg_concat executed CONCAT nodes in the decode run"
|
|
|
|
local c_on c_off c_cand on off cand
|
|
read -r c_on on < <(concat_us "$S/cc-pp-log.log")
|
|
read -r c_off off < <(concat_us "$S/cc-pp-off.log")
|
|
read -r c_cand cand < <(concat_us "$S/cc-pp-cand.log")
|
|
local ok
|
|
ok=$(awk -v on="$on" -v off="$off" -v c="$cand" 'BEGIN {print (c > 0 && on > 0 && off/c >= 2 && c/on < 1.5 && on/c < 1.5) ? 0 : 1}')
|
|
check concat/candidate-timing "$ok" \
|
|
"prefill CONCAT avg: candidate ${cand} us, knob-on ${on} us, knob-off ${off} us (off/cand $(awk -v a="$off" -v b="$cand" 'BEGIN {printf "%.1f", (b > 0) ? a / b : 0}')x)"
|
|
local t_on t_off t_cand
|
|
t_on=$(grep -E '\| +pp512 ' "$S/cc-pp-log.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | head -1)
|
|
t_off=$(grep -E '\| +pp512 ' "$S/cc-pp-off.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | tr '\n' ' ')
|
|
t_cand=$(grep -E '\| +pp512 ' "$S/cc-pp-cand.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | tr '\n' ' ')
|
|
log "concat: pp512 t/s (perf logger on, so absolute speed is not representative): on $t_on | off $t_off | candidate $t_cand"
|
|
}
|
|
|
|
# ================================================================ 3. arbiter
|
|
ARB_ENV=(LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$ARENA" LLAMA_SLEEP_EVICT_KV=1 LLAMA_MMPROJ_ONDEMAND=1)
|
|
ARB_ARGS=(-m "$SMALL" -c 8192 -ngl 99 -fa 1 -lm none --parallel 1 --no-warmup --reasoning off)
|
|
A_PID= ; B_PID=
|
|
start_a() { start_server A "$BUILD" "$PA" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --mmproj "$MMPROJ"; A_PID=$SPID; }
|
|
start_b() { start_server B "$BUILD" "$PB" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --no-mmproj; B_PID=$SPID; }
|
|
sleeps() { grep -c 'server entering sleeping state' "$1"; }
|
|
|
|
make_red_ppm() { # 128x128 solid red, binary PPM (stb_image reads it; no image tools needed)
|
|
local f=$S/red.ppm i
|
|
printf '\377\000\000' > "$S/px"
|
|
for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14; do cat "$S/px" "$S/px" > "$S/px2"; mv "$S/px2" "$S/px"; done
|
|
{ printf 'P6\n128 128\n255\n'; cat "$S/px"; } > "$f"
|
|
base64 -w0 "$f"
|
|
}
|
|
|
|
run_arbiter() {
|
|
[ -f "$SMALL" ] && [ -f "$MMPROJ" ] || { fail arbiter "model/mmproj missing: $SMALL $MMPROJ"; return; }
|
|
local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 ))
|
|
if [ $free_mb -lt 1536 ]; then
|
|
fail arbiter "only $free_mb MiB VRAM free (< 1536): run inside the swapdown window"; return
|
|
fi
|
|
log "arbiter: $free_mb MiB VRAM free; starting A (mmproj) on :$PA"
|
|
start_a
|
|
wait_health "$PA" "$A_PID" 180 || { fail arbiter/start "A did not become healthy, see $S/A.log"; return; }
|
|
|
|
# no-handoff reference: A has held the token since its load and never released
|
|
local code ref_a
|
|
code=$(complete "$PA" "$S/ref-a.json" "$(greedy_body 48 false)")
|
|
ref_a=$(json_str content "$S/ref-a.json")
|
|
[ "$code" = 200 ] && [ -n "$ref_a" ] || { fail arbiter/start "reference request on A: HTTP $code"; return; }
|
|
[ "$(token_holder)" = "$A_PID" ] || { fail arbiter/start "A serves but does not hold token.lock"; return; }
|
|
|
|
log "arbiter: starting B (text-only) on :$PB"
|
|
start_b
|
|
wait_health "$PB" "$B_PID" 180 || { fail arbiter/start "B did not become healthy, see $S/B.log"; return; }
|
|
pass arbiter/start "A=$A_PID B=$B_PID up, reference output ${#ref_a} chars"
|
|
|
|
# ---- (a)+(b) alternating handoffs
|
|
local i port pid name bad=0 stranded=0 diff_a=0 diff_b=0 ref_b= lat=() holder_bad=0
|
|
for i in $(seq 0 30); do
|
|
if [ $((i % 2)) -eq 0 ]; then port=$PB pid=$B_PID name=B; else port=$PA pid=$A_PID name=A; fi
|
|
local t0 t1 out=$S/h$i.json
|
|
t0=$(now_ms)
|
|
code=$(complete "$port" "$out" "$(greedy_body 48 false)" 60)
|
|
t1=$(now_ms)
|
|
[ $i -gt 0 ] && lat+=($((t1 - t0)))
|
|
if [ "$code" = 000 ]; then stranded=$((stranded + 1)); continue; fi
|
|
[ "$code" != 200 ] && { bad=$((bad + 1)); continue; }
|
|
local c; c=$(json_str content "$out")
|
|
if [ $name = A ]; then [ "$c" = "$ref_a" ] || diff_a=$((diff_a + 1))
|
|
else if [ -z "$ref_b" ]; then ref_b=$c; elif [ "$c" != "$ref_b" ]; then diff_b=$((diff_b + 1)); fi; fi
|
|
[ "$(token_holder)" = "$pid" ] || holder_bad=$((holder_bad + 1))
|
|
done
|
|
local med max
|
|
med=$(printf '%s\n' "${lat[@]}" | sort -n | awk '{a[NR]=$1} END {print a[int((NR+1)/2)]}')
|
|
max=$(printf '%s\n' "${lat[@]}" | sort -n | tail -1)
|
|
check arbiter/a-handoffs $(( stranded == 0 && bad == 0 && holder_bad == 0 ? 0 : 1 )) \
|
|
"30 handoffs: $stranded stranded, $bad non-200, $holder_bad served-without-token; request latency median ${med} ms, max ${max} ms"
|
|
check arbiter/b-greedy-identical $(( diff_a == 0 && diff_b == 0 ? 0 : 1 )) \
|
|
"A: 15/15 == no-handoff reference ($diff_a differ); B: $diff_b of 15 differ from its first; A==B: $([ "$ref_a" = "$ref_b" ] && echo yes || echo no)"
|
|
|
|
# ---- (b') KV / prompt cache survives an evict + restore (LLAMA_SLEEP_EVICT_KV=1)
|
|
local p1 p2 c1 c2
|
|
code=$(complete "$PA" "$S/kv1.json" "$(greedy_body 32 true 'Explain in detail how a refrigerator moves heat from inside to outside')")
|
|
complete "$PB" "$S/kv-b.json" "$(greedy_body 8 false)" > /dev/null
|
|
[ "$(token_holder)" = "$B_PID" ] || log "warning: B does not hold the token before the KV re-check"
|
|
complete "$PA" "$S/kv2.json" "$(greedy_body 32 true 'Explain in detail how a refrigerator moves heat from inside to outside')" > /dev/null
|
|
p1=$(json_num prompt_n "$S/kv1.json"); p2=$(json_num prompt_n "$S/kv2.json")
|
|
c1=$(json_str content "$S/kv1.json"); c2=$(json_str content "$S/kv2.json")
|
|
check arbiter/b-kv-survives $(( ${p2:-99} < ${p1:-0} && ${#c1} > 0 ? 0 : 1 )) \
|
|
"prompt_n $p1 -> $p2 after evict/restore (cache reused), output $([ "$c1" = "$c2" ] && echo identical || echo DIFFERENT)"
|
|
[ "$c1" = "$c2" ] || fail arbiter/b-kv-output "cached-prompt output changed across evict/restore"
|
|
|
|
# ---- (c) image after release/restore
|
|
complete "$PB" "$S/c-b.json" "$(greedy_body 8 false)" > /dev/null # A goes cold
|
|
local cold_ok=1; [ "$(token_holder)" = "$B_PID" ] && cold_ok=0
|
|
local img; img=$(make_red_ppm)
|
|
printf '{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/x-portable-pixmap;base64,%s"}},{"type":"text","text":"What is the main color of this image? Answer with one word."}]}],"max_tokens":16,%s}' \
|
|
"$img" "$GREEDY" > "$S/c-body.json"
|
|
code=$(curl -sS -m 120 -o "$S/c-a.json" -w '%{http_code}' -H 'Content-Type: application/json' \
|
|
-d @"$S/c-body.json" "http://127.0.0.1:$PA/v1/chat/completions" 2>>"$S/curl.err")
|
|
local ans; ans=$(json_str content "$S/c-a.json")
|
|
check arbiter/c-image-after-restore $(( cold_ok == 0 && ${#code} > 0 && code == 200 ? 0 : 1 )) \
|
|
"A cold before: $([ $cold_ok -eq 0 ] && echo yes || echo NO), HTTP $code, answer: '$ans'"
|
|
echo "$ans" | grep -qi red || fail arbiter/c-image-answer "expected 'red', got '$ans'"
|
|
# text right after the image path (encoder out, weights back)
|
|
code=$(complete "$PA" "$S/c-a2.json" "$(greedy_body 48 false)")
|
|
[ "$code" = 200 ] && [ "$(json_str content "$S/c-a2.json")" = "$ref_a" ] \
|
|
|| fail arbiter/c-text-after-image "HTTP $code, output differs from reference after the image request"
|
|
|
|
# ---- (g) kill -9 a waiter that holds want.lock -> the holder stops yielding
|
|
stream_bg "$PA" "$S/g-a.sse" "$LONG_BODY"; local gcl=$CPID
|
|
if ! wait_tokens "$S/g-a.sse" 3 30000; then fail arbiter/g-kill9-waiter "A did not start streaming"; else
|
|
local s_before; s_before=$(sleeps "$S/A.log")
|
|
complete "$PB" "$S/g-b.json" "$(greedy_body 8 false)" 60 > /dev/null &
|
|
local gbc=$!; track "$gbc"
|
|
if ! wait_want "$B_PID" 10000; then
|
|
fail arbiter/g-kill9-waiter "B never registered on want.lock (setup)"
|
|
else
|
|
kill -KILL "$B_PID"; wait "$B_PID" 2>/dev/null
|
|
kill "$gcl" 2>/dev/null; wait "$gcl" 2>/dev/null # end A's generation (client abort)
|
|
sleep 2
|
|
local wl; wl=$(lock_pids "$ARENA/want.lock" READ | wc -l)
|
|
check arbiter/g-kill9-waiter $(( $(holder_num) == A_PID && $(sleeps "$S/A.log") == s_before && wl == 0 ? 0 : 1 )) \
|
|
"after kill -9 of waiting B: want.lock readers $wl, token holder $(token_holder) (A=$A_PID), A sleeps +$(( $(sleeps "$S/A.log") - s_before ))"
|
|
fi
|
|
wait "$gbc" 2>/dev/null
|
|
fi
|
|
log "arbiter: restarting B"
|
|
start_b
|
|
wait_health "$PB" "$B_PID" 180 || { fail arbiter/restart "B did not come back"; return; }
|
|
|
|
# ---- (e) SIGTERM while A waits for the token -> exits fast, never restores
|
|
stream_bg "$PB" "$S/e-b.sse" "$LONG_BODY"; local ecl=$CPID
|
|
wait_tokens "$S/e-b.sse" 3 30000 || { fail arbiter/e-term-waiting "B did not start streaming"; }
|
|
complete "$PA" "$S/e-a.json" "$(greedy_body 8 false)" 30 > /dev/null &
|
|
local eac=$!; track "$eac"
|
|
if wait_want "$A_PID" 10000; then
|
|
sleep 0.3
|
|
local t0 ms took_token=0
|
|
t0=$(now_ms); kill -TERM "$A_PID"
|
|
while pid_alive "$A_PID"; do
|
|
[ "$(token_holder)" = "$A_PID" ] && took_token=1
|
|
[ $(( $(now_ms) - t0 )) -gt 10000 ] && break
|
|
sleep 0.01
|
|
done
|
|
ms=$(( $(now_ms) - t0 )); wait "$A_PID"; RC=$?
|
|
check arbiter/e-term-waiting $(( RC == 0 && ms <= 1500 && took_token == 0 && $(holder_num) == B_PID ? 0 : 1 )) \
|
|
"exit $RC in ${ms} ms, A took the token: $([ $took_token = 1 ] && echo YES || echo no), holder now $(token_holder) (B=$B_PID), log: $(grep -c 'no longer waiting for the GPU token' "$S/A.log") abort line(s)"
|
|
else
|
|
fail arbiter/e-term-waiting "A never registered on want.lock (setup)"
|
|
stop_server "$A_PID"
|
|
fi
|
|
kill "$ecl" 2>/dev/null; wait "$ecl" "$eac" 2>/dev/null
|
|
log "arbiter: restarting A"
|
|
start_a
|
|
wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; }
|
|
|
|
# ---- (d) SIGTERM during A's VRAM release -> exit 0 within ~2 s, B gets the token
|
|
# A is warm (fresh load, first release also captures the host shadow: the longest release)
|
|
local how sig_ms
|
|
if command -v gdb > /dev/null; then
|
|
how=gdb
|
|
gdb -p "$A_PID" -batch -nx -ex 'set pagination off' -ex 'handle all nostop noprint pass' \
|
|
-ex 'break llama_context_release_device' -ex 'continue' \
|
|
-ex "shell date +%s%3N > $S/d.sig" -ex 'delete' -ex 'signal SIGTERM' > "$S/d-gdb.log" 2>&1 &
|
|
local gpid=$!; track "$gpid"
|
|
local k=0; until grep -q '^Continuing' "$S/d-gdb.log" || [ $k -gt 300 ]; do sleep 0.1; k=$((k + 1)); done
|
|
sleep 0.5
|
|
else
|
|
how=log-poll
|
|
fi
|
|
local s0; s0=$(sleeps "$S/A.log")
|
|
complete "$PB" "$S/d-b.json" "$(greedy_body 8 false)" 60 > "$S/d-b.code" &
|
|
local dbc=$!; track "$dbc"
|
|
if [ $how = log-poll ]; then
|
|
local k=0; until [ "$(sleeps "$S/A.log")" -gt "$s0" ] || [ $k -gt 2000 ]; do sleep 0.005; k=$((k + 1)); done
|
|
now_ms > "$S/d.sig"; kill -TERM "$A_PID"
|
|
else
|
|
local k=0; until [ -s "$S/d.sig" ] || [ $k -gt 600 ]; do sleep 0.05; k=$((k + 1)); done
|
|
fi
|
|
if [ ! -s "$S/d.sig" ]; then
|
|
fail arbiter/d-term-in-release "release breakpoint never hit ($how), see $S/d-gdb.log"
|
|
kill "$gpid" 2>/dev/null; stop_server "$A_PID"
|
|
else
|
|
while pid_alive "$A_PID" && [ $(( $(now_ms) - $(cat "$S/d.sig") )) -lt 20000 ]; do sleep 0.01; done
|
|
sig_ms=$(( $(now_ms) - $(cat "$S/d.sig") ))
|
|
wait "$A_PID"; RC=$?
|
|
wait "$dbc"
|
|
local dcode; dcode=$(cat "$S/d-b.code")
|
|
sleep 0.2
|
|
check arbiter/d-term-in-release $(( RC == 0 && sig_ms <= 2500 && dcode == 200 && $(holder_num) == B_PID ? 0 : 1 )) \
|
|
"SIGTERM inside llama_context_release_device ($how): A exit $RC in ${sig_ms} ms, B HTTP $dcode, holder $(token_holder) (B=$B_PID)"
|
|
fi
|
|
log "arbiter: restarting A"
|
|
start_a
|
|
wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; }
|
|
|
|
# ---- (f) SIGTERM mid-generation (streaming)
|
|
stream_bg "$PA" "$S/f-a.sse" "$LONG_BODY"; local fcl=$CPID
|
|
if ! wait_tokens "$S/f-a.sse" 10 30000; then fail arbiter/f-term-streaming "A did not start streaming"; else
|
|
local t0 free_ms=-1 cl_ms=-1 exit_ms=-1
|
|
t0=$(now_ms); kill -TERM "$A_PID"
|
|
while [ $(( $(now_ms) - t0 )) -lt 10000 ]; do
|
|
[ $free_ms -lt 0 ] && [ -z "$(token_holder)" ] && free_ms=$(( $(now_ms) - t0 ))
|
|
[ $cl_ms -lt 0 ] && ! pid_alive "$fcl" && cl_ms=$(( $(now_ms) - t0 ))
|
|
[ $exit_ms -lt 0 ] && ! pid_alive "$A_PID" && exit_ms=$(( $(now_ms) - t0 ))
|
|
[ $free_ms -ge 0 ] && [ $cl_ms -ge 0 ] && [ $exit_ms -ge 0 ] && break
|
|
sleep 0.01
|
|
done
|
|
wait "$A_PID"; RC=$?; wait "$fcl" 2>/dev/null
|
|
local errev; errev=$(grep -c '"error"' "$S/f-a.sse")
|
|
check arbiter/f-term-streaming $(( RC == 0 && errev > 0 && cl_ms >= 0 && free_ms >= 0 && free_ms <= 2500 && exit_ms >= 0 ? 0 : 1 )) \
|
|
"A exit $RC in ${exit_ms} ms, token free in ${free_ms} ms, client closed in ${cl_ms} ms with $errev error event(s) after $(grep -c '^data: ' "$S/f-a.sse") data lines"
|
|
fi
|
|
log "arbiter: restarting A"
|
|
start_a
|
|
wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; }
|
|
|
|
# ---- (h) clean shutdown warm and cold
|
|
[ "$(token_holder)" = "$A_PID" ] || log "warning: A is not warm before the warm-shutdown case"
|
|
stop_server "$A_PID"
|
|
check arbiter/h-shutdown-warm $(( RC == 0 ? 0 : 1 )) "warm A: exit $RC in ${EXIT_MS} ms"
|
|
start_a
|
|
wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; }
|
|
complete "$PB" "$S/h-b.json" "$(greedy_body 8 false)" > /dev/null # A goes cold
|
|
local acold=no; [ "$(token_holder)" = "$B_PID" ] && acold=yes
|
|
stop_server "$A_PID"
|
|
check arbiter/h-shutdown-cold $([ $RC -eq 0 ] && [ $acold = yes ] && echo 0 || echo 1) \
|
|
"cold A (B holds token: $acold): exit $RC in ${EXIT_MS} ms"
|
|
stop_server "$B_PID"
|
|
check arbiter/h-shutdown-warm-b $(( RC == 0 ? 0 : 1 )) "warm B: exit $RC in ${EXIT_MS} ms"
|
|
|
|
local bad_lines; bad_lines=$(scan_logs "$S/A.log" "$S/B.log")
|
|
check arbiter/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) \
|
|
"$([ -z "$bad_lines" ] && echo 'no asserts / Vulkan errors in A/B logs' || echo "$bad_lines" | head -3 | tr '\n' ' ')"
|
|
|
|
run_handoff_ab
|
|
}
|
|
|
|
# handoff_block BUILD TAG N : a fresh pair of BUILD on the scratch arena (A with mmproj, B text-only,
|
|
# the same config as case a), one warm-up request on A, then N+1 alternating requests starting with
|
|
# B. Request 0 is not a handoff (B holds the token after its load); the latency of requests 1..N is
|
|
# appended to $S/lat-TAG.txt. HB_BAD counts setup failures, non-200s and answers without the token.
|
|
handoff_block() {
|
|
local build=$1 tag=$2 n=$3 a b i port pid code t0 t1
|
|
HB_BAD=0
|
|
: > "$S/lat-$tag.txt"
|
|
start_server "L$tag-A" "$build" "$PA" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --mmproj "$MMPROJ"; a=$SPID
|
|
if ! wait_health "$PA" "$a" 180; then HB_BAD=1; stop_server "$a"; return 1; fi
|
|
complete "$PA" "$S/L$tag-warm.json" "$(greedy_body 8 false)" > /dev/null
|
|
start_server "L$tag-B" "$build" "$PB" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --no-mmproj; b=$SPID
|
|
if ! wait_health "$PB" "$b" 180; then HB_BAD=1; stop_server "$b"; stop_server "$a"; return 1; fi
|
|
for i in $(seq 0 "$n"); do
|
|
if [ $((i % 2)) -eq 0 ]; then port=$PB pid=$b; else port=$PA pid=$a; fi
|
|
t0=$(now_ms)
|
|
code=$(complete "$port" "$S/L$tag-$i.json" "$(greedy_body 48 false)" 60)
|
|
t1=$(now_ms)
|
|
if [ "$code" != 200 ] || [ "$(token_holder)" != "$pid" ]; then HB_BAD=$((HB_BAD + 1)); continue; fi
|
|
[ "$i" -gt 0 ] && echo $((t1 - t0)) >> "$S/lat-$tag.txt"
|
|
done
|
|
stop_server "$b"; stop_server "$a"
|
|
}
|
|
|
|
# ---- (a') ordinary handoff latency, candidate vs reference, in one session
|
|
# Two production deploys were lost to arbiter changes that passed the pathological cases but made
|
|
# every ordinary handoff slower. Blocks run ref, cand, cand, ref (a fixed order once gave the later
|
|
# slot about 9%); each build's median is over all its handoffs. FAIL if the candidate median exceeds
|
|
# the reference median by more than max(25%, 200 ms): small absolute numbers are noisy.
|
|
run_handoff_ab() {
|
|
local n=16 blk k=0 bad_r=0 bad_c=0 tag
|
|
log "arbiter: handoff latency A/B, blocks ref,cand,cand,ref x $n handoffs"
|
|
: > "$S/lat-ref.txt"; : > "$S/lat-cand.txt"
|
|
for blk in ref cand cand ref; do
|
|
k=$((k + 1)); tag=$k-$blk
|
|
local b; [ $blk = ref ] && b=$REF || b=$BUILD
|
|
handoff_block "$b" "$tag" "$n"
|
|
if [ $blk = ref ]; then bad_r=$((bad_r + HB_BAD)); else bad_c=$((bad_c + HB_BAD)); fi
|
|
cat "$S/lat-$tag.txt" >> "$S/lat-$blk.txt"
|
|
log "arbiter: block $tag median $(median < "$S/lat-$tag.txt") ms over $(wc -l < "$S/lat-$tag.txt") handoffs, $HB_BAD bad"
|
|
done
|
|
local mr mc nr nc maxr maxc limit
|
|
mr=$(median < "$S/lat-ref.txt"); mc=$(median < "$S/lat-cand.txt")
|
|
nr=$(wc -l < "$S/lat-ref.txt"); nc=$(wc -l < "$S/lat-cand.txt")
|
|
maxr=$(sort -n "$S/lat-ref.txt" | tail -1); maxc=$(sort -n "$S/lat-cand.txt" | tail -1)
|
|
if [ -z "$mr" ] || [ -z "$mc" ] || [ $bad_r -gt 0 ] || [ $bad_c -gt 0 ]; then
|
|
fail arbiter/a-handoff-latency "blocks incomplete: reference $nr handoffs ($bad_r bad), candidate $nc ($bad_c bad); see $S/L*-*.log"
|
|
return
|
|
fi
|
|
limit=$(awk -v r="$mr" 'BEGIN {d = r * 0.25; if (d < 200) d = 200; printf "%.0f", r + d}')
|
|
check arbiter/a-handoff-latency "$(awk -v c="$mc" -v l="$limit" 'BEGIN {print (c > l) ? 1 : 0}')" \
|
|
"median handoff latency candidate ${mc} ms (n $nc, max $maxc) vs reference ${mr} ms (n $nr, max $maxr); limit ${limit} ms = ref + max(25%, 200 ms)"
|
|
local bad_lines; bad_lines=$(scan_logs "$S"/L*-A.log "$S"/L*-B.log)
|
|
[ -z "$bad_lines" ] || fail arbiter/a-handoff-latency-logs "$(echo "$bad_lines" | head -3 | tr '\n' ' ')"
|
|
}
|
|
|
|
# ================================================================ 4. perf
|
|
vram_sampler() { # OUTFILE : max of mem_info_vram_used every 0.25 s, written continuously
|
|
local m=0 v
|
|
while :; do v=$(cat "$VRAM_USED"); [ "$v" -gt "$m" ] && { m=$v; echo "$m" > "$1"; }; sleep 0.25; done
|
|
}
|
|
# median of the numbers on stdin (one per line); empty input -> empty output
|
|
median() { sort -g | awk '{a[NR] = $1} END {if (NR) printf "%.2f", (NR % 2) ? a[(NR + 1) / 2] : (a[NR / 2] + a[NR / 2 + 1]) / 2}'; }
|
|
bench_samples() { # JSONL (llama-bench -o jsonl) -> "test<TAB>t/s", one row per repetition sample
|
|
sed -n 's/.*"n_prompt": *\([0-9]*\),.*"n_gen": *\([0-9]*\),.*"n_depth": *\([0-9]*\),.*"samples_ts": *\[\([^]]*\)\].*/\1 \2 \3 \4/p' "$1" |
|
|
awk '{t = ($1 > 0) ? "pp" $1 : "tg" $2; if ($3 > 0) t = t " @ d" $3
|
|
for (i = 4; i <= NF; i++) {v = $i; gsub(/,/, "", v); if (v != "") print t "\t" v}}'
|
|
}
|
|
run_perf() {
|
|
if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then
|
|
fail perf "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return
|
|
fi
|
|
local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l)
|
|
[ "$others" -eq 0 ] || { fail perf "$others other llama processes are running"; return; }
|
|
[ -f "$PERF_MODEL" ] || { fail perf "model missing: $PERF_MODEL"; return; }
|
|
[ -x "$REF/bin/llama-bench" ] && [ -x "$BUILD/bin/llama-bench" ] || { fail perf "llama-bench missing"; return; }
|
|
local common=(-m "$PERF_MODEL" -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -o jsonl -oe md)
|
|
local arm k=0 base; base=$(vram_mb)
|
|
log "perf: idle VRAM $base MiB; arms ref,cand,cand,ref"
|
|
for arm in ref cand cand ref; do
|
|
k=$((k + 1))
|
|
local b; [ $arm = ref ] && b=$REF || b=$BUILD
|
|
local tag=$k-$arm
|
|
vram_sampler "$S/perf-$tag.vram" & local vs=$!; track "$vs"
|
|
# stdout: one JSON line per test with every repetition sample; stderr: the markdown table + logs
|
|
log "perf: arm $tag pp sweep"
|
|
LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 2048 -n 0 -d 0,16384,32768 -r 2 \
|
|
> "$S/perf-$tag-pp.jsonl" 2> "$S/perf-$tag-pp.log"
|
|
log "perf: arm $tag tg"
|
|
LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 0 -n 256 -d 4096 -r 3 \
|
|
> "$S/perf-$tag-tg.jsonl" 2> "$S/perf-$tag-tg.log"
|
|
kill "$vs" 2>/dev/null; wait "$vs" 2>/dev/null
|
|
{ bench_samples "$S/perf-$tag-pp.jsonl"; bench_samples "$S/perf-$tag-tg.jsonl"; } | sed "s/^/$arm\t/" >> "$S/perf.tsv"
|
|
echo "$arm $(cat "$S/perf-$tag.vram" 2>/dev/null || echo 0)" >> "$S/perf-vram.txt"
|
|
done
|
|
perf_report "$S/perf.tsv"
|
|
local total; total=$(cat "$VRAM_TOTAL")
|
|
local vr vc
|
|
vr=$(awk '$1=="ref" && $2>m {m=$2} END {print m+0}' "$S/perf-vram.txt")
|
|
vc=$(awk '$1=="cand" && $2>m {m=$2} END {print m+0}' "$S/perf-vram.txt")
|
|
check perf/peak-vram $(( vc * 100 <= total * 95 ? 0 : 1 )) \
|
|
"peak VRAM reference $((vr / 1048576)) MiB, candidate $((vc / 1048576)) MiB of $((total / 1048576)) MiB (limit 95%)"
|
|
}
|
|
# perf_report TSV : "arm<TAB>test<TAB>t/s" rows, one per llama-bench repetition sample (arm = ref|cand).
|
|
# Gates on the MEDIAN of all samples of a build (pooled over its arms): a single catastrophic
|
|
# sample (CPU contention from the other pods on the node) moves a mean, not a median.
|
|
perf_report() {
|
|
local tsv=$1
|
|
echo
|
|
echo "perf: median t/s over every repetition sample of both arms (order ref,cand,cand,ref)"
|
|
printf '%-18s %24s %24s %8s\n' test "reference med (min-max, n)" "candidate med (min-max, n)" delta
|
|
local tests t r c d rs cs
|
|
tests=$(awk -F'\t' '{print $2}' "$tsv" | awk '!s[$0]++')
|
|
local IFS_=$IFS; IFS=$'\n'
|
|
for t in $tests; do
|
|
r=$(awk -F'\t' -v t="$t" '$1=="ref" && $2==t {print $3}' "$tsv" | median)
|
|
c=$(awk -F'\t' -v t="$t" '$1=="cand" && $2==t {print $3}' "$tsv" | median)
|
|
rs=$(awk -F'\t' -v t="$t" '$1=="ref" && $2==t {n++; if (n == 1 || $3 < lo) lo = $3; if (n == 1 || $3 > hi) hi = $3} END {if (n) printf "%.1f-%.1f, %d", lo, hi, n}' "$tsv")
|
|
cs=$(awk -F'\t' -v t="$t" '$1=="cand" && $2==t {n++; if (n == 1 || $3 < lo) lo = $3; if (n == 1 || $3 > hi) hi = $3} END {if (n) printf "%.1f-%.1f, %d", lo, hi, n}' "$tsv")
|
|
d=$(awk -v r="$r" -v c="$c" 'BEGIN {if (r > 0) printf "%+.1f%%", (c/r - 1)*100}')
|
|
printf '%-18s %24s %24s %8s\n' "$t" "$r ($rs)" "$c ($cs)" "$d"
|
|
case "$t" in pp*)
|
|
if [ -z "$r" ] || [ -z "$c" ] || [ "$(awk -v r="$r" -v c="$c" 'BEGIN {print (c < 0.95*r)}')" = 1 ]; then
|
|
fail "perf/$t" "candidate median $c vs reference median $r t/s ($d, limit -5%)"
|
|
else
|
|
pass "perf/$t" "candidate median $c vs reference median $r t/s ($d)"
|
|
fi ;;
|
|
tg*) RESULTS+=("INFO perf/$t: candidate median $c vs reference median $r t/s ($d; decode is CPU-bound and noisy here, not gated)") ;;
|
|
esac
|
|
done
|
|
IFS=$IFS_
|
|
local n_tests; n_tests=$(echo "$tests" | grep -c '^pp')
|
|
[ "$n_tests" -ge 3 ] || fail perf/rows "expected 3 pp rows, got $n_tests (see $S/perf-*-pp.log)"
|
|
}
|
|
|
|
# ================================================================ 5. MoE cache release/restore smoke
|
|
run_moecache() {
|
|
[ -f "$MOECACHE_MODEL" ] || { fail moecache "model missing: $MOECACHE_MODEL"; return; }
|
|
local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 ))
|
|
log "moecache: $free_mb MiB VRAM free; server on :$PC with --moe-cache-mib $MOECACHE_MIB"
|
|
local arena=$S/arena-moec; mkdir -p "$arena"
|
|
# -lv 4: the cache size and hit/miss stats are only logged at that verbosity
|
|
# shellcheck disable=SC2086
|
|
start_server M "$BUILD" "$PC" LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$arena" LLAMA_SLEEP_EVICT_KV=1 -- \
|
|
-m "$MOECACHE_MODEL" $MOECACHE_ARGS -fa 1 -lm none --parallel 1 --no-warmup --reasoning off -lv 4 \
|
|
--moe-cache-mib "$MOECACHE_MIB"
|
|
local mp=$SPID
|
|
wait_health "$PC" "$mp" 600 || { fail moecache "server did not start, see $S/M.log"; return; }
|
|
# chat endpoint: an instruct model fed a raw prompt can loop, which says nothing about the cache
|
|
local body="{\"messages\":[{\"role\":\"user\",\"content\":\"What are the three primary colors of light? Answer in one short sentence.\"}],\"max_tokens\":64,$GREEDY}"
|
|
local outs=() i code w
|
|
for i in 0 1 2; do
|
|
code=$(curl -sS -m 300 -o "$S/m$i.json" -w '%{http_code}' -H 'Content-Type: application/json' \
|
|
-d "$body" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err")
|
|
outs+=("$(json_str content "$S/m$i.json")")
|
|
[ "$code" = 200 ] || { fail moecache "request $i: HTTP $code"; stop_server "$mp"; return; }
|
|
[ $i -eq 2 ] && break
|
|
# force a release: a fake waiter holds want.lock shared until the server drops the token
|
|
( exec 9<"$arena/want.lock"; flock -s 9; exec sleep 120 ) & w=$!; track "$w"
|
|
local k=0; until [ -z "$(awk -v id="$(lock_id "$arena/token.lock")" '$2=="FLOCK" && $4=="WRITE" && $6==id' /proc/locks)" ] || [ $k -gt 600 ]; do sleep 0.05; k=$((k + 1)); done
|
|
log "moecache: cycle $((i + 1)) cold, VRAM $(vram_mb) MiB"
|
|
kill "$w"; wait "$w" 2>/dev/null
|
|
done
|
|
stop_server "$mp"
|
|
local size hits rel res
|
|
size=$(sed -n 's/.*MoE cache size = *\([0-9.]*\) MiB.*/\1/p' "$S/M.log" | head -1)
|
|
hits=$(sed -n 's/.*llama_moe_cache: .* hits = \([0-9]*\),.*/\1/p' "$S/M.log" | awk '{s += $1} END {print s+0}')
|
|
rel=$(grep -c 'released .* MiB of MoE cache' "$S/M.log"); res=$(grep -c 'restored .* MiB of MoE cache' "$S/M.log")
|
|
check moecache/active $(( hits > 0 ? 0 : 1 )) \
|
|
"cache ${size:-?} MiB, $hits hits (0 hits = cache too small to ever reuse an expert, so a stale SLRU would go unnoticed), released ${rel}x restored ${res}x"
|
|
local same=1; [ "${outs[0]}" = "${outs[1]}" ] && [ "${outs[1]}" = "${outs[2]}" ] && [ -n "${outs[0]}" ] && same=0
|
|
check moecache/identical $same "3 greedy answers across 2 cold/warm cycles $([ $same = 0 ] && echo identical || echo DIFFER): '$(echo "${outs[0]}" | cut -c1-80)'"
|
|
local coh=1; echo "${outs[2]}" | grep -qi red && echo "${outs[2]}" | grep -qi green && echo "${outs[2]}" | grep -qi blue && coh=0
|
|
check moecache/coherent $coh "answer after the 2nd restore names red, green and blue"
|
|
check moecache/shutdown $(( RC == 0 ? 0 : 1 )) "exit $RC"
|
|
local bad_lines; bad_lines=$(scan_logs "$S/M.log")
|
|
check moecache/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}"
|
|
}
|
|
|
|
# ================================================================ main
|
|
if [ -n "$PERF_REPORT" ]; then
|
|
[ -r "$PERF_REPORT" ] || { echo "cannot read $PERF_REPORT" >&2; exit 2; }
|
|
perf_report "$PERF_REPORT"; exit 0
|
|
fi
|
|
[ $DO_OPS -eq 1 ] && run_ops
|
|
[ $DO_CONCAT -eq 1 ] && run_concat
|
|
[ $DO_ARB -eq 1 ] && run_arbiter
|
|
[ $DO_PERF -eq 1 ] && run_perf
|
|
[ $DO_MOEC -eq 1 ] && run_moecache
|
|
exit 0
|