scripts: fork-validate.sh - --moe-exact, greedy identity for the readback skip

F1 (the routing-ids readback skip in sched_copy_experts) had no automated
check. --moe-exact MODEL reads expert_count / expert_used_count from the GGUF
header, computes the skip threshold ceil(4 * n_expert / n_expert_used) tokens
per ubatch (128 on the 35B), and runs one greedy request with a prompt of
about 2x that in a single ubatch, every expert on the host, so the prefill
takes the skip and each decode step takes the readback. The candidate's 64
tokens must equal the reference's byte for byte; with the LLAMA_MOE_IDS_SKIP
knob in the candidate, also the candidate with the skip forced off (=0), which
proves the skip exact even when the reference carries it, and =log must show
skips, all at >= 4 * n_expert ids. Without the knob skip-engaged SKIPs with
the batch-size argument. It needs the GPU to itself (same guard as --perf,
now shared as gpu_exclusive); FV_TEST_SHARED_GPU=1 bypasses the guard for
script tests against stub binaries only.

Also unset inherited LLAMA_MOE* variables.

Not run on the GPU yet (needs a swapdown window). Dry-run on the pod
against stub servers and the real 35B GGUF header: threshold 128, all-PASS
when outputs match, FAIL on a differing reference, SKIP without the knob,
FAIL for a dense model and while llama-swap runs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
LumpiastyandClaude Opus 5.5 committed 2026-10-25 02:47:16 +02:00
1 parent ceab1a91f8
commit e5fc474b87
1 file changed
+115 -9
+115 -9
View File
@@ -40,6 +40,15 @@
# one arbiter server with --moe-cache-mib N on a MoE model, forced cold/warm twice
# -> greedy output identical before/after (the expert-cache SLRU is re-initialised on
# release; a stale SLRU maps experts to empty banks and produces garbage).
# --moe-exact MODEL
# the MoE routing-ids readback skip (F1, sched_copy_experts) is exact: MODEL (a MoE,
# e.g. the --perf model) with every expert on the host, greedy, one prompt of >= 2x
# the skip threshold ceil(4 * n_expert / n_expert_used) tokens (read from the GGUF;
# 128 on the 35B) -> candidate output byte-identical to the reference, and, if the
# candidate has the LLAMA_MOE_IDS_SKIP knob, to the candidate with the skip forced off
# (=0); =log proves the skip engaged. Without the knob skip-engaged is SKIPped with the
# batch-size argument. Needs the GPU to itself like --perf (same guard); about 3 model
# loads, so run it in the same swapdown window as --perf.
#
# Options:
# --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build)
@@ -50,11 +59,17 @@
# --moecache-mib N cache size for --moecache (default 2048; 256 never hits on a 26B and proves nothing)
# --port-base N scratch ports N..N+2 (default 8091)
# --keep keep the scratch dir even when everything passed (it is kept on any FAIL)
# --moe-exact-args S server args for --moe-exact (default: "-ngl 99 --n-cpu-moe 99 -c 4096 -b 2048
# -ub 2048 -t 6"; keep the experts on the host and -ub >= the prompt)
# --perf-report TSV only re-aggregate a saved <scratch>/perf.tsv of an earlier --perf run and
# apply the same gates (no GPU, no --build/--ref needed); runs nothing else
#
# Requirements: bash, coreutils, awk, sed, grep, curl, flock(1); gdb optional (makes the
# SIGTERM-during-release case deterministic). No python/jq.
# SIGTERM-during-release case deterministic; it is not in the pod image and is lost on a pod restart,
# then the case falls back to polling the log). No python/jq.
#
# Test only: FV_TEST_SHARED_GPU=1 skips the exclusive-GPU guard of --perf / --moe-exact, for running
# the script against stub binaries. Never set it for a real measurement.
set -u
@@ -67,10 +82,12 @@ PERF_MODEL=$HF/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906
MOECACHE_MODEL=
MOECACHE_ARGS="-ngl 99 --n-cpu-moe 30 -c 4096"
MOECACHE_MIB=2048
MOEX_MODEL=
MOEX_ARGS="-ngl 99 --n-cpu-moe 99 -c 4096 -b 2048 -ub 2048 -t 6"
BUILD= ; REF= ; CONCAT_BUILD=
PORT_BASE=8091
KEEP=0
DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0
DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 DO_MOEX=0
PERF_REPORT=
usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; }
@@ -86,6 +103,8 @@ while [ $# -gt 0 ]; do
--moecache) DO_MOEC=1; MOECACHE_MODEL=$2; shift 2 ;;
--moecache-args) MOECACHE_ARGS=$2; shift 2 ;;
--moecache-mib) MOECACHE_MIB=$2; shift 2 ;;
--moe-exact) DO_MOEX=1; MOEX_MODEL=$2; shift 2 ;;
--moe-exact-args) MOEX_ARGS=$2; shift 2 ;;
--concat-build) CONCAT_BUILD=$2; shift 2 ;;
--small) SMALL=$2; shift 2 ;;
--mmproj) MMPROJ=$2; shift 2 ;;
@@ -98,7 +117,7 @@ while [ $# -gt 0 ]; do
esac
done
[ -n "$PERF_REPORT" ] || { [ -n "$BUILD" ] && [ -n "$REF" ]; } || usage
if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC)) -eq 0 ]; then
if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC + DO_MOEX)) -eq 0 ]; then
DO_OPS=1 DO_CONCAT=1 DO_ARB=1
fi
[ -n "$MMPROJ" ] || MMPROJ=$(dirname "$SMALL")/mmproj-F16.gguf
@@ -173,7 +192,7 @@ trap finish EXIT
trap 'echo "interrupted"; NFAIL=$((NFAIL + 1)); exit 130' INT TERM HUP
# ---------------------------------------------------------------- environment sanity
for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\)[A-Za-z0-9_]*\)=.*/\1/p'); do
for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\|MOE\)[A-Za-z0-9_]*\)=.*/\1/p'); do
log "unsetting inherited $v"; unset "$v"
done
[ -n "$PERF_REPORT" ] || for d in "$BUILD" "$REF"; do
@@ -722,11 +741,7 @@ bench_samples() { # JSONL (llama-bench -o jsonl) -> "test<TAB>t/s", one row per
for (i = 4; i <= NF; i++) {v = $i; gsub(/,/, "", v); if (v != "") print t "\t" v}}'
}
run_perf() {
if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then
fail perf "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return
fi
local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l)
[ "$others" -eq 0 ] || { fail perf "$others other llama processes are running"; return; }
gpu_exclusive perf || return
[ -f "$PERF_MODEL" ] || { fail perf "model missing: $PERF_MODEL"; return; }
[ -x "$REF/bin/llama-bench" ] && [ -x "$BUILD/bin/llama-bench" ] || { fail perf "llama-bench missing"; return; }
local common=(-m "$PERF_MODEL" -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -o jsonl -oe md)
@@ -832,6 +847,96 @@ run_moecache() {
check moecache/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}"
}
# ================================================================ 6. MoE routing-ids readback skip (F1) exactness
# sched_copy_experts skips the ids readback when a split's ids tensor has n_tokens * n_expert_used >=
# 4 * n_expert entries, i.e. for any ubatch of at least ceil(4 * n_expert / n_expert_used) tokens
# (128 on a 256-expert / 8-used model). Method: the model with every MoE layer's experts on the host
# (so the expert-copy callback runs), greedy, one prompt of >= 2x that many tokens in a single
# ubatch: the prefill takes the skip and every decode step takes the readback. The candidate's
# output must be byte-identical to the reference's and, when the build has the LLAMA_MOE_IDS_SKIP
# knob, to the candidate with the skip forced off (=0), which proves the skip itself is exact even
# when the reference build carries it too; =log then shows the skip engaged.
gguf_u32() { # FILE KEY-SUFFIX -> uint32 value of the first GGUF metadata key ending in KEY-SUFFIX
local off t
off=$(head -c 67108864 "$1" | grep -abo -m1 -F "$2" | head -1 | cut -d: -f1)
[ -n "$off" ] || return 1
off=$((off + ${#2})) # key string -> uint32 value type -> value
t=$(od -An -tu4 -j "$off" -N4 "$1" | tr -d ' ')
[ "$t" = 4 ] || [ "$t" = 5 ] || return 1 # GGUF_TYPE_UINT32 / INT32
od -An -tu4 -j $((off + 4)) -N4 "$1" | tr -d ' '
}
gpu_exclusive() { # CHECK-NAME -> FAILs it and returns 1 unless nothing else can use the GPU
[ "${FV_TEST_SHARED_GPU:-}" = 1 ] && { log "FV_TEST_SHARED_GPU=1: skipping the exclusive-GPU guard (script testing only)"; return 0; }
if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then
fail "$1" "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return 1
fi
local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l)
[ "$others" -eq 0 ] || { fail "$1" "$others other llama processes are running"; return 1; }
}
moex_prompt() { # N_LINES -> a deterministic prose-like prompt (JSON-safe: no quotes or backslashes)
local i
for i in $(seq 1 "$1"); do
printf 'Entry %d: the river town of Station%d ships %d barrels of grain and %d crates of salt each week to the harbour, and its council meets on day %d of the month. ' \
"$i" "$i" $(( (i * 37) % 101 + 3 )) $(( (i * 53) % 89 + 2 )) $(( (i * 7) % 28 + 1 ))
done
printf 'Question: which town ships the most barrels of grain each week? Answer:'
}
run_moe_exact() {
gpu_exclusive moe-exact || return
[ -f "$MOEX_MODEL" ] || { fail moe-exact "model missing: $MOEX_MODEL"; return; }
local n_exp n_used
n_exp=$(gguf_u32 "$MOEX_MODEL" .expert_count); n_used=$(gguf_u32 "$MOEX_MODEL" .expert_used_count)
if [ -z "$n_exp" ] || [ -z "$n_used" ] || [ "$n_exp" -eq 0 ] || [ "$n_used" -eq 0 ]; then
fail moe-exact "no expert_count / expert_used_count in $MOEX_MODEL (not a MoE model?)"; return
fi
local thr=$(( (4 * n_exp + n_used - 1) / n_used )) # tokens per ubatch where the skip starts
local lines=$(( (2 * thr + 39) / 40 + 2 )) # ~45 tokens per line -> >= 2x the threshold
local body
body="{\"prompt\":\"$(moex_prompt "$lines")\",\"n_predict\":64,$GREEDY,\"cache_prompt\":false}"
local knob=0; grep -qa LLAMA_MOE_IDS_SKIP "$BUILD/bin/libllama.so" 2>/dev/null && knob=1
log "moe-exact: n_expert $n_exp, n_expert_used $n_used -> skip at >= $thr tokens per ubatch; prompt of $lines lines; knob in candidate: $([ $knob = 1 ] && echo yes || echo no)"
# arm NAME BUILD ENV... : one server, one request; sets MX_CONTENT / MX_PN, returns 1 on failure
mx_arm() {
local name=$1 b=$2; shift 2
# shellcheck disable=SC2086
start_server "$name" "$b" "$PC" "$@" -- -m "$MOEX_MODEL" $MOEX_ARGS -fa 1 -lm none --parallel 1 --no-warmup
local p=$SPID code
if ! wait_health "$PC" "$p" 900; then fail "moe-exact/$name" "server did not start, see $S/$name.log"; stop_server "$p"; return 1; fi
code=$(complete "$PC" "$S/$name.json" "$body" 600)
stop_server "$p"
MX_CONTENT=$(json_str content "$S/$name.json"); MX_PN=$(json_num prompt_n "$S/$name.json")
[ "$code" = 200 ] && [ -n "$MX_CONTENT" ] || { fail "moe-exact/$name" "HTTP $code, empty output; see $S/$name.log"; return 1; }
[ "$RC" -eq 0 ] || log "warning: $name exited $RC"
}
local c_cand c_ref c_off pn
if [ $knob = 1 ]; then mx_arm X-cand "$BUILD" LLAMA_MOE_IDS_SKIP=log || return
else mx_arm X-cand "$BUILD" || return; fi
c_cand=$MX_CONTENT; pn=${MX_PN:-0}
if [ "$pn" -lt "$thr" ]; then
fail moe-exact/prompt "prompt_n $pn < threshold $thr: the skip never ran, nothing was tested"; return
fi
mx_arm X-ref "$REF" || return
c_ref=$MX_CONTENT
check moe-exact/identical-ref $([ "$c_cand" = "$c_ref" ] && echo 0 || echo 1) \
"prompt_n $pn (>= $thr), 64 greedy tokens: candidate $([ "$c_cand" = "$c_ref" ] && echo == || echo '!=') reference ('$(echo "$c_cand" | cut -c1-60)')"
if [ $knob = 1 ]; then
mx_arm X-cand-noskip "$BUILD" LLAMA_MOE_IDS_SKIP=0 || return
c_off=$MX_CONTENT
check moe-exact/identical-noskip $([ "$c_cand" = "$c_off" ] && echo 0 || echo 1) \
"candidate with the skip (LLAMA_MOE_IDS_SKIP=log) $([ "$c_cand" = "$c_off" ] && echo == || echo '!=') candidate with the readback forced (=0)"
local n_skip n_small
n_skip=$(grep -c 'sched_copy_experts: ids readback skipped' "$S/X-cand.log")
n_small=$(sed -n 's/.*ids readback skipped, n_ids=\([0-9]*\) n_expert=\([0-9]*\).*/\1 \2/p' "$S/X-cand.log" | awk '$1 < 4 * $2' | wc -l)
check moe-exact/skip-engaged $(( n_skip > 0 && n_small == 0 ? 0 : 1 )) \
"$n_skip skips logged ($n_small below 4*n_expert), $(grep -c 'ids readback skipped' "$S/X-cand-noskip.log") with the skip forced off"
else
skip moe-exact/skip-engaged "candidate has no LLAMA_MOE_IDS_SKIP knob to observe it; by construction the prefill ubatch had $pn x $n_used = $((pn * n_used)) ids >= 4 x $n_exp, so the skip branch ran"
fi
local bad_lines; bad_lines=$(scan_logs "$S"/X-*.log)
check moe-exact/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}"
}
# ================================================================ main
if [ -n "$PERF_REPORT" ]; then
[ -r "$PERF_REPORT" ] || { echo "cannot read $PERF_REPORT" >&2; exit 2; }
@@ -842,4 +947,5 @@ fi
[ $DO_ARB -eq 1 ] && run_arbiter
[ $DO_PERF -eq 1 ] && run_perf
[ $DO_MOEC -eq 1 ] && run_moecache
[ $DO_MOEX -eq 1 ] && run_moe_exact
exit 0