From e5fc474b871afa9fa5466e34570e35e840cdb83e Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 25 Oct 2026 02:47:16 +0200 Subject: [PATCH] scripts: fork-validate.sh - --moe-exact, greedy identity for the readback skip F1 (the routing-ids readback skip in sched_copy_experts) had no automated check. --moe-exact MODEL reads expert_count / expert_used_count from the GGUF header, computes the skip threshold ceil(4 * n_expert / n_expert_used) tokens per ubatch (128 on the 35B), and runs one greedy request with a prompt of about 2x that in a single ubatch, every expert on the host, so the prefill takes the skip and each decode step takes the readback. The candidate's 64 tokens must equal the reference's byte for byte; with the LLAMA_MOE_IDS_SKIP knob in the candidate, also the candidate with the skip forced off (=0), which proves the skip exact even when the reference carries it, and =log must show skips, all at >= 4 * n_expert ids. Without the knob skip-engaged SKIPs with the batch-size argument. It needs the GPU to itself (same guard as --perf, now shared as gpu_exclusive); FV_TEST_SHARED_GPU=1 bypasses the guard for script tests against stub binaries only. Also unset inherited LLAMA_MOE* variables. Not run on the GPU yet (needs a swapdown window). Dry-run on the pod against stub servers and the real 35B GGUF header: threshold 128, all-PASS when outputs match, FAIL on a differing reference, SKIP without the knob, FAIL for a dense model and while llama-swap runs. Co-Authored-By: Claude Opus 5.5 --- scripts/fork-validate.sh | 124 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 115 insertions(+), 9 deletions(-) diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh index 7f337e10c..1e6904d74 100644 --- a/scripts/fork-validate.sh +++ b/scripts/fork-validate.sh @@ -40,6 +40,15 @@ # one arbiter server with --moe-cache-mib N on a MoE model, forced cold/warm twice # -> greedy output identical before/after (the expert-cache SLRU is re-initialised on # release; a stale SLRU maps experts to empty banks and produces garbage). +# --moe-exact MODEL +# the MoE routing-ids readback skip (F1, sched_copy_experts) is exact: MODEL (a MoE, +# e.g. the --perf model) with every expert on the host, greedy, one prompt of >= 2x +# the skip threshold ceil(4 * n_expert / n_expert_used) tokens (read from the GGUF; +# 128 on the 35B) -> candidate output byte-identical to the reference, and, if the +# candidate has the LLAMA_MOE_IDS_SKIP knob, to the candidate with the skip forced off +# (=0); =log proves the skip engaged. Without the knob skip-engaged is SKIPped with the +# batch-size argument. Needs the GPU to itself like --perf (same guard); about 3 model +# loads, so run it in the same swapdown window as --perf. # # Options: # --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build) @@ -50,11 +59,17 @@ # --moecache-mib N cache size for --moecache (default 2048; 256 never hits on a 26B and proves nothing) # --port-base N scratch ports N..N+2 (default 8091) # --keep keep the scratch dir even when everything passed (it is kept on any FAIL) +# --moe-exact-args S server args for --moe-exact (default: "-ngl 99 --n-cpu-moe 99 -c 4096 -b 2048 +# -ub 2048 -t 6"; keep the experts on the host and -ub >= the prompt) # --perf-report TSV only re-aggregate a saved /perf.tsv of an earlier --perf run and # apply the same gates (no GPU, no --build/--ref needed); runs nothing else # # Requirements: bash, coreutils, awk, sed, grep, curl, flock(1); gdb optional (makes the -# SIGTERM-during-release case deterministic). No python/jq. +# SIGTERM-during-release case deterministic; it is not in the pod image and is lost on a pod restart, +# then the case falls back to polling the log). No python/jq. +# +# Test only: FV_TEST_SHARED_GPU=1 skips the exclusive-GPU guard of --perf / --moe-exact, for running +# the script against stub binaries. Never set it for a real measurement. set -u @@ -67,10 +82,12 @@ PERF_MODEL=$HF/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906 MOECACHE_MODEL= MOECACHE_ARGS="-ngl 99 --n-cpu-moe 30 -c 4096" MOECACHE_MIB=2048 +MOEX_MODEL= +MOEX_ARGS="-ngl 99 --n-cpu-moe 99 -c 4096 -b 2048 -ub 2048 -t 6" BUILD= ; REF= ; CONCAT_BUILD= PORT_BASE=8091 KEEP=0 -DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 +DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 DO_MOEX=0 PERF_REPORT= usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; } @@ -86,6 +103,8 @@ while [ $# -gt 0 ]; do --moecache) DO_MOEC=1; MOECACHE_MODEL=$2; shift 2 ;; --moecache-args) MOECACHE_ARGS=$2; shift 2 ;; --moecache-mib) MOECACHE_MIB=$2; shift 2 ;; + --moe-exact) DO_MOEX=1; MOEX_MODEL=$2; shift 2 ;; + --moe-exact-args) MOEX_ARGS=$2; shift 2 ;; --concat-build) CONCAT_BUILD=$2; shift 2 ;; --small) SMALL=$2; shift 2 ;; --mmproj) MMPROJ=$2; shift 2 ;; @@ -98,7 +117,7 @@ while [ $# -gt 0 ]; do esac done [ -n "$PERF_REPORT" ] || { [ -n "$BUILD" ] && [ -n "$REF" ]; } || usage -if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC)) -eq 0 ]; then +if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC + DO_MOEX)) -eq 0 ]; then DO_OPS=1 DO_CONCAT=1 DO_ARB=1 fi [ -n "$MMPROJ" ] || MMPROJ=$(dirname "$SMALL")/mmproj-F16.gguf @@ -173,7 +192,7 @@ trap finish EXIT trap 'echo "interrupted"; NFAIL=$((NFAIL + 1)); exit 130' INT TERM HUP # ---------------------------------------------------------------- environment sanity -for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\)[A-Za-z0-9_]*\)=.*/\1/p'); do +for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\|MOE\)[A-Za-z0-9_]*\)=.*/\1/p'); do log "unsetting inherited $v"; unset "$v" done [ -n "$PERF_REPORT" ] || for d in "$BUILD" "$REF"; do @@ -722,11 +741,7 @@ bench_samples() { # JSONL (llama-bench -o jsonl) -> "testt/s", one row per for (i = 4; i <= NF; i++) {v = $i; gsub(/,/, "", v); if (v != "") print t "\t" v}}' } run_perf() { - if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then - fail perf "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return - fi - local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l) - [ "$others" -eq 0 ] || { fail perf "$others other llama processes are running"; return; } + gpu_exclusive perf || return [ -f "$PERF_MODEL" ] || { fail perf "model missing: $PERF_MODEL"; return; } [ -x "$REF/bin/llama-bench" ] && [ -x "$BUILD/bin/llama-bench" ] || { fail perf "llama-bench missing"; return; } local common=(-m "$PERF_MODEL" -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -o jsonl -oe md) @@ -832,6 +847,96 @@ run_moecache() { check moecache/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}" } +# ================================================================ 6. MoE routing-ids readback skip (F1) exactness +# sched_copy_experts skips the ids readback when a split's ids tensor has n_tokens * n_expert_used >= +# 4 * n_expert entries, i.e. for any ubatch of at least ceil(4 * n_expert / n_expert_used) tokens +# (128 on a 256-expert / 8-used model). Method: the model with every MoE layer's experts on the host +# (so the expert-copy callback runs), greedy, one prompt of >= 2x that many tokens in a single +# ubatch: the prefill takes the skip and every decode step takes the readback. The candidate's +# output must be byte-identical to the reference's and, when the build has the LLAMA_MOE_IDS_SKIP +# knob, to the candidate with the skip forced off (=0), which proves the skip itself is exact even +# when the reference build carries it too; =log then shows the skip engaged. +gguf_u32() { # FILE KEY-SUFFIX -> uint32 value of the first GGUF metadata key ending in KEY-SUFFIX + local off t + off=$(head -c 67108864 "$1" | grep -abo -m1 -F "$2" | head -1 | cut -d: -f1) + [ -n "$off" ] || return 1 + off=$((off + ${#2})) # key string -> uint32 value type -> value + t=$(od -An -tu4 -j "$off" -N4 "$1" | tr -d ' ') + [ "$t" = 4 ] || [ "$t" = 5 ] || return 1 # GGUF_TYPE_UINT32 / INT32 + od -An -tu4 -j $((off + 4)) -N4 "$1" | tr -d ' ' +} +gpu_exclusive() { # CHECK-NAME -> FAILs it and returns 1 unless nothing else can use the GPU + [ "${FV_TEST_SHARED_GPU:-}" = 1 ] && { log "FV_TEST_SHARED_GPU=1: skipping the exclusive-GPU guard (script testing only)"; return 0; } + if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then + fail "$1" "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return 1 + fi + local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l) + [ "$others" -eq 0 ] || { fail "$1" "$others other llama processes are running"; return 1; } +} +moex_prompt() { # N_LINES -> a deterministic prose-like prompt (JSON-safe: no quotes or backslashes) + local i + for i in $(seq 1 "$1"); do + printf 'Entry %d: the river town of Station%d ships %d barrels of grain and %d crates of salt each week to the harbour, and its council meets on day %d of the month. ' \ + "$i" "$i" $(( (i * 37) % 101 + 3 )) $(( (i * 53) % 89 + 2 )) $(( (i * 7) % 28 + 1 )) + done + printf 'Question: which town ships the most barrels of grain each week? Answer:' +} +run_moe_exact() { + gpu_exclusive moe-exact || return + [ -f "$MOEX_MODEL" ] || { fail moe-exact "model missing: $MOEX_MODEL"; return; } + local n_exp n_used + n_exp=$(gguf_u32 "$MOEX_MODEL" .expert_count); n_used=$(gguf_u32 "$MOEX_MODEL" .expert_used_count) + if [ -z "$n_exp" ] || [ -z "$n_used" ] || [ "$n_exp" -eq 0 ] || [ "$n_used" -eq 0 ]; then + fail moe-exact "no expert_count / expert_used_count in $MOEX_MODEL (not a MoE model?)"; return + fi + local thr=$(( (4 * n_exp + n_used - 1) / n_used )) # tokens per ubatch where the skip starts + local lines=$(( (2 * thr + 39) / 40 + 2 )) # ~45 tokens per line -> >= 2x the threshold + local body + body="{\"prompt\":\"$(moex_prompt "$lines")\",\"n_predict\":64,$GREEDY,\"cache_prompt\":false}" + local knob=0; grep -qa LLAMA_MOE_IDS_SKIP "$BUILD/bin/libllama.so" 2>/dev/null && knob=1 + log "moe-exact: n_expert $n_exp, n_expert_used $n_used -> skip at >= $thr tokens per ubatch; prompt of $lines lines; knob in candidate: $([ $knob = 1 ] && echo yes || echo no)" + + # arm NAME BUILD ENV... : one server, one request; sets MX_CONTENT / MX_PN, returns 1 on failure + mx_arm() { + local name=$1 b=$2; shift 2 + # shellcheck disable=SC2086 + start_server "$name" "$b" "$PC" "$@" -- -m "$MOEX_MODEL" $MOEX_ARGS -fa 1 -lm none --parallel 1 --no-warmup + local p=$SPID code + if ! wait_health "$PC" "$p" 900; then fail "moe-exact/$name" "server did not start, see $S/$name.log"; stop_server "$p"; return 1; fi + code=$(complete "$PC" "$S/$name.json" "$body" 600) + stop_server "$p" + MX_CONTENT=$(json_str content "$S/$name.json"); MX_PN=$(json_num prompt_n "$S/$name.json") + [ "$code" = 200 ] && [ -n "$MX_CONTENT" ] || { fail "moe-exact/$name" "HTTP $code, empty output; see $S/$name.log"; return 1; } + [ "$RC" -eq 0 ] || log "warning: $name exited $RC" + } + local c_cand c_ref c_off pn + if [ $knob = 1 ]; then mx_arm X-cand "$BUILD" LLAMA_MOE_IDS_SKIP=log || return + else mx_arm X-cand "$BUILD" || return; fi + c_cand=$MX_CONTENT; pn=${MX_PN:-0} + if [ "$pn" -lt "$thr" ]; then + fail moe-exact/prompt "prompt_n $pn < threshold $thr: the skip never ran, nothing was tested"; return + fi + mx_arm X-ref "$REF" || return + c_ref=$MX_CONTENT + check moe-exact/identical-ref $([ "$c_cand" = "$c_ref" ] && echo 0 || echo 1) \ + "prompt_n $pn (>= $thr), 64 greedy tokens: candidate $([ "$c_cand" = "$c_ref" ] && echo == || echo '!=') reference ('$(echo "$c_cand" | cut -c1-60)')" + if [ $knob = 1 ]; then + mx_arm X-cand-noskip "$BUILD" LLAMA_MOE_IDS_SKIP=0 || return + c_off=$MX_CONTENT + check moe-exact/identical-noskip $([ "$c_cand" = "$c_off" ] && echo 0 || echo 1) \ + "candidate with the skip (LLAMA_MOE_IDS_SKIP=log) $([ "$c_cand" = "$c_off" ] && echo == || echo '!=') candidate with the readback forced (=0)" + local n_skip n_small + n_skip=$(grep -c 'sched_copy_experts: ids readback skipped' "$S/X-cand.log") + n_small=$(sed -n 's/.*ids readback skipped, n_ids=\([0-9]*\) n_expert=\([0-9]*\).*/\1 \2/p' "$S/X-cand.log" | awk '$1 < 4 * $2' | wc -l) + check moe-exact/skip-engaged $(( n_skip > 0 && n_small == 0 ? 0 : 1 )) \ + "$n_skip skips logged ($n_small below 4*n_expert), $(grep -c 'ids readback skipped' "$S/X-cand-noskip.log") with the skip forced off" + else + skip moe-exact/skip-engaged "candidate has no LLAMA_MOE_IDS_SKIP knob to observe it; by construction the prefill ubatch had $pn x $n_used = $((pn * n_used)) ids >= 4 x $n_exp, so the skip branch ran" + fi + local bad_lines; bad_lines=$(scan_logs "$S"/X-*.log) + check moe-exact/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}" +} + # ================================================================ main if [ -n "$PERF_REPORT" ]; then [ -r "$PERF_REPORT" ] || { echo "cannot read $PERF_REPORT" >&2; exit 2; } @@ -842,4 +947,5 @@ fi [ $DO_ARB -eq 1 ] && run_arbiter [ $DO_PERF -eq 1 ] && run_perf [ $DO_MOEC -eq 1 ] && run_moecache +[ $DO_MOEX -eq 1 ] && run_moe_exact exit 0