diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh index 7f337e10c..1e6904d74 100644 --- a/scripts/fork-validate.sh +++ b/scripts/fork-validate.sh @@ -40,6 +40,15 @@ # one arbiter server with --moe-cache-mib N on a MoE model, forced cold/warm twice # -> greedy output identical before/after (the expert-cache SLRU is re-initialised on # release; a stale SLRU maps experts to empty banks and produces garbage). +# --moe-exact MODEL +# the MoE routing-ids readback skip (F1, sched_copy_experts) is exact: MODEL (a MoE, +# e.g. the --perf model) with every expert on the host, greedy, one prompt of >= 2x +# the skip threshold ceil(4 * n_expert / n_expert_used) tokens (read from the GGUF; +# 128 on the 35B) -> candidate output byte-identical to the reference, and, if the +# candidate has the LLAMA_MOE_IDS_SKIP knob, to the candidate with the skip forced off +# (=0); =log proves the skip engaged. Without the knob skip-engaged is SKIPped with the +# batch-size argument. Needs the GPU to itself like --perf (same guard); about 3 model +# loads, so run it in the same swapdown window as --perf. # # Options: # --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build) @@ -50,11 +59,17 @@ # --moecache-mib N cache size for --moecache (default 2048; 256 never hits on a 26B and proves nothing) # --port-base N scratch ports N..N+2 (default 8091) # --keep keep the scratch dir even when everything passed (it is kept on any FAIL) +# --moe-exact-args S server args for --moe-exact (default: "-ngl 99 --n-cpu-moe 99 -c 4096 -b 2048 +# -ub 2048 -t 6"; keep the experts on the host and -ub >= the prompt) # --perf-report TSV only re-aggregate a saved /perf.tsv of an earlier --perf run and # apply the same gates (no GPU, no --build/--ref needed); runs nothing else # # Requirements: bash, coreutils, awk, sed, grep, curl, flock(1); gdb optional (makes the -# SIGTERM-during-release case deterministic). No python/jq. +# SIGTERM-during-release case deterministic; it is not in the pod image and is lost on a pod restart, +# then the case falls back to polling the log). No python/jq. +# +# Test only: FV_TEST_SHARED_GPU=1 skips the exclusive-GPU guard of --perf / --moe-exact, for running +# the script against stub binaries. Never set it for a real measurement. set -u @@ -67,10 +82,12 @@ PERF_MODEL=$HF/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906 MOECACHE_MODEL= MOECACHE_ARGS="-ngl 99 --n-cpu-moe 30 -c 4096" MOECACHE_MIB=2048 +MOEX_MODEL= +MOEX_ARGS="-ngl 99 --n-cpu-moe 99 -c 4096 -b 2048 -ub 2048 -t 6" BUILD= ; REF= ; CONCAT_BUILD= PORT_BASE=8091 KEEP=0 -DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 +DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 DO_MOEX=0 PERF_REPORT= usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; } @@ -86,6 +103,8 @@ while [ $# -gt 0 ]; do --moecache) DO_MOEC=1; MOECACHE_MODEL=$2; shift 2 ;; --moecache-args) MOECACHE_ARGS=$2; shift 2 ;; --moecache-mib) MOECACHE_MIB=$2; shift 2 ;; + --moe-exact) DO_MOEX=1; MOEX_MODEL=$2; shift 2 ;; + --moe-exact-args) MOEX_ARGS=$2; shift 2 ;; --concat-build) CONCAT_BUILD=$2; shift 2 ;; --small) SMALL=$2; shift 2 ;; --mmproj) MMPROJ=$2; shift 2 ;; @@ -98,7 +117,7 @@ while [ $# -gt 0 ]; do esac done [ -n "$PERF_REPORT" ] || { [ -n "$BUILD" ] && [ -n "$REF" ]; } || usage -if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC)) -eq 0 ]; then +if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC + DO_MOEX)) -eq 0 ]; then DO_OPS=1 DO_CONCAT=1 DO_ARB=1 fi [ -n "$MMPROJ" ] || MMPROJ=$(dirname "$SMALL")/mmproj-F16.gguf @@ -173,7 +192,7 @@ trap finish EXIT trap 'echo "interrupted"; NFAIL=$((NFAIL + 1)); exit 130' INT TERM HUP # ---------------------------------------------------------------- environment sanity -for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\)[A-Za-z0-9_]*\)=.*/\1/p'); do +for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\|MOE\)[A-Za-z0-9_]*\)=.*/\1/p'); do log "unsetting inherited $v"; unset "$v" done [ -n "$PERF_REPORT" ] || for d in "$BUILD" "$REF"; do @@ -722,11 +741,7 @@ bench_samples() { # JSONL (llama-bench -o jsonl) -> "testt/s", one row per for (i = 4; i <= NF; i++) {v = $i; gsub(/,/, "", v); if (v != "") print t "\t" v}}' } run_perf() { - if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then - fail perf "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return - fi - local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l) - [ "$others" -eq 0 ] || { fail perf "$others other llama processes are running"; return; } + gpu_exclusive perf || return [ -f "$PERF_MODEL" ] || { fail perf "model missing: $PERF_MODEL"; return; } [ -x "$REF/bin/llama-bench" ] && [ -x "$BUILD/bin/llama-bench" ] || { fail perf "llama-bench missing"; return; } local common=(-m "$PERF_MODEL" -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -o jsonl -oe md) @@ -832,6 +847,96 @@ run_moecache() { check moecache/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}" } +# ================================================================ 6. MoE routing-ids readback skip (F1) exactness +# sched_copy_experts skips the ids readback when a split's ids tensor has n_tokens * n_expert_used >= +# 4 * n_expert entries, i.e. for any ubatch of at least ceil(4 * n_expert / n_expert_used) tokens +# (128 on a 256-expert / 8-used model). Method: the model with every MoE layer's experts on the host +# (so the expert-copy callback runs), greedy, one prompt of >= 2x that many tokens in a single +# ubatch: the prefill takes the skip and every decode step takes the readback. The candidate's +# output must be byte-identical to the reference's and, when the build has the LLAMA_MOE_IDS_SKIP +# knob, to the candidate with the skip forced off (=0), which proves the skip itself is exact even +# when the reference build carries it too; =log then shows the skip engaged. +gguf_u32() { # FILE KEY-SUFFIX -> uint32 value of the first GGUF metadata key ending in KEY-SUFFIX + local off t + off=$(head -c 67108864 "$1" | grep -abo -m1 -F "$2" | head -1 | cut -d: -f1) + [ -n "$off" ] || return 1 + off=$((off + ${#2})) # key string -> uint32 value type -> value + t=$(od -An -tu4 -j "$off" -N4 "$1" | tr -d ' ') + [ "$t" = 4 ] || [ "$t" = 5 ] || return 1 # GGUF_TYPE_UINT32 / INT32 + od -An -tu4 -j $((off + 4)) -N4 "$1" | tr -d ' ' +} +gpu_exclusive() { # CHECK-NAME -> FAILs it and returns 1 unless nothing else can use the GPU + [ "${FV_TEST_SHARED_GPU:-}" = 1 ] && { log "FV_TEST_SHARED_GPU=1: skipping the exclusive-GPU guard (script testing only)"; return 0; } + if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then + fail "$1" "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return 1 + fi + local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l) + [ "$others" -eq 0 ] || { fail "$1" "$others other llama processes are running"; return 1; } +} +moex_prompt() { # N_LINES -> a deterministic prose-like prompt (JSON-safe: no quotes or backslashes) + local i + for i in $(seq 1 "$1"); do + printf 'Entry %d: the river town of Station%d ships %d barrels of grain and %d crates of salt each week to the harbour, and its council meets on day %d of the month. ' \ + "$i" "$i" $(( (i * 37) % 101 + 3 )) $(( (i * 53) % 89 + 2 )) $(( (i * 7) % 28 + 1 )) + done + printf 'Question: which town ships the most barrels of grain each week? Answer:' +} +run_moe_exact() { + gpu_exclusive moe-exact || return + [ -f "$MOEX_MODEL" ] || { fail moe-exact "model missing: $MOEX_MODEL"; return; } + local n_exp n_used + n_exp=$(gguf_u32 "$MOEX_MODEL" .expert_count); n_used=$(gguf_u32 "$MOEX_MODEL" .expert_used_count) + if [ -z "$n_exp" ] || [ -z "$n_used" ] || [ "$n_exp" -eq 0 ] || [ "$n_used" -eq 0 ]; then + fail moe-exact "no expert_count / expert_used_count in $MOEX_MODEL (not a MoE model?)"; return + fi + local thr=$(( (4 * n_exp + n_used - 1) / n_used )) # tokens per ubatch where the skip starts + local lines=$(( (2 * thr + 39) / 40 + 2 )) # ~45 tokens per line -> >= 2x the threshold + local body + body="{\"prompt\":\"$(moex_prompt "$lines")\",\"n_predict\":64,$GREEDY,\"cache_prompt\":false}" + local knob=0; grep -qa LLAMA_MOE_IDS_SKIP "$BUILD/bin/libllama.so" 2>/dev/null && knob=1 + log "moe-exact: n_expert $n_exp, n_expert_used $n_used -> skip at >= $thr tokens per ubatch; prompt of $lines lines; knob in candidate: $([ $knob = 1 ] && echo yes || echo no)" + + # arm NAME BUILD ENV... : one server, one request; sets MX_CONTENT / MX_PN, returns 1 on failure + mx_arm() { + local name=$1 b=$2; shift 2 + # shellcheck disable=SC2086 + start_server "$name" "$b" "$PC" "$@" -- -m "$MOEX_MODEL" $MOEX_ARGS -fa 1 -lm none --parallel 1 --no-warmup + local p=$SPID code + if ! wait_health "$PC" "$p" 900; then fail "moe-exact/$name" "server did not start, see $S/$name.log"; stop_server "$p"; return 1; fi + code=$(complete "$PC" "$S/$name.json" "$body" 600) + stop_server "$p" + MX_CONTENT=$(json_str content "$S/$name.json"); MX_PN=$(json_num prompt_n "$S/$name.json") + [ "$code" = 200 ] && [ -n "$MX_CONTENT" ] || { fail "moe-exact/$name" "HTTP $code, empty output; see $S/$name.log"; return 1; } + [ "$RC" -eq 0 ] || log "warning: $name exited $RC" + } + local c_cand c_ref c_off pn + if [ $knob = 1 ]; then mx_arm X-cand "$BUILD" LLAMA_MOE_IDS_SKIP=log || return + else mx_arm X-cand "$BUILD" || return; fi + c_cand=$MX_CONTENT; pn=${MX_PN:-0} + if [ "$pn" -lt "$thr" ]; then + fail moe-exact/prompt "prompt_n $pn < threshold $thr: the skip never ran, nothing was tested"; return + fi + mx_arm X-ref "$REF" || return + c_ref=$MX_CONTENT + check moe-exact/identical-ref $([ "$c_cand" = "$c_ref" ] && echo 0 || echo 1) \ + "prompt_n $pn (>= $thr), 64 greedy tokens: candidate $([ "$c_cand" = "$c_ref" ] && echo == || echo '!=') reference ('$(echo "$c_cand" | cut -c1-60)')" + if [ $knob = 1 ]; then + mx_arm X-cand-noskip "$BUILD" LLAMA_MOE_IDS_SKIP=0 || return + c_off=$MX_CONTENT + check moe-exact/identical-noskip $([ "$c_cand" = "$c_off" ] && echo 0 || echo 1) \ + "candidate with the skip (LLAMA_MOE_IDS_SKIP=log) $([ "$c_cand" = "$c_off" ] && echo == || echo '!=') candidate with the readback forced (=0)" + local n_skip n_small + n_skip=$(grep -c 'sched_copy_experts: ids readback skipped' "$S/X-cand.log") + n_small=$(sed -n 's/.*ids readback skipped, n_ids=\([0-9]*\) n_expert=\([0-9]*\).*/\1 \2/p' "$S/X-cand.log" | awk '$1 < 4 * $2' | wc -l) + check moe-exact/skip-engaged $(( n_skip > 0 && n_small == 0 ? 0 : 1 )) \ + "$n_skip skips logged ($n_small below 4*n_expert), $(grep -c 'ids readback skipped' "$S/X-cand-noskip.log") with the skip forced off" + else + skip moe-exact/skip-engaged "candidate has no LLAMA_MOE_IDS_SKIP knob to observe it; by construction the prefill ubatch had $pn x $n_used = $((pn * n_used)) ids >= 4 x $n_exp, so the skip branch ran" + fi + local bad_lines; bad_lines=$(scan_logs "$S"/X-*.log) + check moe-exact/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}" +} + # ================================================================ main if [ -n "$PERF_REPORT" ]; then [ -r "$PERF_REPORT" ] || { echo "cannot read $PERF_REPORT" >&2; exit 2; } @@ -842,4 +947,5 @@ fi [ $DO_ARB -eq 1 ] && run_arbiter [ $DO_PERF -eq 1 ] && run_perf [ $DO_MOEC -eq 1 ] && run_moecache +[ $DO_MOEX -eq 1 ] && run_moe_exact exit 0