From 149a8cb0ae3ca3fb7724e7046401d5c76345da3f Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sat, 24 Oct 2026 20:08:11 +0200 Subject: [PATCH] scripts: add fork-validate.sh, post-rebase validation of the fork Checks the op kernels, the concat fast path, the VRAM arbiter incl. every shutdown path that has caused a production incident, and prefill/decode against a reference build. Prints PASS/FAIL per check. Co-Authored-By: Claude Opus 5.5 --- scripts/fork-validate.sh | 693 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 693 insertions(+) create mode 100644 scripts/fork-validate.sh diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh new file mode 100644 index 000000000..c8d18c19d --- /dev/null +++ b/scripts/fork-validate.sh @@ -0,0 +1,693 @@ +#!/usr/bin/env bash +# fork-validate.sh - validate a candidate build of the RX 580 llama.cpp fork against a reference build. +# +# Runs ON THE POD (Vulkan / RADV, RX 580). Prints PASS/FAIL/SKIP per check plus a summary; exits +# nonzero if any check FAILs. Only touches its own scratch dir under /tmp, its own scratch ports and +# processes it started itself (killed by explicit PID). Build dirs are only read. +# +# fork-validate.sh --build /root/arms/X/build --ref /root/llama.cpp/build [checks] [options] +# +# Checks (no check flag = --ops --concat --arbiter): +# --ops test-backend-ops on Vulkan0 for CONCAT, FLASH_ATTN_EXT, MUL_MAT, MUL_MAT_ID, GET_ROWS +# -> the fork's kernels and upstream's agree with the CPU reference; asserts the case count +# is > 0 because "-b Vulkan" (no index) silently runs nothing and still prints OK. +# --concat Vulkan concat tiled-transpose fast path (ggml_vk_concat, gate src_transposed_2d) +# -> it fires at prefill and NOT at decode on a real hybrid (delta-net) model; it is the +# fork's biggest prefill win, and taking it at decode once cost -17% decode. +# Needs a build carrying the GGML_VK_CONCAT_FASTPATH knob (=0 off, =log report); +# either --build has it or pass --concat-build DIR (candidate + the knob patch). +# --arbiter on-demand VRAM sharing (LLAMA_SLEEP_VRAM_ONLY ...) with two real servers on a scratch +# arena -> handoffs never strand, release/restore is bit-exact, multimodal survives it, +# and every shutdown path (mid-release, mid-wait, mid-generation, cold, warm, kill -9 of +# a waiter) exits cleanly and hands the GPU token on. These are the past production +# incidents (stranded requests, wedged token, exit 134 on cold teardown). +# --perf llama-bench, candidate vs reference, alternating ref,cand,cand,ref, on the 35B MoE +# -> no prefill regression > 5% at depth 0/16k/32k (FAIL), decode reported; peak VRAM +# per arm (FAIL if > 95% of the card). Needs llama-swap STOPPED (run it under +# /root/bench/moe/swapdown.sh); refuses to run if any other llama process is up. +# --moecache MODEL +# one arbiter server with --moe-cache-mib N on a MoE model, forced cold/warm twice +# -> greedy output identical before/after (the expert-cache SLRU is re-initialised on +# release; a stale SLRU maps experts to empty banks and produces garbage). +# +# Options: +# --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build) +# --small MODEL small hybrid model for --concat/--arbiter (default: Qwen3.5-0.8B UD-Q4_K_XL) +# --mmproj FILE its multimodal projector (default: mmproj-F16 next to it) +# --perf-model MODEL model for --perf (default: Qwen3.6-35B-A3B UD-Q4_K_XL) +# --moecache-args S extra server args for --moecache (default: "-ngl 99 --n-cpu-moe 30 -c 4096") +# --moecache-mib N cache size for --moecache (default 2048; 256 never hits on a 26B and proves nothing) +# --port-base N scratch ports N..N+2 (default 8091) +# --keep keep the scratch dir even when everything passed (it is kept on any FAIL) +# +# Requirements: bash, coreutils, awk, sed, grep, curl, flock(1); gdb optional (makes the +# SIGTERM-during-release case deterministic). No python/jq. + +set -u + +# ---------------------------------------------------------------- defaults +HF=/root/.cache/huggingface/hub +SMALL_DIR=$HF/models--unsloth--Qwen3.5-0.8B-GGUF/snapshots/6ab461498e2023f6e3c1baea90a8f0fe38ab64d0 +SMALL=$SMALL_DIR/Qwen3.5-0.8B-UD-Q4_K_XL.gguf +MMPROJ= +PERF_MODEL=$HF/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf +MOECACHE_MODEL= +MOECACHE_ARGS="-ngl 99 --n-cpu-moe 30 -c 4096" +MOECACHE_MIB=2048 +BUILD= ; REF= ; CONCAT_BUILD= +PORT_BASE=8091 +KEEP=0 +DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 + +usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; } + +while [ $# -gt 0 ]; do + case "$1" in + --build) BUILD=$2; shift 2 ;; + --ref) REF=$2; shift 2 ;; + --ops) DO_OPS=1; shift ;; + --concat) DO_CONCAT=1; shift ;; + --arbiter) DO_ARB=1; shift ;; + --perf) DO_PERF=1; shift ;; + --moecache) DO_MOEC=1; MOECACHE_MODEL=$2; shift 2 ;; + --moecache-args) MOECACHE_ARGS=$2; shift 2 ;; + --moecache-mib) MOECACHE_MIB=$2; shift 2 ;; + --concat-build) CONCAT_BUILD=$2; shift 2 ;; + --small) SMALL=$2; shift 2 ;; + --mmproj) MMPROJ=$2; shift 2 ;; + --perf-model) PERF_MODEL=$2; shift 2 ;; + --port-base) PORT_BASE=$2; shift 2 ;; + --keep) KEEP=1; shift ;; + -h|--help) usage ;; + *) echo "unknown argument: $1" >&2; usage ;; + esac +done +[ -n "$BUILD" ] && [ -n "$REF" ] || usage +if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC)) -eq 0 ]; then + DO_OPS=1 DO_CONCAT=1 DO_ARB=1 +fi +[ -n "$MMPROJ" ] || MMPROJ=$(dirname "$SMALL")/mmproj-F16.gguf +PA=$PORT_BASE PB=$((PORT_BASE + 1)) PC=$((PORT_BASE + 2)) + +# ---------------------------------------------------------------- reporting +RESULTS=() +NFAIL=0 +ts() { date +%H:%M:%S; } +log() { echo "[$(ts)] $*"; } +pass() { RESULTS+=("PASS $1: $2"); echo "[$(ts)] PASS $1: $2"; } +fail() { RESULTS+=("FAIL $1: $2"); echo "[$(ts)] FAIL $1: $2"; NFAIL=$((NFAIL + 1)); } +skip() { RESULTS+=("SKIP $1: $2"); echo "[$(ts)] SKIP $1: $2"; } +# check NAME CONDITION-RESULT(0/1) MESSAGE +check() { if [ "$2" -eq 0 ]; then pass "$1" "$3"; else fail "$1" "$3"; fi; } +now_ms() { date +%s%3N; } + +# ---------------------------------------------------------------- scratch + cleanup +S=$(mktemp -d /tmp/fork-validate.XXXXXX) +ARENA=$S/arena +mkdir -p "$ARENA" +cd "$S" || exit 2 # anything written to a relative path lands in the scratch dir +PIDS=() # "pid:starttime" of every background process we start (starttime guards PID reuse) + +pid_alive() { # alive and not a zombie + [ -r "/proc/$1/stat" ] || return 1 + local st; st=$(sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | awk '{print $1}') || return 1 + [ -n "$st" ] && [ "$st" != Z ] +} +starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | awk '{print $20}'; } +track() { PIDS+=("$1:$(starttime "$1")"); } +ours() { # "pid:starttime" -> pid if that exact process is still alive + local p=${1%%:*} t=${1#*:} + pid_alive "$p" && [ "$(starttime "$p")" = "$t" ] && echo "$p" +} + +cleanup() { + local e p + for e in "${PIDS[@]}"; do p=$(ours "$e") && kill -TERM "$p" 2>/dev/null; done + local i=0 + while [ $i -lt 50 ]; do + local any=0 + for e in "${PIDS[@]}"; do ours "$e" > /dev/null && any=1; done + [ $any -eq 0 ] && break + sleep 0.1; i=$((i + 1)) + done + for e in "${PIDS[@]}"; do p=$(ours "$e") && kill -KILL "$p" 2>/dev/null; done + wait 2>/dev/null +} +finish() { + local rc=$? + trap - EXIT INT TERM HUP + cleanup + echo + echo "==================== fork-validate summary ====================" + echo "candidate: $BUILD ($(git -C "$BUILD/.." rev-parse --short HEAD 2>/dev/null || echo '?'))" + echo "reference: $REF ($(git -C "$REF/.." rev-parse --short HEAD 2>/dev/null || echo '?'))" + local r; for r in "${RESULTS[@]}"; do echo "$r"; done + local np ns + np=$(printf '%s\n' "${RESULTS[@]}" | grep -c '^PASS') + ns=$(printf '%s\n' "${RESULTS[@]}" | grep -c '^SKIP') + echo "total: $np PASS, $NFAIL FAIL, $ns SKIP" + if [ $NFAIL -gt 0 ] || [ $KEEP -eq 1 ] || [ $rc -ne 0 ]; then + echo "scratch kept: $S" + else + rm -rf "$S" + fi + [ $NFAIL -gt 0 ] && exit 1 + exit $rc +} +trap finish EXIT +trap 'echo "interrupted"; NFAIL=$((NFAIL + 1)); exit 130' INT TERM HUP + +# ---------------------------------------------------------------- environment sanity +for v in $(env | sed -n 's/^\(GGML_[A-Za-z0-9_]*\|LLAMA_\(SLEEP\|VRAM\|MMPROJ\|ARG\|CACHE\|LOG\)[A-Za-z0-9_]*\)=.*/\1/p'); do + log "unsetting inherited $v"; unset "$v" +done +for d in "$BUILD" "$REF"; do + [ -x "$d/bin/llama-server" ] || { echo "no llama-server in $d/bin" >&2; exit 2; } +done +case "$ARENA" in /dev/shm/llama-vram*) echo "refusing the production arena" >&2; exit 2 ;; esac +VRAM_USED=$(ls /sys/class/drm/card*/device/mem_info_vram_used 2>/dev/null | head -1) +VRAM_TOTAL=$(ls /sys/class/drm/card*/device/mem_info_vram_total 2>/dev/null | head -1) +vram_mb() { echo $(( $(cat "$VRAM_USED") / 1048576 )); } +port_busy() { # LISTEN socket on the port (v4 or v6) + local hex; hex=$(printf '%04X' "$1") + awk -v h=":$hex" '$4=="0A" && substr($2, length($2)-4)==h {f=1} END {exit !f}' /proc/net/tcp /proc/net/tcp6 2>/dev/null +} +for p in $PA $PB $PC; do + port_busy "$p" && { echo "scratch port $p is in use" >&2; exit 2; } +done +log "scratch dir $S, ports $PA/$PB/$PC, VRAM used $(vram_mb) MiB" + +# ---------------------------------------------------------------- helpers: servers +# start_server NAME BUILD PORT ENV-ASSIGNMENTS... -- SERVER-ARGS... (sets SPID, log at $S/NAME.log) +start_server() { + local name=$1 build=$2 port=$3; shift 3 + local envs=() + while [ $# -gt 0 ] && [ "$1" != -- ]; do envs+=("$1"); shift; done + shift + env LD_LIBRARY_PATH="$build/bin" "${envs[@]}" "$build/bin/llama-server" \ + --host 127.0.0.1 --port "$port" "$@" >> "$S/$name.log" 2>&1 & + SPID=$! + track "$SPID" + echo "=== start pid $SPID $(date +%T.%N)" >> "$S/$name.log.events" +} +# wait_health PORT PID TIMEOUT_S +wait_health() { + local t0; t0=$(now_ms) + while [ $(( $(now_ms) - t0 )) -lt $(( $3 * 1000 )) ]; do + pid_alive "$2" || return 1 + [ "$(curl -s -m 2 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$1/health")" = 200 ] && return 0 + sleep 0.2 + done + return 1 +} +# wait_exit PID TIMEOUT_MS -> prints elapsed ms, returns 1 on timeout. Then reap with `wait PID`. +wait_exit() { + local t0; t0=$(now_ms) + while pid_alive "$1"; do + [ $(( $(now_ms) - t0 )) -ge "$2" ] && { echo $(( $(now_ms) - t0 )); return 1; } + sleep 0.02 + done + echo $(( $(now_ms) - t0 )) +} +# stop_server PID -> SIGTERM, wait up to 20 s, sets RC and EXIT_MS +stop_server() { + kill -TERM "$1" 2>/dev/null + EXIT_MS=$(wait_exit "$1" 20000) || { kill -KILL "$1" 2>/dev/null; } + wait "$1" 2>/dev/null; RC=$? +} + +# ---------------------------------------------------------------- helpers: locks (/proc/locks) +lock_id() { # maj:min:inode as /proc/locks prints it (hex dev, decimal inode) + local d; d=$(( 16#$(stat -c %D "$1") )) + printf '%02x:%02x:%s' $(( (d >> 8) & 0xfff )) $(( (d & 0xff) | ((d >> 12) & 0xfff00) )) "$(stat -c %i "$1")" +} +lock_pids() { # FILE WRITE|READ -> PIDs holding a flock on it (granted, not blocked) + [ -e "$1" ] || return 0 + awk -v id="$(lock_id "$1")" -v m="$2" '$2=="FLOCK" && $4==m && $6==id {print $5}' /proc/locks +} +token_holder() { lock_pids "$ARENA/token.lock" WRITE | head -1; } +holder_num() { local h; h=$(token_holder); echo "${h:-0}"; } # for arithmetic: 0 = nobody +# wait_holder PID TIMEOUT_MS : until PID holds token.lock ("" = until nobody does) +wait_holder() { + local t0; t0=$(now_ms) + while [ "$(token_holder)" != "$1" ]; do + [ $(( $(now_ms) - t0 )) -ge "$2" ] && return 1 + sleep 0.01 + done + echo $(( $(now_ms) - t0 )) +} +wait_want() { # PID TIMEOUT_MS : until PID holds want.lock shared (registered waiter) + local t0; t0=$(now_ms) + until lock_pids "$ARENA/want.lock" READ | grep -qx "$1"; do + [ $(( $(now_ms) - t0 )) -ge "$2" ] && return 1 + sleep 0.01 + done +} + +# ---------------------------------------------------------------- helpers: requests +json_str() { sed -n 's/.*"'"$1"'":"\(\([^"\\]\|\\.\)*\)".*/\1/p' "$2" | head -1; } +json_num() { grep -o "\"$1\":[0-9]*" "$2" | head -1 | cut -d: -f2; } +# complete PORT OUTFILE JSON-BODY [TIMEOUT_S] -> prints HTTP code ("000" on timeout/refused) +complete() { + curl -sS -m "${4:-60}" -o "$2" -w '%{http_code}' -H 'Content-Type: application/json' \ + -d "$3" "http://127.0.0.1:$1/completion" 2>>"$S/curl.err" || true +} +PROMPT='The history of the printing press began in' +GREEDY="\"temperature\":0,\"top_k\":1,\"seed\":42" +greedy_body() { # N_PREDICT CACHE_PROMPT [PROMPT] + echo "{\"prompt\":\"${3:-$PROMPT}\",\"n_predict\":$1,$GREEDY,\"cache_prompt\":$2}" +} +LONG_BODY="{\"prompt\":\"Write a very long story about a lighthouse keeper.\",\"n_predict\":4000,\"ignore_eos\":true,$GREEDY,\"stream\":true}" +# stream_bg PORT OUTFILE BODY -> starts a streaming request, sets CPID +stream_bg() { + curl -sS -N -m 300 -H 'Content-Type: application/json' -d "$3" \ + "http://127.0.0.1:$1/completion" > "$2" 2>&1 & + CPID=$!; track "$CPID" +} +wait_tokens() { # FILE N TIMEOUT_MS : until N SSE data lines arrived + local t0; t0=$(now_ms) + while [ "$(grep -c '^data: ' "$1" 2>/dev/null)" -lt "$2" ]; do + [ $(( $(now_ms) - t0 )) -ge "$3" ] && return 1 + sleep 0.05 + done +} +scan_logs() { # NAME-PATTERN... -> prints offending lines + grep -h -n -E 'GGML_ASSERT|Assertion|terminate called|Segmentation fault|core dumped|ErrorDeviceLost|ErrorOutOfDeviceMemory|DEVICE_LOST|vk::.*Error|GGML_ABORT|fatal error' "$@" 2>/dev/null | head -20 +} + +# ================================================================ 1. ops +run_ops() { + local op out passed total + for op in CONCAT FLASH_ATTN_EXT MUL_MAT MUL_MAT_ID GET_ROWS; do + out=$S/ops-$op.log + log "ops: test-backend-ops -b Vulkan0 -o $op" + ( cd "$S" && LD_LIBRARY_PATH="$BUILD/bin" timeout 1800 "$BUILD/bin/test-backend-ops" -b Vulkan0 -o "$op" ) \ + > "$out" 2>&1 + local rc=$? + # strip ANSI colours; "N/M tests passed" is printed once per tested backend + read -r passed total < <(sed 's/\x1b\[[0-9;]*m//g' "$out" | sed -n 's/^ *\([0-9]*\)\/\([0-9]*\) tests passed.*/\1 \2/p' | head -1) + passed=${passed:-0}; total=${total:-0} + if [ $rc -eq 0 ] && [ "$total" -gt 0 ] && [ "$passed" -eq "$total" ] \ + && sed 's/\x1b\[[0-9;]*m//g' "$out" | grep -q 'Backend Vulkan0: OK'; then + pass "ops/$op" "$passed/$total cases on Vulkan0" + else + fail "ops/$op" "rc=$rc, $passed/$total passed (0 cases = wrong backend name); see $out" + fi + done +} + +# ================================================================ 2. concat fast path +# Method: llama-bench on the small hybrid model (qwen35: delta-net layers concat the conv state with +# the transposed qkv in build_conv_state). (1) with the knob =log, count fast-path dispatches in a +# prefill-only run (must be > 0, all with >= 32 tokens) and a decode-only run (must be 0, while the +# perf logger proves CONCAT nodes did execute); (2) GGML_VK_PERF_LOGGER CONCAT time at prefill: the +# unmodified candidate must match knob-on and be >= 2x faster than knob-off, which ties the +# candidate binary itself (no knob) to the fast path. +concat_us() { # LOG -> "count avg_us" over all perf-logger CONCAT lines + awk '/^CONCAT: / {n += $2; t += $(NF-1)} END {if (n) printf "%d %.2f\n", n, t/n; else print "0 0"}' "$1" +} +run_concat() { + local tb=$CONCAT_BUILD + [ -n "$tb" ] || tb=$BUILD + if ! grep -qa GGML_VK_CONCAT_FASTPATH "$tb/bin/libggml-vulkan.so" 2>/dev/null; then + fail concat "no GGML_VK_CONCAT_FASTPATH knob in $tb (build the candidate + knob patch, pass --concat-build)" + return + fi + [ -f "$SMALL" ] || { fail concat "small model missing: $SMALL"; return; } + local common=(-m "$SMALL" -ngl 99 -fa 1 -lm none -t 6) + local pp=(-p 512 -n 0 -b 512 -ub 512 -r 3) + local tg=(-p 0 -n 32 -r 1) + lb() { # BUILD OUT ENV... -- ARGS + local b=$1 out=$2; shift 2 + local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift + env LD_LIBRARY_PATH="$b/bin" GGML_VK_PERF_LOGGER=1 "${envs[@]}" "$b/bin/llama-bench" "${common[@]}" "$@" \ + > "$out" 2>&1 + } + log "concat: llama-bench prefill/decode arms (perf logger on)" + lb "$tb" "$S/cc-pp-log.log" GGML_VK_CONCAT_FASTPATH=log -- "${pp[@]}" + lb "$tb" "$S/cc-tg-log.log" GGML_VK_CONCAT_FASTPATH=log -- "${tg[@]}" + lb "$tb" "$S/cc-pp-off.log" GGML_VK_CONCAT_FASTPATH=0 -- "${pp[@]}" + lb "$BUILD" "$S/cc-pp-cand.log" -- "${pp[@]}" + lb "$tb" "$S/cc-pp-off2.log" GGML_VK_CONCAT_FASTPATH=0 -- "${pp[@]}" + lb "$BUILD" "$S/cc-pp-cand2.log" -- "${pp[@]}" + cat "$S/cc-pp-off2.log" >> "$S/cc-pp-off.log"; cat "$S/cc-pp-cand2.log" >> "$S/cc-pp-cand.log" + + local n_pp n_pp_small n_tg n_tg_concat + n_pp=$(grep -c 'transpose fast path' "$S/cc-pp-log.log") + n_pp_small=$(sed -n 's/.*transpose fast path, transposed ne0=\([0-9]*\).*/\1/p' "$S/cc-pp-log.log" | awk '$1 < 32' | wc -l) + n_tg=$(grep -c 'transpose fast path' "$S/cc-tg-log.log") + read -r n_tg_concat _ < <(concat_us "$S/cc-tg-log.log") + local ne0s; ne0s=$(sed -n 's/.*transposed ne0=\([0-9]*\).*/\1/p' "$S/cc-pp-log.log" | sort -n | uniq -c | awk '{printf "%sx%s ", $1, $2}') + check concat/prefill-fires $(( n_pp > 0 && n_pp_small == 0 ? 0 : 1 )) \ + "$n_pp fast-path dispatches in the prefill run (transposed ne0: ${ne0s:-none})" + check concat/decode-skips $(( n_tg == 0 && n_tg_concat > 0 ? 0 : 1 )) \ + "$n_tg fast-path dispatches over $n_tg_concat executed CONCAT nodes in the decode run" + + local c_on c_off c_cand on off cand + read -r c_on on < <(concat_us "$S/cc-pp-log.log") + read -r c_off off < <(concat_us "$S/cc-pp-off.log") + read -r c_cand cand < <(concat_us "$S/cc-pp-cand.log") + local ok + ok=$(awk -v on="$on" -v off="$off" -v c="$cand" 'BEGIN {print (c > 0 && on > 0 && off/c >= 2 && c/on < 1.5 && on/c < 1.5) ? 0 : 1}') + check concat/candidate-timing "$ok" \ + "prefill CONCAT avg: candidate ${cand} us, knob-on ${on} us, knob-off ${off} us (off/cand $(awk -v a="$off" -v b="$cand" 'BEGIN {printf "%.1f", (b > 0) ? a / b : 0}')x)" + local t_on t_off t_cand + t_on=$(grep -E '\| +pp512 ' "$S/cc-pp-log.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | head -1) + t_off=$(grep -E '\| +pp512 ' "$S/cc-pp-off.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | tr '\n' ' ') + t_cand=$(grep -E '\| +pp512 ' "$S/cc-pp-cand.log" | awk -F'|' '{print $(NF-1)}' | awk '{print $1}' | tr '\n' ' ') + log "concat: pp512 t/s (perf logger on, so absolute speed is not representative): on $t_on | off $t_off | candidate $t_cand" +} + +# ================================================================ 3. arbiter +ARB_ENV=(LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$ARENA" LLAMA_SLEEP_EVICT_KV=1 LLAMA_MMPROJ_ONDEMAND=1) +ARB_ARGS=(-m "$SMALL" -c 8192 -ngl 99 -fa 1 -lm none --parallel 1 --no-warmup --reasoning off) +A_PID= ; B_PID= +start_a() { start_server A "$BUILD" "$PA" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --mmproj "$MMPROJ"; A_PID=$SPID; } +start_b() { start_server B "$BUILD" "$PB" "${ARB_ENV[@]}" -- "${ARB_ARGS[@]}" --no-mmproj; B_PID=$SPID; } +sleeps() { grep -c 'server entering sleeping state' "$1"; } + +make_red_ppm() { # 128x128 solid red, binary PPM (stb_image reads it; no image tools needed) + local f=$S/red.ppm i + printf '\377\000\000' > "$S/px" + for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14; do cat "$S/px" "$S/px" > "$S/px2"; mv "$S/px2" "$S/px"; done + { printf 'P6\n128 128\n255\n'; cat "$S/px"; } > "$f" + base64 -w0 "$f" +} + +run_arbiter() { + [ -f "$SMALL" ] && [ -f "$MMPROJ" ] || { fail arbiter "model/mmproj missing: $SMALL $MMPROJ"; return; } + local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 )) + if [ $free_mb -lt 1536 ]; then + fail arbiter "only $free_mb MiB VRAM free (< 1536): run inside the swapdown window"; return + fi + log "arbiter: $free_mb MiB VRAM free; starting A (mmproj) on :$PA" + start_a + wait_health "$PA" "$A_PID" 180 || { fail arbiter/start "A did not become healthy, see $S/A.log"; return; } + + # no-handoff reference: A has held the token since its load and never released + local code ref_a + code=$(complete "$PA" "$S/ref-a.json" "$(greedy_body 48 false)") + ref_a=$(json_str content "$S/ref-a.json") + [ "$code" = 200 ] && [ -n "$ref_a" ] || { fail arbiter/start "reference request on A: HTTP $code"; return; } + [ "$(token_holder)" = "$A_PID" ] || { fail arbiter/start "A serves but does not hold token.lock"; return; } + + log "arbiter: starting B (text-only) on :$PB" + start_b + wait_health "$PB" "$B_PID" 180 || { fail arbiter/start "B did not become healthy, see $S/B.log"; return; } + pass arbiter/start "A=$A_PID B=$B_PID up, reference output ${#ref_a} chars" + + # ---- (a)+(b) alternating handoffs + local i port pid name bad=0 stranded=0 diff_a=0 diff_b=0 ref_b= lat=() holder_bad=0 + for i in $(seq 0 30); do + if [ $((i % 2)) -eq 0 ]; then port=$PB pid=$B_PID name=B; else port=$PA pid=$A_PID name=A; fi + local t0 t1 out=$S/h$i.json + t0=$(now_ms) + code=$(complete "$port" "$out" "$(greedy_body 48 false)" 60) + t1=$(now_ms) + [ $i -gt 0 ] && lat+=($((t1 - t0))) + if [ "$code" = 000 ]; then stranded=$((stranded + 1)); continue; fi + [ "$code" != 200 ] && { bad=$((bad + 1)); continue; } + local c; c=$(json_str content "$out") + if [ $name = A ]; then [ "$c" = "$ref_a" ] || diff_a=$((diff_a + 1)) + else if [ -z "$ref_b" ]; then ref_b=$c; elif [ "$c" != "$ref_b" ]; then diff_b=$((diff_b + 1)); fi; fi + [ "$(token_holder)" = "$pid" ] || holder_bad=$((holder_bad + 1)) + done + local med max + med=$(printf '%s\n' "${lat[@]}" | sort -n | awk '{a[NR]=$1} END {print a[int((NR+1)/2)]}') + max=$(printf '%s\n' "${lat[@]}" | sort -n | tail -1) + check arbiter/a-handoffs $(( stranded == 0 && bad == 0 && holder_bad == 0 ? 0 : 1 )) \ + "30 handoffs: $stranded stranded, $bad non-200, $holder_bad served-without-token; request latency median ${med} ms, max ${max} ms" + check arbiter/b-greedy-identical $(( diff_a == 0 && diff_b == 0 ? 0 : 1 )) \ + "A: 15/15 == no-handoff reference ($diff_a differ); B: $diff_b of 15 differ from its first; A==B: $([ "$ref_a" = "$ref_b" ] && echo yes || echo no)" + + # ---- (b') KV / prompt cache survives an evict + restore (LLAMA_SLEEP_EVICT_KV=1) + local p1 p2 c1 c2 + code=$(complete "$PA" "$S/kv1.json" "$(greedy_body 32 true 'Explain in detail how a refrigerator moves heat from inside to outside')") + complete "$PB" "$S/kv-b.json" "$(greedy_body 8 false)" > /dev/null + [ "$(token_holder)" = "$B_PID" ] || log "warning: B does not hold the token before the KV re-check" + complete "$PA" "$S/kv2.json" "$(greedy_body 32 true 'Explain in detail how a refrigerator moves heat from inside to outside')" > /dev/null + p1=$(json_num prompt_n "$S/kv1.json"); p2=$(json_num prompt_n "$S/kv2.json") + c1=$(json_str content "$S/kv1.json"); c2=$(json_str content "$S/kv2.json") + check arbiter/b-kv-survives $(( ${p2:-99} < ${p1:-0} && ${#c1} > 0 ? 0 : 1 )) \ + "prompt_n $p1 -> $p2 after evict/restore (cache reused), output $([ "$c1" = "$c2" ] && echo identical || echo DIFFERENT)" + [ "$c1" = "$c2" ] || fail arbiter/b-kv-output "cached-prompt output changed across evict/restore" + + # ---- (c) image after release/restore + complete "$PB" "$S/c-b.json" "$(greedy_body 8 false)" > /dev/null # A goes cold + local cold_ok=1; [ "$(token_holder)" = "$B_PID" ] && cold_ok=0 + local img; img=$(make_red_ppm) + printf '{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/x-portable-pixmap;base64,%s"}},{"type":"text","text":"What is the main color of this image? Answer with one word."}]}],"max_tokens":16,%s}' \ + "$img" "$GREEDY" > "$S/c-body.json" + code=$(curl -sS -m 120 -o "$S/c-a.json" -w '%{http_code}' -H 'Content-Type: application/json' \ + -d @"$S/c-body.json" "http://127.0.0.1:$PA/v1/chat/completions" 2>>"$S/curl.err") + local ans; ans=$(json_str content "$S/c-a.json") + check arbiter/c-image-after-restore $(( cold_ok == 0 && ${#code} > 0 && code == 200 ? 0 : 1 )) \ + "A cold before: $([ $cold_ok -eq 0 ] && echo yes || echo NO), HTTP $code, answer: '$ans'" + echo "$ans" | grep -qi red || fail arbiter/c-image-answer "expected 'red', got '$ans'" + # text right after the image path (encoder out, weights back) + code=$(complete "$PA" "$S/c-a2.json" "$(greedy_body 48 false)") + [ "$code" = 200 ] && [ "$(json_str content "$S/c-a2.json")" = "$ref_a" ] \ + || fail arbiter/c-text-after-image "HTTP $code, output differs from reference after the image request" + + # ---- (g) kill -9 a waiter that holds want.lock -> the holder stops yielding + stream_bg "$PA" "$S/g-a.sse" "$LONG_BODY"; local gcl=$CPID + if ! wait_tokens "$S/g-a.sse" 3 30000; then fail arbiter/g-kill9-waiter "A did not start streaming"; else + local s_before; s_before=$(sleeps "$S/A.log") + complete "$PB" "$S/g-b.json" "$(greedy_body 8 false)" 60 > /dev/null & + local gbc=$!; track "$gbc" + if ! wait_want "$B_PID" 10000; then + fail arbiter/g-kill9-waiter "B never registered on want.lock (setup)" + else + kill -KILL "$B_PID"; wait "$B_PID" 2>/dev/null + kill "$gcl" 2>/dev/null; wait "$gcl" 2>/dev/null # end A's generation (client abort) + sleep 2 + local wl; wl=$(lock_pids "$ARENA/want.lock" READ | wc -l) + check arbiter/g-kill9-waiter $(( $(holder_num) == A_PID && $(sleeps "$S/A.log") == s_before && wl == 0 ? 0 : 1 )) \ + "after kill -9 of waiting B: want.lock readers $wl, token holder $(token_holder) (A=$A_PID), A sleeps +$(( $(sleeps "$S/A.log") - s_before ))" + fi + wait "$gbc" 2>/dev/null + fi + log "arbiter: restarting B" + start_b + wait_health "$PB" "$B_PID" 180 || { fail arbiter/restart "B did not come back"; return; } + + # ---- (e) SIGTERM while A waits for the token -> exits fast, never restores + stream_bg "$PB" "$S/e-b.sse" "$LONG_BODY"; local ecl=$CPID + wait_tokens "$S/e-b.sse" 3 30000 || { fail arbiter/e-term-waiting "B did not start streaming"; } + complete "$PA" "$S/e-a.json" "$(greedy_body 8 false)" 30 > /dev/null & + local eac=$!; track "$eac" + if wait_want "$A_PID" 10000; then + sleep 0.3 + local t0 ms took_token=0 + t0=$(now_ms); kill -TERM "$A_PID" + while pid_alive "$A_PID"; do + [ "$(token_holder)" = "$A_PID" ] && took_token=1 + [ $(( $(now_ms) - t0 )) -gt 10000 ] && break + sleep 0.01 + done + ms=$(( $(now_ms) - t0 )); wait "$A_PID"; RC=$? + check arbiter/e-term-waiting $(( RC == 0 && ms <= 1500 && took_token == 0 && $(holder_num) == B_PID ? 0 : 1 )) \ + "exit $RC in ${ms} ms, A took the token: $([ $took_token = 1 ] && echo YES || echo no), holder now $(token_holder) (B=$B_PID), log: $(grep -c 'no longer waiting for the GPU token' "$S/A.log") abort line(s)" + else + fail arbiter/e-term-waiting "A never registered on want.lock (setup)" + stop_server "$A_PID" + fi + kill "$ecl" 2>/dev/null; wait "$ecl" "$eac" 2>/dev/null + log "arbiter: restarting A" + start_a + wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; } + + # ---- (d) SIGTERM during A's VRAM release -> exit 0 within ~2 s, B gets the token + # A is warm (fresh load, first release also captures the host shadow: the longest release) + local how sig_ms + if command -v gdb > /dev/null; then + how=gdb + gdb -p "$A_PID" -batch -nx -ex 'set pagination off' -ex 'handle all nostop noprint pass' \ + -ex 'break llama_context_release_device' -ex 'continue' \ + -ex "shell date +%s%3N > $S/d.sig" -ex 'delete' -ex 'signal SIGTERM' > "$S/d-gdb.log" 2>&1 & + local gpid=$!; track "$gpid" + local k=0; until grep -q '^Continuing' "$S/d-gdb.log" || [ $k -gt 300 ]; do sleep 0.1; k=$((k + 1)); done + sleep 0.5 + else + how=log-poll + fi + local s0; s0=$(sleeps "$S/A.log") + complete "$PB" "$S/d-b.json" "$(greedy_body 8 false)" 60 > "$S/d-b.code" & + local dbc=$!; track "$dbc" + if [ $how = log-poll ]; then + local k=0; until [ "$(sleeps "$S/A.log")" -gt "$s0" ] || [ $k -gt 2000 ]; do sleep 0.005; k=$((k + 1)); done + now_ms > "$S/d.sig"; kill -TERM "$A_PID" + else + local k=0; until [ -s "$S/d.sig" ] || [ $k -gt 600 ]; do sleep 0.05; k=$((k + 1)); done + fi + if [ ! -s "$S/d.sig" ]; then + fail arbiter/d-term-in-release "release breakpoint never hit ($how), see $S/d-gdb.log" + kill "$gpid" 2>/dev/null; stop_server "$A_PID" + else + while pid_alive "$A_PID" && [ $(( $(now_ms) - $(cat "$S/d.sig") )) -lt 20000 ]; do sleep 0.01; done + sig_ms=$(( $(now_ms) - $(cat "$S/d.sig") )) + wait "$A_PID"; RC=$? + wait "$dbc" + local dcode; dcode=$(cat "$S/d-b.code") + sleep 0.2 + check arbiter/d-term-in-release $(( RC == 0 && sig_ms <= 2500 && dcode == 200 && $(holder_num) == B_PID ? 0 : 1 )) \ + "SIGTERM inside llama_context_release_device ($how): A exit $RC in ${sig_ms} ms, B HTTP $dcode, holder $(token_holder) (B=$B_PID)" + fi + log "arbiter: restarting A" + start_a + wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; } + + # ---- (f) SIGTERM mid-generation (streaming) + stream_bg "$PA" "$S/f-a.sse" "$LONG_BODY"; local fcl=$CPID + if ! wait_tokens "$S/f-a.sse" 10 30000; then fail arbiter/f-term-streaming "A did not start streaming"; else + local t0 free_ms=-1 cl_ms=-1 exit_ms=-1 + t0=$(now_ms); kill -TERM "$A_PID" + while [ $(( $(now_ms) - t0 )) -lt 10000 ]; do + [ $free_ms -lt 0 ] && [ -z "$(token_holder)" ] && free_ms=$(( $(now_ms) - t0 )) + [ $cl_ms -lt 0 ] && ! pid_alive "$fcl" && cl_ms=$(( $(now_ms) - t0 )) + [ $exit_ms -lt 0 ] && ! pid_alive "$A_PID" && exit_ms=$(( $(now_ms) - t0 )) + [ $free_ms -ge 0 ] && [ $cl_ms -ge 0 ] && [ $exit_ms -ge 0 ] && break + sleep 0.01 + done + wait "$A_PID"; RC=$?; wait "$fcl" 2>/dev/null + local errev; errev=$(grep -c '"error"' "$S/f-a.sse") + check arbiter/f-term-streaming $(( RC == 0 && errev > 0 && cl_ms >= 0 && free_ms >= 0 && free_ms <= 2500 && exit_ms >= 0 ? 0 : 1 )) \ + "A exit $RC in ${exit_ms} ms, token free in ${free_ms} ms, client closed in ${cl_ms} ms with $errev error event(s) after $(grep -c '^data: ' "$S/f-a.sse") data lines" + fi + log "arbiter: restarting A" + start_a + wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; } + + # ---- (h) clean shutdown warm and cold + [ "$(token_holder)" = "$A_PID" ] || log "warning: A is not warm before the warm-shutdown case" + stop_server "$A_PID" + check arbiter/h-shutdown-warm $(( RC == 0 ? 0 : 1 )) "warm A: exit $RC in ${EXIT_MS} ms" + start_a + wait_health "$PA" "$A_PID" 180 || { fail arbiter/restart "A did not come back"; return; } + complete "$PB" "$S/h-b.json" "$(greedy_body 8 false)" > /dev/null # A goes cold + local acold=no; [ "$(token_holder)" = "$B_PID" ] && acold=yes + stop_server "$A_PID" + check arbiter/h-shutdown-cold $([ $RC -eq 0 ] && [ $acold = yes ] && echo 0 || echo 1) \ + "cold A (B holds token: $acold): exit $RC in ${EXIT_MS} ms" + stop_server "$B_PID" + check arbiter/h-shutdown-warm-b $(( RC == 0 ? 0 : 1 )) "warm B: exit $RC in ${EXIT_MS} ms" + + local bad_lines; bad_lines=$(scan_logs "$S/A.log" "$S/B.log") + check arbiter/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) \ + "$([ -z "$bad_lines" ] && echo 'no asserts / Vulkan errors in A/B logs' || echo "$bad_lines" | head -3 | tr '\n' ' ')" +} + +# ================================================================ 4. perf +vram_sampler() { # OUTFILE : max of mem_info_vram_used every 0.25 s, written continuously + local m=0 v + while :; do v=$(cat "$VRAM_USED"); [ "$v" -gt "$m" ] && { m=$v; echo "$m" > "$1"; }; sleep 0.25; done +} +bench_rows() { # LOG -> "testt/s" rows from llama-bench markdown + awk -F'|' '/\| *(pp|tg)[0-9]+/ {t=$(NF-2); v=$(NF-1); gsub(/^ +| +$/, "", t); split(v, a, " "); print t "\t" a[1]}' "$1" +} +run_perf() { + if supervisorctl -c /root/supervisord.conf status llama-swap 2>/dev/null | grep -q RUNNING; then + fail perf "llama-swap is RUNNING; run under /root/bench/moe/swapdown.sh"; return + fi + local others; others=$(ps -eo pid=,comm= | awk '$2 ~ /^llama-(server|bench|cli)$/' | wc -l) + [ "$others" -eq 0 ] || { fail perf "$others other llama processes are running"; return; } + [ -f "$PERF_MODEL" ] || { fail perf "model missing: $PERF_MODEL"; return; } + [ -x "$REF/bin/llama-bench" ] && [ -x "$BUILD/bin/llama-bench" ] || { fail perf "llama-bench missing"; return; } + local common=(-m "$PERF_MODEL" -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none) + local arm k=0 base; base=$(vram_mb) + log "perf: idle VRAM $base MiB; arms ref,cand,cand,ref" + for arm in ref cand cand ref; do + k=$((k + 1)) + local b; [ $arm = ref ] && b=$REF || b=$BUILD + local tag=$k-$arm + vram_sampler "$S/perf-$tag.vram" & local vs=$!; track "$vs" + log "perf: arm $tag pp sweep" + LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 2048 -n 0 -d 0,16384,32768 -r 2 > "$S/perf-$tag-pp.log" 2>&1 + log "perf: arm $tag tg" + LD_LIBRARY_PATH="$b/bin" "$b/bin/llama-bench" "${common[@]}" -p 0 -n 256 -d 4096 -r 3 > "$S/perf-$tag-tg.log" 2>&1 + kill "$vs" 2>/dev/null; wait "$vs" 2>/dev/null + { bench_rows "$S/perf-$tag-pp.log"; bench_rows "$S/perf-$tag-tg.log"; } | sed "s/^/$arm\t/" >> "$S/perf.tsv" + echo "$arm $(cat "$S/perf-$tag.vram" 2>/dev/null || echo 0)" >> "$S/perf-vram.txt" + done + echo + echo "perf: mean t/s over 2 arms each (order ref,cand,cand,ref)" + printf '%-18s %10s %10s %8s\n' test reference candidate delta + local tests t r c d bad=0 + tests=$(awk -F'\t' '{print $2}' "$S/perf.tsv" | awk '!s[$0]++') + local IFS_=$IFS; IFS=$'\n' + for t in $tests; do + r=$(awk -F'\t' -v t="$t" '$1=="ref" && $2==t {s+=$3; n++} END {if (n) printf "%.2f", s/n}' "$S/perf.tsv") + c=$(awk -F'\t' -v t="$t" '$1=="cand" && $2==t {s+=$3; n++} END {if (n) printf "%.2f", s/n}' "$S/perf.tsv") + d=$(awk -v r="$r" -v c="$c" 'BEGIN {if (r > 0) printf "%+.1f%%", (c/r - 1)*100}') + printf '%-18s %10s %10s %8s\n' "$t" "$r" "$c" "$d" + case "$t" in pp*) + if [ -z "$r" ] || [ -z "$c" ] || [ "$(awk -v r="$r" -v c="$c" 'BEGIN {print (c < 0.95*r)}')" = 1 ]; then + bad=$((bad + 1)); fail "perf/$t" "candidate $c vs reference $r t/s ($d, limit -5%)" + else + pass "perf/$t" "candidate $c vs reference $r t/s ($d)" + fi ;; + tg*) RESULTS+=("INFO perf/$t: candidate $c vs reference $r t/s ($d; decode is CPU-bound and noisy here, not gated)") ;; + esac + done + IFS=$IFS_ + local n_tests; n_tests=$(echo "$tests" | grep -c '^pp') + [ "$n_tests" -ge 3 ] || fail perf/rows "expected 3 pp rows, got $n_tests (see $S/perf-*-pp.log)" + local total; total=$(cat "$VRAM_TOTAL") + local vr vc + vr=$(awk '$1=="ref" && $2>m {m=$2} END {print m+0}' "$S/perf-vram.txt") + vc=$(awk '$1=="cand" && $2>m {m=$2} END {print m+0}' "$S/perf-vram.txt") + check perf/peak-vram $(( vc * 100 <= total * 95 ? 0 : 1 )) \ + "peak VRAM reference $((vr / 1048576)) MiB, candidate $((vc / 1048576)) MiB of $((total / 1048576)) MiB (limit 95%)" +} + +# ================================================================ 5. MoE cache release/restore smoke +run_moecache() { + [ -f "$MOECACHE_MODEL" ] || { fail moecache "model missing: $MOECACHE_MODEL"; return; } + local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 )) + log "moecache: $free_mb MiB VRAM free; server on :$PC with --moe-cache-mib $MOECACHE_MIB" + local arena=$S/arena-moec; mkdir -p "$arena" + # -lv 4: the cache size and hit/miss stats are only logged at that verbosity + # shellcheck disable=SC2086 + start_server M "$BUILD" "$PC" LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$arena" LLAMA_SLEEP_EVICT_KV=1 -- \ + -m "$MOECACHE_MODEL" $MOECACHE_ARGS -fa 1 -lm none --parallel 1 --no-warmup --reasoning off -lv 4 \ + --moe-cache-mib "$MOECACHE_MIB" + local mp=$SPID + wait_health "$PC" "$mp" 600 || { fail moecache "server did not start, see $S/M.log"; return; } + # chat endpoint: an instruct model fed a raw prompt can loop, which says nothing about the cache + local body="{\"messages\":[{\"role\":\"user\",\"content\":\"What are the three primary colors of light? Answer in one short sentence.\"}],\"max_tokens\":64,$GREEDY}" + local outs=() i code w + for i in 0 1 2; do + code=$(curl -sS -m 300 -o "$S/m$i.json" -w '%{http_code}' -H 'Content-Type: application/json' \ + -d "$body" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err") + outs+=("$(json_str content "$S/m$i.json")") + [ "$code" = 200 ] || { fail moecache "request $i: HTTP $code"; stop_server "$mp"; return; } + [ $i -eq 2 ] && break + # force a release: a fake waiter holds want.lock shared until the server drops the token + ( exec 9<"$arena/want.lock"; flock -s 9; exec sleep 120 ) & w=$!; track "$w" + local k=0; until [ -z "$(awk -v id="$(lock_id "$arena/token.lock")" '$2=="FLOCK" && $4=="WRITE" && $6==id' /proc/locks)" ] || [ $k -gt 600 ]; do sleep 0.05; k=$((k + 1)); done + log "moecache: cycle $((i + 1)) cold, VRAM $(vram_mb) MiB" + kill "$w"; wait "$w" 2>/dev/null + done + stop_server "$mp" + local size hits rel res + size=$(sed -n 's/.*MoE cache size = *\([0-9.]*\) MiB.*/\1/p' "$S/M.log" | head -1) + hits=$(sed -n 's/.*llama_moe_cache: .* hits = \([0-9]*\),.*/\1/p' "$S/M.log" | awk '{s += $1} END {print s+0}') + rel=$(grep -c 'released .* MiB of MoE cache' "$S/M.log"); res=$(grep -c 'restored .* MiB of MoE cache' "$S/M.log") + check moecache/active $(( hits > 0 ? 0 : 1 )) \ + "cache ${size:-?} MiB, $hits hits (0 hits = cache too small to ever reuse an expert, so a stale SLRU would go unnoticed), released ${rel}x restored ${res}x" + local same=1; [ "${outs[0]}" = "${outs[1]}" ] && [ "${outs[1]}" = "${outs[2]}" ] && [ -n "${outs[0]}" ] && same=0 + check moecache/identical $same "3 greedy answers across 2 cold/warm cycles $([ $same = 0 ] && echo identical || echo DIFFER): '$(echo "${outs[0]}" | cut -c1-80)'" + local coh=1; echo "${outs[2]}" | grep -qi red && echo "${outs[2]}" | grep -qi green && echo "${outs[2]}" | grep -qi blue && coh=0 + check moecache/coherent $coh "answer after the 2nd restore names red, green and blue" + check moecache/shutdown $(( RC == 0 ? 0 : 1 )) "exit $RC" + local bad_lines; bad_lines=$(scan_logs "$S/M.log") + check moecache/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}" +} + +# ================================================================ main +[ $DO_OPS -eq 1 ] && run_ops +[ $DO_CONCAT -eq 1 ] && run_concat +[ $DO_ARB -eq 1 ] && run_arbiter +[ $DO_PERF -eq 1 ] && run_perf +[ $DO_MOEC -eq 1 ] && run_moecache +exit 0