From ceab1a91f889bbc81e1d126aeda95c22bd45a5bb Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 25 Oct 2026 02:43:51 +0200 Subject: [PATCH] scripts: fork-validate.sh - check that a cold server gives its VRAM back Nothing checked that a release actually frees VRAM. --arbiter now takes a baseline of mem_info_vram_used before any scratch server starts, then with A warm and alone on the scratch arena makes it release through a fake waiter on want.lock. v-vram-release requires the card-wide delta to drop by at least 70% of what A added warm; v-vram-restore requires it back (and the output equal to the reference) after the next request. Production's own share is re-read at every point from the DRM fdinfo of every process the script did not start (deduplicated by drm-client-id); if it moved by more than 64 MiB the check SKIPs instead of blaming the candidate. Run beside production on the pod (reb1024 vs itself): warm +546 MiB, cold +4 MiB, restored +546 MiB, production steady at 5053 MiB; 16 PASS. Co-Authored-By: Claude Opus 5.5 --- scripts/fork-validate.sh | 67 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh index 58f0c3439..7f337e10c 100644 --- a/scripts/fork-validate.sh +++ b/scripts/fork-validate.sh @@ -17,7 +17,11 @@ # Needs a build carrying the GGML_VK_CONCAT_FASTPATH knob (=0 off, =log report); # either --build has it or pass --concat-build DIR (candidate + the knob patch). # --arbiter on-demand VRAM sharing (LLAMA_SLEEP_VRAM_ONLY ...) with two real servers on a scratch -# arena -> handoffs never strand, release/restore is bit-exact, multimodal survives it, +# arena -> a cold server really gives its VRAM back (v-vram-release: the card-wide +# delta over a baseline taken before the scratch servers start must drop by >= 70% of +# what it added warm, and come back on restore; production's own DRM clients are +# re-read each time and the check SKIPs if they moved by > 64 MiB), handoffs never +# strand, release/restore is bit-exact, multimodal survives it, # and every shutdown path (mid-release, mid-wait, mid-generation, cold, warm, kill -9 of # a waiter) exits cleanly and hands the GPU token on. These are the past production # incidents (stranded requests, wedged token, exit 134 on cold teardown). @@ -255,6 +259,29 @@ wait_want() { # PID TIMEOUT_MS : until PID holds want.lock shared (registered wa done } +# ---------------------------------------------------------------- helpers: VRAM attribution +# mem_info_vram_used is card-wide and includes production. Production's own share is read from the +# DRM fdinfo of every process this script did not start (drm-memory-vram, deduplicated by +# drm-client-id; summing fds double-counts shared buffers), so a production model that warms or +# cools during a measurement is detected instead of being attributed to the scratch servers. +drm_vram_kib() { # PID... -> KiB of VRAM held by the DRM clients of these processes + local p fs=() + for p in "$@"; do [ -d "/proc/$p/fdinfo" ] && fs+=("/proc/$p/fdinfo/"*); done + [ ${#fs[@]} -gt 0 ] || { echo 0; return; } + cat "${fs[@]}" 2>/dev/null | awk '/^pos:/ {c = ""} /^drm-client-id:/ {c = $2} + /^drm-memory-vram:/ && c != "" {v = $2; if ($3 == "MiB") v *= 1024; m[c] = v} + END {for (k in m) s += m[k]; print s + 0}' +} +prod_vram_mb() { # MiB of VRAM held by every process this script did not start + local p e mine=" " others=() + for e in "${PIDS[@]}"; do mine+="${e%%:*} "; done + for p in /proc/[0-9]*; do + p=${p#/proc/} + case "$mine" in *" $p "*) ;; *) others+=("$p") ;; esac + done + echo $(( $(drm_vram_kib "${others[@]}") / 1024 )) +} + # ---------------------------------------------------------------- helpers: requests json_str() { sed -n 's/.*"'"$1"'":"\(\([^"\\]\|\\.\)*\)".*/\1/p' "$2" | head -1; } json_num() { grep -o "\"$1\":[0-9]*" "$2" | head -1 | cut -d: -f2; } @@ -391,7 +418,9 @@ run_arbiter() { if [ $free_mb -lt 1536 ]; then fail arbiter "only $free_mb MiB VRAM free (< 1536): run inside the swapdown window"; return fi - log "arbiter: $free_mb MiB VRAM free; starting A (mmproj) on :$PA" + # card-wide and production VRAM before any scratch server exists (see prod_vram_mb) + local v0 p0; v0=$(vram_mb); p0=$(prod_vram_mb) + log "arbiter: $free_mb MiB VRAM free (card $v0 MiB used, production DRM clients $p0 MiB); starting A (mmproj) on :$PA" start_a wait_health "$PA" "$A_PID" 180 || { fail arbiter/start "A did not become healthy, see $S/A.log"; return; } @@ -402,6 +431,40 @@ run_arbiter() { [ "$code" = 200 ] && [ -n "$ref_a" ] || { fail arbiter/start "reference request on A: HTTP $code"; return; } [ "$(token_holder)" = "$A_PID" ] || { fail arbiter/start "A serves but does not hold token.lock"; return; } + # ---- (v) VRAM really drops when A goes cold, and comes back when it warms again + # A is alone on the scratch arena; a fake waiter on want.lock makes it release. Its footprint is + # the card-wide delta against the baseline taken before A started; production's DRM clients are + # re-read at every point, and if they moved the check is SKIPped (it cannot attribute the delta). + local v_warm p_warm v_cold p_cold v_back p_back f_warm f_cold w k=0 + sleep 0.3 + v_warm=$(vram_mb); p_warm=$(prod_vram_mb); f_warm=$(( $(drm_vram_kib "$A_PID") / 1024 )) + local added=$(( v_warm - v0 )) need + need=$(( added * 7 / 10 )) + ( exec 9<"$ARENA/want.lock"; flock -s 9; exec sleep 120 ) & w=$!; track "$w" + if ! wait_holder "" 10000 > /dev/null; then + fail arbiter/v-vram-release "A did not drop the token for a waiter within 10 s (holder $(token_holder))" + else + # the release frees before it unlocks; give the driver a moment to return the pages + while [ $(( v_warm - $(vram_mb) )) -lt $need ] && [ $k -lt 30 ]; do sleep 0.1; k=$((k + 1)); done + fi + v_cold=$(vram_mb); p_cold=$(prod_vram_mb); f_cold=$(( $(drm_vram_kib "$A_PID") / 1024 )) + kill "$w" 2>/dev/null; wait "$w" 2>/dev/null + code=$(complete "$PA" "$S/v-a.json" "$(greedy_body 48 false)") + sleep 0.3 + v_back=$(vram_mb); p_back=$(prod_vram_mb) + local pmove; pmove=$(printf '%s\n' "$p_warm" "$p_cold" "$p_back" | awk -v b="$p0" '{d = $1 - b; if (d < 0) d = -d; if (d > m) m = d} END {print m + 0}') + local vmsg="card-wide over baseline: warm +${added} MiB, cold +$((v_cold - v0)) MiB (dropped $((v_warm - v_cold)), need >= ${need} = 70%), restored +$((v_back - v0)) MiB; A's own DRM clients warm ${f_warm} / cold ${f_cold} MiB; production ${p0} -> ${p_warm}/${p_cold}/${p_back} MiB" + if [ "$pmove" -gt 64 ]; then + skip arbiter/v-vram-release "production VRAM moved by ${pmove} MiB during the measurement, cannot attribute: $vmsg" + elif [ "$added" -lt 64 ]; then + fail arbiter/v-vram-release "A added only ${added} MiB when warm (measurement broken?): $vmsg" + else + check arbiter/v-vram-release $(( v_warm - v_cold >= need ? 0 : 1 )) "$vmsg" + check arbiter/v-vram-restore $(( code == 200 && v_back - v0 >= need && $(holder_num) == A_PID ? 0 : 1 )) \ + "HTTP $code, A holds the token: $([ "$(token_holder)" = "$A_PID" ] && echo yes || echo NO), restored +$((v_back - v0)) MiB (need >= ${need}), output $([ "$(json_str content "$S/v-a.json")" = "$ref_a" ] && echo identical || echo DIFFERENT)" + [ "$(json_str content "$S/v-a.json")" = "$ref_a" ] || fail arbiter/v-vram-output "output after the release/restore differs from the reference" + fi + log "arbiter: starting B (text-only) on :$PB" start_b wait_health "$PB" "$B_PID" 180 || { fail arbiter/start "B did not become healthy, see $S/B.log"; return; }