scripts: fork-validate.sh - check that a cold server gives its VRAM back
Nothing checked that a release actually frees VRAM. --arbiter now takes a baseline of mem_info_vram_used before any scratch server starts, then with A warm and alone on the scratch arena makes it release through a fake waiter on want.lock. v-vram-release requires the card-wide delta to drop by at least 70% of what A added warm; v-vram-restore requires it back (and the output equal to the reference) after the next request. Production's own share is re-read at every point from the DRM fdinfo of every process the script did not start (deduplicated by drm-client-id); if it moved by more than 64 MiB the check SKIPs instead of blaming the candidate. Run beside production on the pod (reb1024 vs itself): warm +546 MiB, cold +4 MiB, restored +546 MiB, production steady at 5053 MiB; 16 PASS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
80e662a2a1
commit
ceab1a91f8
1 file changed
+65
-2
@@ -17,7 +17,11 @@
|
||||
# Needs a build carrying the GGML_VK_CONCAT_FASTPATH knob (=0 off, =log report);
|
||||
# either --build has it or pass --concat-build DIR (candidate + the knob patch).
|
||||
# --arbiter on-demand VRAM sharing (LLAMA_SLEEP_VRAM_ONLY ...) with two real servers on a scratch
|
||||
# arena -> handoffs never strand, release/restore is bit-exact, multimodal survives it,
|
||||
# arena -> a cold server really gives its VRAM back (v-vram-release: the card-wide
|
||||
# delta over a baseline taken before the scratch servers start must drop by >= 70% of
|
||||
# what it added warm, and come back on restore; production's own DRM clients are
|
||||
# re-read each time and the check SKIPs if they moved by > 64 MiB), handoffs never
|
||||
# strand, release/restore is bit-exact, multimodal survives it,
|
||||
# and every shutdown path (mid-release, mid-wait, mid-generation, cold, warm, kill -9 of
|
||||
# a waiter) exits cleanly and hands the GPU token on. These are the past production
|
||||
# incidents (stranded requests, wedged token, exit 134 on cold teardown).
|
||||
@@ -255,6 +259,29 @@ wait_want() { # PID TIMEOUT_MS : until PID holds want.lock shared (registered wa
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- helpers: VRAM attribution
|
||||
# mem_info_vram_used is card-wide and includes production. Production's own share is read from the
|
||||
# DRM fdinfo of every process this script did not start (drm-memory-vram, deduplicated by
|
||||
# drm-client-id; summing fds double-counts shared buffers), so a production model that warms or
|
||||
# cools during a measurement is detected instead of being attributed to the scratch servers.
|
||||
drm_vram_kib() { # PID... -> KiB of VRAM held by the DRM clients of these processes
|
||||
local p fs=()
|
||||
for p in "$@"; do [ -d "/proc/$p/fdinfo" ] && fs+=("/proc/$p/fdinfo/"*); done
|
||||
[ ${#fs[@]} -gt 0 ] || { echo 0; return; }
|
||||
cat "${fs[@]}" 2>/dev/null | awk '/^pos:/ {c = ""} /^drm-client-id:/ {c = $2}
|
||||
/^drm-memory-vram:/ && c != "" {v = $2; if ($3 == "MiB") v *= 1024; m[c] = v}
|
||||
END {for (k in m) s += m[k]; print s + 0}'
|
||||
}
|
||||
prod_vram_mb() { # MiB of VRAM held by every process this script did not start
|
||||
local p e mine=" " others=()
|
||||
for e in "${PIDS[@]}"; do mine+="${e%%:*} "; done
|
||||
for p in /proc/[0-9]*; do
|
||||
p=${p#/proc/}
|
||||
case "$mine" in *" $p "*) ;; *) others+=("$p") ;; esac
|
||||
done
|
||||
echo $(( $(drm_vram_kib "${others[@]}") / 1024 ))
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- helpers: requests
|
||||
json_str() { sed -n 's/.*"'"$1"'":"\(\([^"\\]\|\\.\)*\)".*/\1/p' "$2" | head -1; }
|
||||
json_num() { grep -o "\"$1\":[0-9]*" "$2" | head -1 | cut -d: -f2; }
|
||||
@@ -391,7 +418,9 @@ run_arbiter() {
|
||||
if [ $free_mb -lt 1536 ]; then
|
||||
fail arbiter "only $free_mb MiB VRAM free (< 1536): run inside the swapdown window"; return
|
||||
fi
|
||||
log "arbiter: $free_mb MiB VRAM free; starting A (mmproj) on :$PA"
|
||||
# card-wide and production VRAM before any scratch server exists (see prod_vram_mb)
|
||||
local v0 p0; v0=$(vram_mb); p0=$(prod_vram_mb)
|
||||
log "arbiter: $free_mb MiB VRAM free (card $v0 MiB used, production DRM clients $p0 MiB); starting A (mmproj) on :$PA"
|
||||
start_a
|
||||
wait_health "$PA" "$A_PID" 180 || { fail arbiter/start "A did not become healthy, see $S/A.log"; return; }
|
||||
|
||||
@@ -402,6 +431,40 @@ run_arbiter() {
|
||||
[ "$code" = 200 ] && [ -n "$ref_a" ] || { fail arbiter/start "reference request on A: HTTP $code"; return; }
|
||||
[ "$(token_holder)" = "$A_PID" ] || { fail arbiter/start "A serves but does not hold token.lock"; return; }
|
||||
|
||||
# ---- (v) VRAM really drops when A goes cold, and comes back when it warms again
|
||||
# A is alone on the scratch arena; a fake waiter on want.lock makes it release. Its footprint is
|
||||
# the card-wide delta against the baseline taken before A started; production's DRM clients are
|
||||
# re-read at every point, and if they moved the check is SKIPped (it cannot attribute the delta).
|
||||
local v_warm p_warm v_cold p_cold v_back p_back f_warm f_cold w k=0
|
||||
sleep 0.3
|
||||
v_warm=$(vram_mb); p_warm=$(prod_vram_mb); f_warm=$(( $(drm_vram_kib "$A_PID") / 1024 ))
|
||||
local added=$(( v_warm - v0 )) need
|
||||
need=$(( added * 7 / 10 ))
|
||||
( exec 9<"$ARENA/want.lock"; flock -s 9; exec sleep 120 ) & w=$!; track "$w"
|
||||
if ! wait_holder "" 10000 > /dev/null; then
|
||||
fail arbiter/v-vram-release "A did not drop the token for a waiter within 10 s (holder $(token_holder))"
|
||||
else
|
||||
# the release frees before it unlocks; give the driver a moment to return the pages
|
||||
while [ $(( v_warm - $(vram_mb) )) -lt $need ] && [ $k -lt 30 ]; do sleep 0.1; k=$((k + 1)); done
|
||||
fi
|
||||
v_cold=$(vram_mb); p_cold=$(prod_vram_mb); f_cold=$(( $(drm_vram_kib "$A_PID") / 1024 ))
|
||||
kill "$w" 2>/dev/null; wait "$w" 2>/dev/null
|
||||
code=$(complete "$PA" "$S/v-a.json" "$(greedy_body 48 false)")
|
||||
sleep 0.3
|
||||
v_back=$(vram_mb); p_back=$(prod_vram_mb)
|
||||
local pmove; pmove=$(printf '%s\n' "$p_warm" "$p_cold" "$p_back" | awk -v b="$p0" '{d = $1 - b; if (d < 0) d = -d; if (d > m) m = d} END {print m + 0}')
|
||||
local vmsg="card-wide over baseline: warm +${added} MiB, cold +$((v_cold - v0)) MiB (dropped $((v_warm - v_cold)), need >= ${need} = 70%), restored +$((v_back - v0)) MiB; A's own DRM clients warm ${f_warm} / cold ${f_cold} MiB; production ${p0} -> ${p_warm}/${p_cold}/${p_back} MiB"
|
||||
if [ "$pmove" -gt 64 ]; then
|
||||
skip arbiter/v-vram-release "production VRAM moved by ${pmove} MiB during the measurement, cannot attribute: $vmsg"
|
||||
elif [ "$added" -lt 64 ]; then
|
||||
fail arbiter/v-vram-release "A added only ${added} MiB when warm (measurement broken?): $vmsg"
|
||||
else
|
||||
check arbiter/v-vram-release $(( v_warm - v_cold >= need ? 0 : 1 )) "$vmsg"
|
||||
check arbiter/v-vram-restore $(( code == 200 && v_back - v0 >= need && $(holder_num) == A_PID ? 0 : 1 )) \
|
||||
"HTTP $code, A holds the token: $([ "$(token_holder)" = "$A_PID" ] && echo yes || echo NO), restored +$((v_back - v0)) MiB (need >= ${need}), output $([ "$(json_str content "$S/v-a.json")" = "$ref_a" ] && echo identical || echo DIFFERENT)"
|
||||
[ "$(json_str content "$S/v-a.json")" = "$ref_a" ] || fail arbiter/v-vram-output "output after the release/restore differs from the reference"
|
||||
fi
|
||||
|
||||
log "arbiter: starting B (text-only) on :$PB"
|
||||
start_b
|
||||
wait_health "$PB" "$B_PID" 180 || { fail arbiter/start "B did not become healthy, see $S/B.log"; return; }
|
||||
|
||||
Reference in new issue
Block a user