From 70d39bdbcaf200b6477edb0610e03e6ba3f5b356 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 22 Nov 2026 20:59:28 +0100 Subject: [PATCH] scripts: fork-validate.sh - scratch arenas on /dev/shm, lock_id failed on overlayfs Since 2026-11-22 the pod's root filesystem persists and /tmp is on overlayfs. There stat reports the overlay device (st_dev 0xb3) while /proc/locks prints the device of the upper filesystem (00:84), so lock_id never matched the scratch token.lock and arbiter/start failed with "A serves but does not hold token.lock" although A held the lock. Production's arena is on tmpfs (/dev/shm/llama-vram), where both agree. Put the scratch arenas there too (/dev/shm/fork-validate., removed by finish()); logs and the rest of the scratch stay in /tmp. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ --- scripts/fork-validate.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh index becd5d499..9c95f2f0b 100644 --- a/scripts/fork-validate.sh +++ b/scripts/fork-validate.sh @@ -149,7 +149,10 @@ now_ms() { date +%s%3N; } # ---------------------------------------------------------------- scratch + cleanup S=$(mktemp -d /tmp/fork-validate.XXXXXX) -ARENA=$S/arena +# the arena lives on tmpfs, like production's /dev/shm/llama-vram: on overlayfs (the pod's rootfs, +# so /tmp, since 2026-11-22) stat reports the overlay device while /proc/locks prints the upper +# filesystem's, so lock_id never matched and every token check failed +ARENA=/dev/shm/fork-validate.${S##*.} mkdir -p "$ARENA" cd "$S" || exit 2 # anything written to a relative path lands in the scratch dir PIDS=() # "pid:starttime" of every background process we start (starttime guards PID reuse) @@ -183,6 +186,7 @@ finish() { local rc=$? trap - EXIT INT TERM HUP cleanup + rm -rf "$ARENA" "$ARENA-moec" echo echo "==================== fork-validate summary ====================" echo "candidate: $BUILD ($(git -C "$BUILD/.." rev-parse --short HEAD 2>/dev/null || echo '?'))" @@ -822,7 +826,7 @@ run_moecache() { [ -f "$MOECACHE_MODEL" ] || { fail moecache "model missing: $MOECACHE_MODEL"; return; } local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 )) log "moecache: $free_mb MiB VRAM free; server on :$PC with --moe-cache-mib $MOECACHE_MIB" - local arena=$S/arena-moec; mkdir -p "$arena" + local arena=$ARENA-moec; mkdir -p "$arena" # -lv 4: the cache size and hit/miss stats are only logged at that verbosity # shellcheck disable=SC2086 start_server M "$BUILD" "$PC" LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$arena" LLAMA_SLEEP_EVICT_KV=1 -- \