scripts: fork-validate.sh --diskcache, the disk prompt cache

Runs beside production. The model is the small hybrid (Qwen3.5-0.8B), CPU only
(-dev none -ngl 0, no VRAM), with chat requests and greedy decoding. A
reference server that never restarts runs turn 1 and turn 2. The candidate
saves turn 1 on SIGTERM and restarts. Turn 2 must then be restored from disk
with the same prompt_n as the reference (only the suffix processed) and the
same answer. Cases:
- restore plain, with the arbiter env warm, and forced cold by a fake waiter
  on want.lock (the save from a released state). Each case records the exit
  time and SIGTERM-to-token-free, also once without the cache;
- no restore for another conversation, or with another n_ctx, KV type or
  model path (other key);
- a truncated file is removed at startup and a byte-flipped one at load,
  never used;
- the size limit evicts the oldest file;
- a write error (RLIMIT_FSIZE 1 MiB, SIGXFSZ ignored: EFBIG) is logged, the
  exit stays 0 with no file left, and the next start serves;
- the same restore on the GPU (-ngl 99, arbiter env, warm and cold) when
  >= 1.5 GiB of VRAM is free, else SKIP.
start_server gains SRV_WRAP, a command prefix that execs the server.

First run (2026-10-25, beside production): 11 PASS, 0 FAIL; the GPU leg
SKIPped (1219 MiB free). The first run found the CPU_REPACK release crash
(exit 139 in the cold case).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
LumpiastyandClaude Opus 5.5 committed 2026-10-25 18:55:57 +01:00
1 parent 2874dcb107
commit 2c6c517ba5
1 file changed
+200 -3
+200 -3
View File
@@ -51,6 +51,13 @@
# (=0); =log proves the skip engaged. Without the knob skip-engaged is SKIPped with the
# batch-size argument. Needs the GPU to itself like --perf (same guard); about 3 model
# loads, so run it in the same swapdown window as --perf.
# --diskcache the disk prompt cache (--cache-disk-path) on the small hybrid model, CPU only (no VRAM;
# runs beside production), plus the GPU path when >= 1.5 GiB VRAM is free
# -> a slot state saved on SIGTERM (plain, arbiter warm, arbiter cold) is restored after a
# restart: only the suffix is processed (prompt_n == a server that never restarted) and
# the greedy answer is identical; no restore for another conversation or another model
# path / n_ctx / KV type; truncated and corrupt files are removed, never used; the size
# limit evicts the oldest file; a write error (RLIMIT_FSIZE) is logged and exit stays 0.
#
# Options:
# --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build)
@@ -89,7 +96,7 @@ MOEX_ARGS="-ngl 99 --n-cpu-moe 99 -c 4096 -b 2048 -ub 2048 -t 6"
BUILD= ; REF= ; CONCAT_BUILD=
PORT_BASE=8091
KEEP=0
DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 DO_MOEX=0
DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 DO_MOEX=0 DO_DC=0
PERF_REPORT=
usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; }
@@ -101,6 +108,7 @@ while [ $# -gt 0 ]; do
--ops) DO_OPS=1; shift ;;
--concat) DO_CONCAT=1; shift ;;
--arbiter) DO_ARB=1; shift ;;
--diskcache) DO_DC=1; shift ;;
--perf) DO_PERF=1; shift ;;
--moecache) DO_MOEC=1; MOECACHE_MODEL=$2; shift 2 ;;
--moecache-args) MOECACHE_ARGS=$2; shift 2 ;;
@@ -119,7 +127,7 @@ while [ $# -gt 0 ]; do
esac
done
[ -n "$PERF_REPORT" ] || { [ -n "$BUILD" ] && [ -n "$REF" ]; } || usage
if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC + DO_MOEX)) -eq 0 ]; then
if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC + DO_MOEX + DO_DC)) -eq 0 ]; then
DO_OPS=1 DO_CONCAT=1 DO_ARB=1
fi
[ -n "$MMPROJ" ] || MMPROJ=$(dirname "$SMALL")/mmproj-F16.gguf
@@ -215,12 +223,14 @@ log "scratch dir $S, ports $PA/$PB/$PC, VRAM used $(vram_mb) MiB"
# ---------------------------------------------------------------- helpers: servers
# start_server NAME BUILD PORT ENV-ASSIGNMENTS... -- SERVER-ARGS... (sets SPID, log at $S/NAME.log)
# SRV_WRAP: optional command prefix that execs its arguments (the PID stays the server's)
SRV_WRAP=()
start_server() {
local name=$1 build=$2 port=$3; shift 3
local envs=()
while [ $# -gt 0 ] && [ "$1" != -- ]; do envs+=("$1"); shift; done
shift
env LD_LIBRARY_PATH="$build/bin" "${envs[@]}" "$build/bin/llama-server" \
"${SRV_WRAP[@]}" env LD_LIBRARY_PATH="$build/bin" "${envs[@]}" "$build/bin/llama-server" \
--host 127.0.0.1 --port "$port" "$@" >> "$S/$name.log" 2>&1 &
SPID=$!
track "$SPID"
@@ -939,6 +949,192 @@ run_moe_exact() {
check moe-exact/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}"
}
# ================================================================ 7. disk prompt cache
# --cache-disk-path: on shutdown each slot's state goes to a file; after a restart a prompt that starts
# with a saved prompt (or with one of its checkpoints, for a hybrid model) is restored from the file.
# Method: chat requests on the small hybrid model, greedy. A reference server that never restarts
# gives turn 2 (turn 1 + answer + a new question); the candidate saves turn 1 on SIGTERM, restarts
# and must give the same answer with the same prompt_n (only the suffix processed). CPU only
# (-dev none -ngl 0, no VRAM), plus the GPU path (-ngl 99, arbiter env, warm and cold) if VRAM allows.
DC_ARGS=(-m "$SMALL" -c 4096 -dev none -ngl 0 -fa 1 -t 4 --parallel 1 --no-warmup --reasoning off --no-mmproj)
DC_GPU=(-dev Vulkan0 -ngl 99)
DC_ENV=(LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$ARENA" LLAMA_SLEEP_EVICT_KV=1)
DC_PID=
dc_chat() { # PORT OUT MESSAGES-JSON -> HTTP code
curl -sS -m 300 -o "$2" -w '%{http_code}' -H 'Content-Type: application/json' \
-d "{\"messages\":$3,\"max_tokens\":32,$GREEDY}" "http://127.0.0.1:$1/v1/chat/completions" 2>>"$S/curl.err" || true
}
dc_msgs() { # USER1 [ANSWER1 USER2] -> messages JSON
if [ $# -eq 1 ]; then echo "[{\"role\":\"user\",\"content\":\"$1\"}]"
else echo "[{\"role\":\"user\",\"content\":\"$1\"},{\"role\":\"assistant\",\"content\":\"$2\"},{\"role\":\"user\",\"content\":\"$3\"}]"; fi
}
# dc_start NAME DIR ENV... -- ARGS... : DIR "" = no disk cache; sets DC_PID, returns 1 if not healthy
dc_start() {
local name=$1 dir=$2; shift 2
local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift
start_server "$name" "$BUILD" "$PC" "${envs[@]}" -- "${DC_ARGS[@]}" ${dir:+--cache-disk-path "$dir"} "$@"
DC_PID=$SPID
wait_health "$PC" "$DC_PID" 180
}
# dc_term : SIGTERM DC_PID; sets RC, DC_EXIT_MS and DC_FREE_MS (token.lock free, -1 if never held)
dc_term() {
local t0; t0=$(now_ms); DC_FREE_MS=-1; DC_EXIT_MS=-1
local held=0; [ -n "$(token_holder)" ] && held=1
kill -TERM "$DC_PID"
while [ $(( $(now_ms) - t0 )) -lt 30000 ]; do
[ $held = 1 ] && [ $DC_FREE_MS -lt 0 ] && [ -z "$(token_holder)" ] && DC_FREE_MS=$(( $(now_ms) - t0 ))
pid_alive "$DC_PID" || { DC_EXIT_MS=$(( $(now_ms) - t0 )); break; }
sleep 0.01
done
pid_alive "$DC_PID" && kill -KILL "$DC_PID"
wait "$DC_PID" 2>/dev/null; RC=$?
}
# dc_ref TAG ARGS... : turn 1 and turn 2 on one server that never restarts -> DC_C1, DC_REF2, DC_PN_REF, DC_N2, DC_MS_FULL
dc_ref() {
local tag=$1; shift
dc_start "$tag" "" -- "$@" || { fail "diskcache/$tag" "reference server did not start, see $S/$tag.log"; return 1; }
local code; code=$(dc_chat "$PC" "$S/$tag-1.json" "$(dc_msgs "$DC_U1")")
DC_C1=$(json_str content "$S/$tag-1.json")
[ "$code" = 200 ] && [ -n "$DC_C1" ] || { fail "diskcache/$tag" "turn 1: HTTP $code"; stop_server "$DC_PID"; return 1; }
DC_M2=$(dc_msgs "$DC_U1" "$DC_C1" "$DC_U2")
code=$(dc_chat "$PC" "$S/$tag-2.json" "$DC_M2")
DC_REF2=$(json_str content "$S/$tag-2.json"); DC_PN_REF=$(json_num prompt_n "$S/$tag-2.json"); DC_N2=$(json_num prompt_tokens "$S/$tag-2.json")
stop_server "$DC_PID"
# the same turn 2 on a fresh server: the full prefill the cache saves
dc_start "$tag-full" "" -- "$@" && dc_chat "$PC" "$S/$tag-full.json" "$DC_M2" > /dev/null
DC_MS_FULL=$(sed -n 's/.*"prompt_ms":\([0-9.]*\).*/\1/p' "$S/$tag-full.json" | head -1)
stop_server "$DC_PID"
[ "$code" = 200 ] && [ -n "$DC_REF2" ] && [ "${DC_PN_REF:-0}" -gt 0 ] || { fail "diskcache/$tag" "turn 2: HTTP $code"; return 1; }
log "diskcache: reference $tag: turn 2 prompt_n $DC_PN_REF of $DC_N2 tokens, a full prefill of turn 2 took ${DC_MS_FULL} ms"
}
# dc_restore TAG DIR COLD ENV... -- ARGS... : turn 1, SIGTERM (COLD=1: forced cold by a fake waiter first),
# restart, turn 2 -> restored from disk, prompt_n == reference, output == reference
dc_restore() {
local tag=$1 dir=$2 cold=$3; shift 3
local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift
rm -rf "$dir"
dc_start "$tag-a" "$dir" "${envs[@]}" -- "$@" || { fail "diskcache/$tag" "server did not start, see $S/$tag-a.log"; return 1; }
local code c1 w=; code=$(dc_chat "$PC" "$S/$tag-a.json" "$(dc_msgs "$DC_U1")"); c1=$(json_str content "$S/$tag-a.json")
if [ "$cold" = 1 ]; then
( exec 9<"$ARENA/want.lock"; flock -s 9; exec sleep 120 ) & w=$!; track "$w"
wait_holder "" 10000 > /dev/null || log "warning: $tag-a did not go cold"
fi
local was_cold=no; [ "$cold" = 1 ] && [ -z "$(token_holder)" ] && grep -q 'server entering sleeping state' "$S/$tag-a.log" && was_cold=yes
dc_term
[ -n "$w" ] && { kill "$w" 2>/dev/null; wait "$w" 2>/dev/null; }
local rc1=$RC exit1=$DC_EXIT_MS free1=$DC_FREE_MS nf
nf=$(find "$dir" -name '*.lpcd' 2>/dev/null | wc -l)
local f1; f1=$(find "$dir" -name '*.lpcd' 2>/dev/null | head -1); [ -n "$f1" ] && cp "$f1" "$S/$tag-turn1.lpcd"
dc_start "$tag-b" "$dir" "${envs[@]}" -- "$@" || { fail "diskcache/$tag" "restart failed, see $S/$tag-b.log"; return 1; }
code=$(dc_chat "$PC" "$S/$tag-b.json" "$DC_M2")
local c2 pn ms nres; c2=$(json_str content "$S/$tag-b.json"); pn=$(json_num prompt_n "$S/$tag-b.json")
ms=$(sed -n 's/.*"prompt_ms":\([0-9.]*\).*/\1/p' "$S/$tag-b.json" | head -1)
nres=$(grep -c 'disk cache: restored' "$S/$tag-b.log")
local ok=1
[ "$code" = 200 ] && [ "$rc1" -eq 0 ] && [ "$nf" -eq 1 ] && [ "$nres" -eq 1 ] && [ "$c1" = "$DC_C1" ] && \
[ "${pn:-0}" = "$DC_PN_REF" ] && [ "${pn:-0}" -lt "${DC_N2:-0}" ] && [ "$c2" = "$DC_REF2" ] && ok=0
[ "$cold" = 1 ] && [ $was_cold != yes ] && ok=1
check "diskcache/$tag" $ok \
"$([ "$cold" = 1 ] && echo "cold before SIGTERM: $was_cold, ")exit $rc1 in ${exit1} ms$([ "$free1" -ge 0 ] && echo ", token free in ${free1} ms"), $nf file(s); restart: $nres restore(s), turn 2 prompt_n $pn (reference $DC_PN_REF) of $DC_N2 tokens in ${ms} ms (full prefill ${DC_MS_FULL} ms), output $([ "$c2" = "$DC_REF2" ] && echo == || echo '!=') reference"
log "diskcache: $tag: $(grep -h 'disk cache: \(copied\|saved\)' "$S/$tag-a.log" | sed 's/.*disk cache: //' | tr '\n' ';') $(grep -h 'disk cache: restored' "$S/$tag-b.log" | sed 's/.*disk cache: //')"
stop_server "$DC_PID"
}
run_diskcache() {
[ -f "$SMALL" ] || { fail diskcache "small model missing: $SMALL"; return; }
DC_U1="$(moex_prompt 12)"
DC_U2="Which town ships the fewest crates of salt each week?"
dc_ref dc-ref || return
# (1) save on SIGTERM, restore after a restart: plain, arbiter env warm, arbiter env cold
dc_restore restore "$S/dc" 0 -- || return
[ -f "$S/restore-turn1.lpcd" ] || { fail diskcache "no file saved on SIGTERM, see $S/restore-a.log"; return; }
dc_restore restore-arbiter "$S/dc-arb" 0 "${DC_ENV[@]}" --
dc_restore restore-cold "$S/dc-cold" 1 "${DC_ENV[@]}" --
# shutdown cost: the same warm arbiter shutdown without the cache
rm -rf "$S/dc-none"
dc_start dc-none "" "${DC_ENV[@]}" -- && dc_chat "$PC" "$S/dc-none.json" "$(dc_msgs "$DC_U1")" > /dev/null
dc_term
log "diskcache: warm arbiter shutdown without the cache: exit $RC in ${DC_EXIT_MS} ms, token free in ${DC_FREE_MS} ms"
# (2) no restore for a prompt that does not start with a saved one, or under another key
local code pn n nres
dc_start dc-np "$S/dc" -- || { fail diskcache/start "see $S/dc-np.log"; return; }
code=$(dc_chat "$PC" "$S/dc-np.json" "$(dc_msgs "Variant B. $DC_U1")")
pn=$(json_num prompt_n "$S/dc-np.json"); n=$(json_num prompt_tokens "$S/dc-np.json"); nres=$(grep -c 'disk cache: restored' "$S/dc-np.log")
check diskcache/no-prefix $(( code == 200 && nres == 0 && pn == n ? 0 : 1 )) "other conversation: HTTP $code, $nres restore(s), prompt_n $pn of $n"
stop_server "$DC_PID"
local tag args
ln -sf "$SMALL" "$S/other-path.gguf"
for tag in key-ctx key-ctk key-model; do
case $tag in
key-ctx) args=(-c 8192) ;;
key-ctk) args=(-ctk q8_0) ;;
key-model) args=(-m "$S/other-path.gguf") ;;
esac
rm -rf "$S/dc-$tag"; mkdir -p "$S/dc-$tag"; cp "$S/restore-turn1.lpcd" "$S/dc-$tag/"
dc_start "dc-$tag" "$S/dc-$tag" -- "${args[@]}" || { fail "diskcache/$tag" "see $S/dc-$tag.log"; continue; }
code=$(dc_chat "$PC" "$S/dc-$tag.json" "$DC_M2")
pn=$(json_num prompt_n "$S/dc-$tag.json"); n=$(json_num prompt_tokens "$S/dc-$tag.json"); nres=$(grep -c 'disk cache: restored' "$S/dc-$tag.log")
check "diskcache/$tag" $(( code == 200 && nres == 0 && pn == n ? 0 : 1 )) \
"${args[*]}: HTTP $code, $nres restore(s), prompt_n $pn of $n; index: $(grep -h 'disk cache: .*prompts of this model' "$S/dc-$tag.log" | sed 's/.*, \([0-9]* prompts\)/\1/')"
stop_server "$DC_PID"
done
# (3) corrupt and truncated files are removed, never used
local size; size=$(stat -c %s "$S/restore-turn1.lpcd")
rm -rf "$S/dc-bad"; mkdir -p "$S/dc-bad"
head -c $((size / 2)) "$S/restore-turn1.lpcd" > "$S/dc-bad/trunc.lpcd"
cp "$S/restore-turn1.lpcd" "$S/dc-bad/flip.lpcd"
printf '\377' | dd of="$S/dc-bad/flip.lpcd" bs=1 seek=$((size / 2)) conv=notrunc 2>/dev/null
dc_start dc-bad "$S/dc-bad" -- || { fail diskcache/corrupt "see $S/dc-bad.log"; return; }
local trunc_gone=no; [ -e "$S/dc-bad/trunc.lpcd" ] || trunc_gone=yes
code=$(dc_chat "$PC" "$S/dc-bad.json" "$DC_M2")
pn=$(json_num prompt_n "$S/dc-bad.json"); nres=$(grep -c 'disk cache: restored' "$S/dc-bad.log")
local flip_gone=no; [ -e "$S/dc-bad/flip.lpcd" ] || flip_gone=yes
check diskcache/corrupt $([ $trunc_gone = yes ] && [ $flip_gone = yes ] && [ "$code" = 200 ] && [ "$nres" -eq 0 ] && [ "$pn" = "$DC_N2" ] && [ "$(json_str content "$S/dc-bad.json")" = "$DC_REF2" ] && echo 0 || echo 1) \
"truncated file removed at startup: $trunc_gone, byte-flipped file removed at load: $flip_gone, $nres restore(s), prompt_n $pn of $DC_N2, output $([ "$(json_str content "$S/dc-bad.json")" = "$DC_REF2" ] && echo == || echo '!=') reference"
stop_server "$DC_PID"
# (3') LRU size limit: room for one file only, the second save evicts the first
local cap=$(( size * 3 / 2 / 1048576 + 1 )) files total
rm -rf "$S/dc-lru"
dc_start dc-lru1 "$S/dc-lru" -- --cache-disk-size "$cap" && dc_chat "$PC" "$S/dc-lru1.json" "$(dc_msgs "$DC_U1")" > /dev/null
stop_server "$DC_PID"
dc_start dc-lru2 "$S/dc-lru" -- --cache-disk-size "$cap" && dc_chat "$PC" "$S/dc-lru2.json" "$(dc_msgs "Variant B. $DC_U1")" > /dev/null
stop_server "$DC_PID"
files=$(find "$S/dc-lru" -name '*.lpcd' | wc -l); total=$(du -cb "$S"/dc-lru/*.lpcd 2>/dev/null | tail -1 | cut -f1)
check diskcache/lru $(( files == 1 && ${total:-0} <= cap * 1048576 && $(grep -c 'size limit, removing' "$S/dc-lru2.log") == 1 ? 0 : 1 )) \
"limit $cap MiB (file ~$((size / 1048576)) MiB): $files file(s), $((${total:-0} / 1048576)) MiB after two saves, $(grep -c 'size limit, removing' "$S/dc-lru2.log") eviction(s)"
# (3'') write error (RLIMIT_FSIZE 1 MiB, SIGXFSZ ignored -> EFBIG): logged, exit 0, nothing left, next start serves
rm -rf "$S/dc-full"
SRV_WRAP=(bash -c 'trap "" XFSZ; ulimit -f 1024; exec "$@"' _)
dc_start dc-full "$S/dc-full" -- && dc_chat "$PC" "$S/dc-full.json" "$(dc_msgs "$DC_U1")" > /dev/null
SRV_WRAP=()
dc_term
local left; left=$(find "$S/dc-full" -type f 2>/dev/null | wc -l)
dc_start dc-full2 "$S/dc-full" -- || { fail diskcache/write-error "restart failed, see $S/dc-full2.log"; return; }
code=$(dc_chat "$PC" "$S/dc-full2.json" "$DC_M2")
check diskcache/write-error $([ "$RC" -eq 0 ] && [ "$left" -eq 0 ] && grep -q 'disk cache: failed to write' "$S/dc-full.log" && [ "$code" = 200 ] && echo 0 || echo 1) \
"exit $RC in ${DC_EXIT_MS} ms, '$(grep -h 'disk cache: failed to write' "$S/dc-full.log" | sed 's/.*: \([^:]*\), not saved/\1/' | head -1)', $left file(s) left, next start HTTP $code"
stop_server "$DC_PID"
# (1') GPU: the state is saved from VRAM (warm) and from a host shadow (cold), restored into VRAM
local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 ))
if [ $free_mb -lt 1536 ]; then
skip diskcache/gpu "only $free_mb MiB VRAM free (< 1536)"
elif dc_ref dc-ref-gpu "${DC_GPU[@]}"; then
dc_restore gpu-warm "$S/dc-gpu" 0 "${DC_ENV[@]}" -- "${DC_GPU[@]}"
dc_restore gpu-cold "$S/dc-gpu" 1 "${DC_ENV[@]}" -- "${DC_GPU[@]}"
dc_start dc-none-gpu "" "${DC_ENV[@]}" -- "${DC_GPU[@]}" && dc_chat "$PC" "$S/dc-none-gpu.json" "$(dc_msgs "$DC_U1")" > /dev/null
dc_term
log "diskcache: warm GPU arbiter shutdown without the cache: exit $RC in ${DC_EXIT_MS} ms, token free in ${DC_FREE_MS} ms"
fi
local bad_lines; bad_lines=$(scan_logs "$S"/dc-*.log "$S"/restore*.log "$S"/gpu-*.log)
check diskcache/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}"
}
# ================================================================ main
if [ -n "$PERF_REPORT" ]; then
[ -r "$PERF_REPORT" ] || { echo "cannot read $PERF_REPORT" >&2; exit 2; }
@@ -950,4 +1146,5 @@ fi
[ $DO_PERF -eq 1 ] && run_perf
[ $DO_MOEC -eq 1 ] && run_moecache
[ $DO_MOEX -eq 1 ] && run_moe_exact
[ $DO_DC -eq 1 ] && run_diskcache
exit 0