diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh index c1f4f5324..8e7b3d5c8 100644 --- a/scripts/fork-validate.sh +++ b/scripts/fork-validate.sh @@ -51,6 +51,13 @@ # (=0); =log proves the skip engaged. Without the knob skip-engaged is SKIPped with the # batch-size argument. Needs the GPU to itself like --perf (same guard); about 3 model # loads, so run it in the same swapdown window as --perf. +# --diskcache the disk prompt cache (--cache-disk-path) on the small hybrid model, CPU only (no VRAM; +# runs beside production), plus the GPU path when >= 1.5 GiB VRAM is free +# -> a slot state saved on SIGTERM (plain, arbiter warm, arbiter cold) is restored after a +# restart: only the suffix is processed (prompt_n == a server that never restarted) and +# the greedy answer is identical; no restore for another conversation or another model +# path / n_ctx / KV type; truncated and corrupt files are removed, never used; the size +# limit evicts the oldest file; a write error (RLIMIT_FSIZE) is logged and exit stays 0. # # Options: # --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build) @@ -89,7 +96,7 @@ MOEX_ARGS="-ngl 99 --n-cpu-moe 99 -c 4096 -b 2048 -ub 2048 -t 6" BUILD= ; REF= ; CONCAT_BUILD= PORT_BASE=8091 KEEP=0 -DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 DO_MOEX=0 +DO_OPS=0 DO_CONCAT=0 DO_ARB=0 DO_PERF=0 DO_MOEC=0 DO_MOEX=0 DO_DC=0 PERF_REPORT= usage() { awk 'NR > 1 && /^#/ {sub(/^# ?/, ""); print; next} NR > 1 {exit}' "$0"; exit 2; } @@ -101,6 +108,7 @@ while [ $# -gt 0 ]; do --ops) DO_OPS=1; shift ;; --concat) DO_CONCAT=1; shift ;; --arbiter) DO_ARB=1; shift ;; + --diskcache) DO_DC=1; shift ;; --perf) DO_PERF=1; shift ;; --moecache) DO_MOEC=1; MOECACHE_MODEL=$2; shift 2 ;; --moecache-args) MOECACHE_ARGS=$2; shift 2 ;; @@ -119,7 +127,7 @@ while [ $# -gt 0 ]; do esac done [ -n "$PERF_REPORT" ] || { [ -n "$BUILD" ] && [ -n "$REF" ]; } || usage -if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC + DO_MOEX)) -eq 0 ]; then +if [ $((DO_OPS + DO_CONCAT + DO_ARB + DO_PERF + DO_MOEC + DO_MOEX + DO_DC)) -eq 0 ]; then DO_OPS=1 DO_CONCAT=1 DO_ARB=1 fi [ -n "$MMPROJ" ] || MMPROJ=$(dirname "$SMALL")/mmproj-F16.gguf @@ -215,12 +223,14 @@ log "scratch dir $S, ports $PA/$PB/$PC, VRAM used $(vram_mb) MiB" # ---------------------------------------------------------------- helpers: servers # start_server NAME BUILD PORT ENV-ASSIGNMENTS... -- SERVER-ARGS... (sets SPID, log at $S/NAME.log) +# SRV_WRAP: optional command prefix that execs its arguments (the PID stays the server's) +SRV_WRAP=() start_server() { local name=$1 build=$2 port=$3; shift 3 local envs=() while [ $# -gt 0 ] && [ "$1" != -- ]; do envs+=("$1"); shift; done shift - env LD_LIBRARY_PATH="$build/bin" "${envs[@]}" "$build/bin/llama-server" \ + "${SRV_WRAP[@]}" env LD_LIBRARY_PATH="$build/bin" "${envs[@]}" "$build/bin/llama-server" \ --host 127.0.0.1 --port "$port" "$@" >> "$S/$name.log" 2>&1 & SPID=$! track "$SPID" @@ -939,6 +949,192 @@ run_moe_exact() { check moe-exact/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}" } +# ================================================================ 7. disk prompt cache +# --cache-disk-path: on shutdown each slot's state goes to a file; after a restart a prompt that starts +# with a saved prompt (or with one of its checkpoints, for a hybrid model) is restored from the file. +# Method: chat requests on the small hybrid model, greedy. A reference server that never restarts +# gives turn 2 (turn 1 + answer + a new question); the candidate saves turn 1 on SIGTERM, restarts +# and must give the same answer with the same prompt_n (only the suffix processed). CPU only +# (-dev none -ngl 0, no VRAM), plus the GPU path (-ngl 99, arbiter env, warm and cold) if VRAM allows. +DC_ARGS=(-m "$SMALL" -c 4096 -dev none -ngl 0 -fa 1 -t 4 --parallel 1 --no-warmup --reasoning off --no-mmproj) +DC_GPU=(-dev Vulkan0 -ngl 99) +DC_ENV=(LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$ARENA" LLAMA_SLEEP_EVICT_KV=1) +DC_PID= +dc_chat() { # PORT OUT MESSAGES-JSON -> HTTP code + curl -sS -m 300 -o "$2" -w '%{http_code}' -H 'Content-Type: application/json' \ + -d "{\"messages\":$3,\"max_tokens\":32,$GREEDY}" "http://127.0.0.1:$1/v1/chat/completions" 2>>"$S/curl.err" || true +} +dc_msgs() { # USER1 [ANSWER1 USER2] -> messages JSON + if [ $# -eq 1 ]; then echo "[{\"role\":\"user\",\"content\":\"$1\"}]" + else echo "[{\"role\":\"user\",\"content\":\"$1\"},{\"role\":\"assistant\",\"content\":\"$2\"},{\"role\":\"user\",\"content\":\"$3\"}]"; fi +} +# dc_start NAME DIR ENV... -- ARGS... : DIR "" = no disk cache; sets DC_PID, returns 1 if not healthy +dc_start() { + local name=$1 dir=$2; shift 2 + local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift + start_server "$name" "$BUILD" "$PC" "${envs[@]}" -- "${DC_ARGS[@]}" ${dir:+--cache-disk-path "$dir"} "$@" + DC_PID=$SPID + wait_health "$PC" "$DC_PID" 180 +} +# dc_term : SIGTERM DC_PID; sets RC, DC_EXIT_MS and DC_FREE_MS (token.lock free, -1 if never held) +dc_term() { + local t0; t0=$(now_ms); DC_FREE_MS=-1; DC_EXIT_MS=-1 + local held=0; [ -n "$(token_holder)" ] && held=1 + kill -TERM "$DC_PID" + while [ $(( $(now_ms) - t0 )) -lt 30000 ]; do + [ $held = 1 ] && [ $DC_FREE_MS -lt 0 ] && [ -z "$(token_holder)" ] && DC_FREE_MS=$(( $(now_ms) - t0 )) + pid_alive "$DC_PID" || { DC_EXIT_MS=$(( $(now_ms) - t0 )); break; } + sleep 0.01 + done + pid_alive "$DC_PID" && kill -KILL "$DC_PID" + wait "$DC_PID" 2>/dev/null; RC=$? +} +# dc_ref TAG ARGS... : turn 1 and turn 2 on one server that never restarts -> DC_C1, DC_REF2, DC_PN_REF, DC_N2, DC_MS_FULL +dc_ref() { + local tag=$1; shift + dc_start "$tag" "" -- "$@" || { fail "diskcache/$tag" "reference server did not start, see $S/$tag.log"; return 1; } + local code; code=$(dc_chat "$PC" "$S/$tag-1.json" "$(dc_msgs "$DC_U1")") + DC_C1=$(json_str content "$S/$tag-1.json") + [ "$code" = 200 ] && [ -n "$DC_C1" ] || { fail "diskcache/$tag" "turn 1: HTTP $code"; stop_server "$DC_PID"; return 1; } + DC_M2=$(dc_msgs "$DC_U1" "$DC_C1" "$DC_U2") + code=$(dc_chat "$PC" "$S/$tag-2.json" "$DC_M2") + DC_REF2=$(json_str content "$S/$tag-2.json"); DC_PN_REF=$(json_num prompt_n "$S/$tag-2.json"); DC_N2=$(json_num prompt_tokens "$S/$tag-2.json") + stop_server "$DC_PID" + # the same turn 2 on a fresh server: the full prefill the cache saves + dc_start "$tag-full" "" -- "$@" && dc_chat "$PC" "$S/$tag-full.json" "$DC_M2" > /dev/null + DC_MS_FULL=$(sed -n 's/.*"prompt_ms":\([0-9.]*\).*/\1/p' "$S/$tag-full.json" | head -1) + stop_server "$DC_PID" + [ "$code" = 200 ] && [ -n "$DC_REF2" ] && [ "${DC_PN_REF:-0}" -gt 0 ] || { fail "diskcache/$tag" "turn 2: HTTP $code"; return 1; } + log "diskcache: reference $tag: turn 2 prompt_n $DC_PN_REF of $DC_N2 tokens, a full prefill of turn 2 took ${DC_MS_FULL} ms" +} +# dc_restore TAG DIR COLD ENV... -- ARGS... : turn 1, SIGTERM (COLD=1: forced cold by a fake waiter first), +# restart, turn 2 -> restored from disk, prompt_n == reference, output == reference +dc_restore() { + local tag=$1 dir=$2 cold=$3; shift 3 + local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift + rm -rf "$dir" + dc_start "$tag-a" "$dir" "${envs[@]}" -- "$@" || { fail "diskcache/$tag" "server did not start, see $S/$tag-a.log"; return 1; } + local code c1 w=; code=$(dc_chat "$PC" "$S/$tag-a.json" "$(dc_msgs "$DC_U1")"); c1=$(json_str content "$S/$tag-a.json") + if [ "$cold" = 1 ]; then + ( exec 9<"$ARENA/want.lock"; flock -s 9; exec sleep 120 ) & w=$!; track "$w" + wait_holder "" 10000 > /dev/null || log "warning: $tag-a did not go cold" + fi + local was_cold=no; [ "$cold" = 1 ] && [ -z "$(token_holder)" ] && grep -q 'server entering sleeping state' "$S/$tag-a.log" && was_cold=yes + dc_term + [ -n "$w" ] && { kill "$w" 2>/dev/null; wait "$w" 2>/dev/null; } + local rc1=$RC exit1=$DC_EXIT_MS free1=$DC_FREE_MS nf + nf=$(find "$dir" -name '*.lpcd' 2>/dev/null | wc -l) + local f1; f1=$(find "$dir" -name '*.lpcd' 2>/dev/null | head -1); [ -n "$f1" ] && cp "$f1" "$S/$tag-turn1.lpcd" + dc_start "$tag-b" "$dir" "${envs[@]}" -- "$@" || { fail "diskcache/$tag" "restart failed, see $S/$tag-b.log"; return 1; } + code=$(dc_chat "$PC" "$S/$tag-b.json" "$DC_M2") + local c2 pn ms nres; c2=$(json_str content "$S/$tag-b.json"); pn=$(json_num prompt_n "$S/$tag-b.json") + ms=$(sed -n 's/.*"prompt_ms":\([0-9.]*\).*/\1/p' "$S/$tag-b.json" | head -1) + nres=$(grep -c 'disk cache: restored' "$S/$tag-b.log") + local ok=1 + [ "$code" = 200 ] && [ "$rc1" -eq 0 ] && [ "$nf" -eq 1 ] && [ "$nres" -eq 1 ] && [ "$c1" = "$DC_C1" ] && \ + [ "${pn:-0}" = "$DC_PN_REF" ] && [ "${pn:-0}" -lt "${DC_N2:-0}" ] && [ "$c2" = "$DC_REF2" ] && ok=0 + [ "$cold" = 1 ] && [ $was_cold != yes ] && ok=1 + check "diskcache/$tag" $ok \ + "$([ "$cold" = 1 ] && echo "cold before SIGTERM: $was_cold, ")exit $rc1 in ${exit1} ms$([ "$free1" -ge 0 ] && echo ", token free in ${free1} ms"), $nf file(s); restart: $nres restore(s), turn 2 prompt_n $pn (reference $DC_PN_REF) of $DC_N2 tokens in ${ms} ms (full prefill ${DC_MS_FULL} ms), output $([ "$c2" = "$DC_REF2" ] && echo == || echo '!=') reference" + log "diskcache: $tag: $(grep -h 'disk cache: \(copied\|saved\)' "$S/$tag-a.log" | sed 's/.*disk cache: //' | tr '\n' ';') $(grep -h 'disk cache: restored' "$S/$tag-b.log" | sed 's/.*disk cache: //')" + stop_server "$DC_PID" +} +run_diskcache() { + [ -f "$SMALL" ] || { fail diskcache "small model missing: $SMALL"; return; } + DC_U1="$(moex_prompt 12)" + DC_U2="Which town ships the fewest crates of salt each week?" + dc_ref dc-ref || return + + # (1) save on SIGTERM, restore after a restart: plain, arbiter env warm, arbiter env cold + dc_restore restore "$S/dc" 0 -- || return + [ -f "$S/restore-turn1.lpcd" ] || { fail diskcache "no file saved on SIGTERM, see $S/restore-a.log"; return; } + dc_restore restore-arbiter "$S/dc-arb" 0 "${DC_ENV[@]}" -- + dc_restore restore-cold "$S/dc-cold" 1 "${DC_ENV[@]}" -- + # shutdown cost: the same warm arbiter shutdown without the cache + rm -rf "$S/dc-none" + dc_start dc-none "" "${DC_ENV[@]}" -- && dc_chat "$PC" "$S/dc-none.json" "$(dc_msgs "$DC_U1")" > /dev/null + dc_term + log "diskcache: warm arbiter shutdown without the cache: exit $RC in ${DC_EXIT_MS} ms, token free in ${DC_FREE_MS} ms" + + # (2) no restore for a prompt that does not start with a saved one, or under another key + local code pn n nres + dc_start dc-np "$S/dc" -- || { fail diskcache/start "see $S/dc-np.log"; return; } + code=$(dc_chat "$PC" "$S/dc-np.json" "$(dc_msgs "Variant B. $DC_U1")") + pn=$(json_num prompt_n "$S/dc-np.json"); n=$(json_num prompt_tokens "$S/dc-np.json"); nres=$(grep -c 'disk cache: restored' "$S/dc-np.log") + check diskcache/no-prefix $(( code == 200 && nres == 0 && pn == n ? 0 : 1 )) "other conversation: HTTP $code, $nres restore(s), prompt_n $pn of $n" + stop_server "$DC_PID" + local tag args + ln -sf "$SMALL" "$S/other-path.gguf" + for tag in key-ctx key-ctk key-model; do + case $tag in + key-ctx) args=(-c 8192) ;; + key-ctk) args=(-ctk q8_0) ;; + key-model) args=(-m "$S/other-path.gguf") ;; + esac + rm -rf "$S/dc-$tag"; mkdir -p "$S/dc-$tag"; cp "$S/restore-turn1.lpcd" "$S/dc-$tag/" + dc_start "dc-$tag" "$S/dc-$tag" -- "${args[@]}" || { fail "diskcache/$tag" "see $S/dc-$tag.log"; continue; } + code=$(dc_chat "$PC" "$S/dc-$tag.json" "$DC_M2") + pn=$(json_num prompt_n "$S/dc-$tag.json"); n=$(json_num prompt_tokens "$S/dc-$tag.json"); nres=$(grep -c 'disk cache: restored' "$S/dc-$tag.log") + check "diskcache/$tag" $(( code == 200 && nres == 0 && pn == n ? 0 : 1 )) \ + "${args[*]}: HTTP $code, $nres restore(s), prompt_n $pn of $n; index: $(grep -h 'disk cache: .*prompts of this model' "$S/dc-$tag.log" | sed 's/.*, \([0-9]* prompts\)/\1/')" + stop_server "$DC_PID" + done + + # (3) corrupt and truncated files are removed, never used + local size; size=$(stat -c %s "$S/restore-turn1.lpcd") + rm -rf "$S/dc-bad"; mkdir -p "$S/dc-bad" + head -c $((size / 2)) "$S/restore-turn1.lpcd" > "$S/dc-bad/trunc.lpcd" + cp "$S/restore-turn1.lpcd" "$S/dc-bad/flip.lpcd" + printf '\377' | dd of="$S/dc-bad/flip.lpcd" bs=1 seek=$((size / 2)) conv=notrunc 2>/dev/null + dc_start dc-bad "$S/dc-bad" -- || { fail diskcache/corrupt "see $S/dc-bad.log"; return; } + local trunc_gone=no; [ -e "$S/dc-bad/trunc.lpcd" ] || trunc_gone=yes + code=$(dc_chat "$PC" "$S/dc-bad.json" "$DC_M2") + pn=$(json_num prompt_n "$S/dc-bad.json"); nres=$(grep -c 'disk cache: restored' "$S/dc-bad.log") + local flip_gone=no; [ -e "$S/dc-bad/flip.lpcd" ] || flip_gone=yes + check diskcache/corrupt $([ $trunc_gone = yes ] && [ $flip_gone = yes ] && [ "$code" = 200 ] && [ "$nres" -eq 0 ] && [ "$pn" = "$DC_N2" ] && [ "$(json_str content "$S/dc-bad.json")" = "$DC_REF2" ] && echo 0 || echo 1) \ + "truncated file removed at startup: $trunc_gone, byte-flipped file removed at load: $flip_gone, $nres restore(s), prompt_n $pn of $DC_N2, output $([ "$(json_str content "$S/dc-bad.json")" = "$DC_REF2" ] && echo == || echo '!=') reference" + stop_server "$DC_PID" + + # (3') LRU size limit: room for one file only, the second save evicts the first + local cap=$(( size * 3 / 2 / 1048576 + 1 )) files total + rm -rf "$S/dc-lru" + dc_start dc-lru1 "$S/dc-lru" -- --cache-disk-size "$cap" && dc_chat "$PC" "$S/dc-lru1.json" "$(dc_msgs "$DC_U1")" > /dev/null + stop_server "$DC_PID" + dc_start dc-lru2 "$S/dc-lru" -- --cache-disk-size "$cap" && dc_chat "$PC" "$S/dc-lru2.json" "$(dc_msgs "Variant B. $DC_U1")" > /dev/null + stop_server "$DC_PID" + files=$(find "$S/dc-lru" -name '*.lpcd' | wc -l); total=$(du -cb "$S"/dc-lru/*.lpcd 2>/dev/null | tail -1 | cut -f1) + check diskcache/lru $(( files == 1 && ${total:-0} <= cap * 1048576 && $(grep -c 'size limit, removing' "$S/dc-lru2.log") == 1 ? 0 : 1 )) \ + "limit $cap MiB (file ~$((size / 1048576)) MiB): $files file(s), $((${total:-0} / 1048576)) MiB after two saves, $(grep -c 'size limit, removing' "$S/dc-lru2.log") eviction(s)" + + # (3'') write error (RLIMIT_FSIZE 1 MiB, SIGXFSZ ignored -> EFBIG): logged, exit 0, nothing left, next start serves + rm -rf "$S/dc-full" + SRV_WRAP=(bash -c 'trap "" XFSZ; ulimit -f 1024; exec "$@"' _) + dc_start dc-full "$S/dc-full" -- && dc_chat "$PC" "$S/dc-full.json" "$(dc_msgs "$DC_U1")" > /dev/null + SRV_WRAP=() + dc_term + local left; left=$(find "$S/dc-full" -type f 2>/dev/null | wc -l) + dc_start dc-full2 "$S/dc-full" -- || { fail diskcache/write-error "restart failed, see $S/dc-full2.log"; return; } + code=$(dc_chat "$PC" "$S/dc-full2.json" "$DC_M2") + check diskcache/write-error $([ "$RC" -eq 0 ] && [ "$left" -eq 0 ] && grep -q 'disk cache: failed to write' "$S/dc-full.log" && [ "$code" = 200 ] && echo 0 || echo 1) \ + "exit $RC in ${DC_EXIT_MS} ms, '$(grep -h 'disk cache: failed to write' "$S/dc-full.log" | sed 's/.*: \([^:]*\), not saved/\1/' | head -1)', $left file(s) left, next start HTTP $code" + stop_server "$DC_PID" + + # (1') GPU: the state is saved from VRAM (warm) and from a host shadow (cold), restored into VRAM + local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 )) + if [ $free_mb -lt 1536 ]; then + skip diskcache/gpu "only $free_mb MiB VRAM free (< 1536)" + elif dc_ref dc-ref-gpu "${DC_GPU[@]}"; then + dc_restore gpu-warm "$S/dc-gpu" 0 "${DC_ENV[@]}" -- "${DC_GPU[@]}" + dc_restore gpu-cold "$S/dc-gpu" 1 "${DC_ENV[@]}" -- "${DC_GPU[@]}" + dc_start dc-none-gpu "" "${DC_ENV[@]}" -- "${DC_GPU[@]}" && dc_chat "$PC" "$S/dc-none-gpu.json" "$(dc_msgs "$DC_U1")" > /dev/null + dc_term + log "diskcache: warm GPU arbiter shutdown without the cache: exit $RC in ${DC_EXIT_MS} ms, token free in ${DC_FREE_MS} ms" + fi + + local bad_lines; bad_lines=$(scan_logs "$S"/dc-*.log "$S"/restore*.log "$S"/gpu-*.log) + check diskcache/log-scan $([ -z "$bad_lines" ] && echo 0 || echo 1) "${bad_lines:-clean}" +} + # ================================================================ main if [ -n "$PERF_REPORT" ]; then [ -r "$PERF_REPORT" ] || { echo "cannot read $PERF_REPORT" >&2; exit 2; } @@ -950,4 +1146,5 @@ fi [ $DO_PERF -eq 1 ] && run_perf [ $DO_MOEC -eq 1 ] && run_moecache [ $DO_MOEX -eq 1 ] && run_moe_exact +[ $DO_DC -eq 1 ] && run_diskcache exit 0