From 383dbf4e6c19a9033eaa67a3dfad0d765389a8df Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 25 Oct 2026 21:38:58 +0100 Subject: [PATCH] scripts: fork-validate.sh --diskcache - the mmproj cases Production runs the 35B and gemma with an mmproj, and the first deploy of the disk prompt cache aborted on that path. Every case had used --no-mmproj. Now --no-mmproj is in DC_MM, separate from DC_ARGS, and two cases run with the 0.8B's mmproj (--no-mmproj-offload, arbiter env, LLAMA_MMPROJ_ONDEMAND=1), against their own reference: - restore-mmproj: turn 1 is saved on SIGTERM and restored after the restart, prompt_n equal to the reference, output identical; - mmproj-image: an image request (no restore, answer "red"), then the text chat (restored, identical), then an image again, then SIGTERM with media in the slot (no state copied, exit 0, the file count unchanged). Co-Authored-By: Claude Opus 5.5 --- scripts/fork-validate.sh | 44 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh index 8e7b3d5c8..becd5d499 100644 --- a/scripts/fork-validate.sh +++ b/scripts/fork-validate.sh @@ -57,7 +57,9 @@ # restart: only the suffix is processed (prompt_n == a server that never restarted) and # the greedy answer is identical; no restore for another conversation or another model # path / n_ctx / KV type; truncated and corrupt files are removed, never used; the size -# limit evicts the oldest file; a write error (RLIMIT_FSIZE) is logged and exit stays 0. +# limit evicts the oldest file; a write error (RLIMIT_FSIZE) is logged and exit stays 0; +# with an mmproj loaded (as production runs) a text chat is restored the same way, and an +# image prompt is never restored or saved and does not crash. # # Options: # --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build) @@ -956,7 +958,8 @@ run_moe_exact() { # gives turn 2 (turn 1 + answer + a new question); the candidate saves turn 1 on SIGTERM, restarts # and must give the same answer with the same prompt_n (only the suffix processed). CPU only # (-dev none -ngl 0, no VRAM), plus the GPU path (-ngl 99, arbiter env, warm and cold) if VRAM allows. -DC_ARGS=(-m "$SMALL" -c 4096 -dev none -ngl 0 -fa 1 -t 4 --parallel 1 --no-warmup --reasoning off --no-mmproj) +DC_ARGS=(-m "$SMALL" -c 4096 -dev none -ngl 0 -fa 1 -t 4 --parallel 1 --no-warmup --reasoning off) +DC_MM=(--no-mmproj) # the mmproj case sets --mmproj DC_GPU=(-dev Vulkan0 -ngl 99) DC_ENV=(LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$ARENA" LLAMA_SLEEP_EVICT_KV=1) DC_PID= @@ -972,7 +975,7 @@ dc_msgs() { # USER1 [ANSWER1 USER2] -> messages JSON dc_start() { local name=$1 dir=$2; shift 2 local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift - start_server "$name" "$BUILD" "$PC" "${envs[@]}" -- "${DC_ARGS[@]}" ${dir:+--cache-disk-path "$dir"} "$@" + start_server "$name" "$BUILD" "$PC" "${envs[@]}" -- "${DC_ARGS[@]}" "${DC_MM[@]}" ${dir:+--cache-disk-path "$dir"} "$@" DC_PID=$SPID wait_health "$PC" "$DC_PID" 180 } @@ -1119,6 +1122,41 @@ run_diskcache() { "exit $RC in ${DC_EXIT_MS} ms, '$(grep -h 'disk cache: failed to write' "$S/dc-full.log" | sed 's/.*: \([^:]*\), not saved/\1/' | head -1)', $left file(s) left, next start HTTP $code" stop_server "$DC_PID" + # (4) with an mmproj loaded, as the production 35B and gemma-26B run: a text-only chat is saved and + # restored (server_tokens::get_tokens() asserts in that mode); a prompt with an image is never + # restored or saved and does not crash. Own reference: the CPU references above stay as they are. + if [ ! -f "$MMPROJ" ]; then + skip diskcache/mmproj "mmproj missing: $MMPROJ" + else + DC_MM=(--mmproj "$MMPROJ" --no-mmproj-offload) + local DC_ENV_MM=("${DC_ENV[@]}" LLAMA_MMPROJ_ONDEMAND=1) + if dc_ref dc-ref-mm; then + dc_restore restore-mmproj "$S/dc-mm" 0 "${DC_ENV_MM[@]}" -- + local img ans1 ans2 c2 nres1 ncopy + img=$(make_red_ppm) + printf '{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/x-portable-pixmap;base64,%s"}},{"type":"text","text":"What is the main color of this image? Answer with one word."}]}],"max_tokens":16,%s}' \ + "$img" "$GREEDY" > "$S/dc-img-body.json" + rm -rf "$S/dc-img"; mkdir -p "$S/dc-img"; cp "$S/restore-mmproj-turn1.lpcd" "$S/dc-img/" 2>/dev/null + if dc_start dc-img "$S/dc-img" "${DC_ENV_MM[@]}" --; then + # image first (no restore), then the text chat (restored), then an image again: the slot holds media at the SIGTERM + code=$(curl -sS -m 300 -o "$S/dc-img1.json" -w '%{http_code}' -H 'Content-Type: application/json' -d @"$S/dc-img-body.json" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err") + ans1=$(json_str content "$S/dc-img1.json"); nres1=$(grep -c 'disk cache: restored' "$S/dc-img.log") + dc_chat "$PC" "$S/dc-img2.json" "$DC_M2" > /dev/null; c2=$(json_str content "$S/dc-img2.json") + local code3; code3=$(curl -sS -m 300 -o "$S/dc-img3.json" -w '%{http_code}' -H 'Content-Type: application/json' -d @"$S/dc-img-body.json" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err") + ans2=$(json_str content "$S/dc-img3.json") + dc_term + ncopy=$(grep -c 'disk cache: copied' "$S/dc-img.log") + local nf; nf=$(find "$S/dc-img" -name '*.lpcd' | wc -l) + check diskcache/mmproj-image $([ "$code" = 200 ] && [ "$code3" = 200 ] && [ "$nres1" -eq 0 ] && [ "$(grep -c 'disk cache: restored' "$S/dc-img.log")" -eq 1 ] && \ + [ "$c2" = "$DC_REF2" ] && [ "$RC" -eq 0 ] && [ "$ncopy" -eq 0 ] && [ "$nf" -eq 1 ] && echo "$ans1$ans2" | grep -qi red && echo 0 || echo 1) \ + "image: HTTP $code '$ans1', $nres1 restore(s); text chat after it: $(grep -c 'disk cache: restored' "$S/dc-img.log") restore(s), output $([ "$c2" = "$DC_REF2" ] && echo == || echo '!=') reference; image again: HTTP $code3 '$ans2'; SIGTERM with media in the slot: exit $RC, $ncopy state(s) copied, $nf file(s)" + else + fail diskcache/mmproj-image "server did not start, see $S/dc-img.log" + fi + fi + DC_MM=(--no-mmproj) + fi + # (1') GPU: the state is saved from VRAM (warm) and from a host shadow (cold), restored into VRAM local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 )) if [ $free_mb -lt 1536 ]; then