scripts: fork-validate.sh --diskcache - the mmproj cases

Production runs the 35B and gemma with an mmproj, and the first deploy of the
disk prompt cache aborted on that path. Every case had used --no-mmproj. Now
--no-mmproj is in DC_MM, separate from DC_ARGS, and two cases run with the
0.8B's mmproj (--no-mmproj-offload, arbiter env, LLAMA_MMPROJ_ONDEMAND=1),
against their own reference:
- restore-mmproj: turn 1 is saved on SIGTERM and restored after the restart,
  prompt_n equal to the reference, output identical;
- mmproj-image: an image request (no restore, answer "red"), then the text
  chat (restored, identical), then an image again, then SIGTERM with media in
  the slot (no state copied, exit 0, the file count unchanged).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
LumpiastyandClaude Opus 5.5 committed 2026-11-21 19:39:57 +01:00
1 parent 060e46516e
commit 383dbf4e6c
1 file changed
+41 -3
+41 -3
View File
@@ -57,7 +57,9 @@
# restart: only the suffix is processed (prompt_n == a server that never restarted) and
# the greedy answer is identical; no restore for another conversation or another model
# path / n_ctx / KV type; truncated and corrupt files are removed, never used; the size
# limit evicts the oldest file; a write error (RLIMIT_FSIZE) is logged and exit stays 0.
# limit evicts the oldest file; a write error (RLIMIT_FSIZE) is logged and exit stays 0;
# with an mmproj loaded (as production runs) a text chat is restored the same way, and an
# image prompt is never restored or saved and does not crash.
#
# Options:
# --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build)
@@ -956,7 +958,8 @@ run_moe_exact() {
# gives turn 2 (turn 1 + answer + a new question); the candidate saves turn 1 on SIGTERM, restarts
# and must give the same answer with the same prompt_n (only the suffix processed). CPU only
# (-dev none -ngl 0, no VRAM), plus the GPU path (-ngl 99, arbiter env, warm and cold) if VRAM allows.
DC_ARGS=(-m "$SMALL" -c 4096 -dev none -ngl 0 -fa 1 -t 4 --parallel 1 --no-warmup --reasoning off --no-mmproj)
DC_ARGS=(-m "$SMALL" -c 4096 -dev none -ngl 0 -fa 1 -t 4 --parallel 1 --no-warmup --reasoning off)
DC_MM=(--no-mmproj) # the mmproj case sets --mmproj
DC_GPU=(-dev Vulkan0 -ngl 99)
DC_ENV=(LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$ARENA" LLAMA_SLEEP_EVICT_KV=1)
DC_PID=
@@ -972,7 +975,7 @@ dc_msgs() { # USER1 [ANSWER1 USER2] -> messages JSON
dc_start() {
local name=$1 dir=$2; shift 2
local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift
start_server "$name" "$BUILD" "$PC" "${envs[@]}" -- "${DC_ARGS[@]}" ${dir:+--cache-disk-path "$dir"} "$@"
start_server "$name" "$BUILD" "$PC" "${envs[@]}" -- "${DC_ARGS[@]}" "${DC_MM[@]}" ${dir:+--cache-disk-path "$dir"} "$@"
DC_PID=$SPID
wait_health "$PC" "$DC_PID" 180
}
@@ -1119,6 +1122,41 @@ run_diskcache() {
"exit $RC in ${DC_EXIT_MS} ms, '$(grep -h 'disk cache: failed to write' "$S/dc-full.log" | sed 's/.*: \([^:]*\), not saved/\1/' | head -1)', $left file(s) left, next start HTTP $code"
stop_server "$DC_PID"
# (4) with an mmproj loaded, as the production 35B and gemma-26B run: a text-only chat is saved and
# restored (server_tokens::get_tokens() asserts in that mode); a prompt with an image is never
# restored or saved and does not crash. Own reference: the CPU references above stay as they are.
if [ ! -f "$MMPROJ" ]; then
skip diskcache/mmproj "mmproj missing: $MMPROJ"
else
DC_MM=(--mmproj "$MMPROJ" --no-mmproj-offload)
local DC_ENV_MM=("${DC_ENV[@]}" LLAMA_MMPROJ_ONDEMAND=1)
if dc_ref dc-ref-mm; then
dc_restore restore-mmproj "$S/dc-mm" 0 "${DC_ENV_MM[@]}" --
local img ans1 ans2 c2 nres1 ncopy
img=$(make_red_ppm)
printf '{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/x-portable-pixmap;base64,%s"}},{"type":"text","text":"What is the main color of this image? Answer with one word."}]}],"max_tokens":16,%s}' \
"$img" "$GREEDY" > "$S/dc-img-body.json"
rm -rf "$S/dc-img"; mkdir -p "$S/dc-img"; cp "$S/restore-mmproj-turn1.lpcd" "$S/dc-img/" 2>/dev/null
if dc_start dc-img "$S/dc-img" "${DC_ENV_MM[@]}" --; then
# image first (no restore), then the text chat (restored), then an image again: the slot holds media at the SIGTERM
code=$(curl -sS -m 300 -o "$S/dc-img1.json" -w '%{http_code}' -H 'Content-Type: application/json' -d @"$S/dc-img-body.json" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err")
ans1=$(json_str content "$S/dc-img1.json"); nres1=$(grep -c 'disk cache: restored' "$S/dc-img.log")
dc_chat "$PC" "$S/dc-img2.json" "$DC_M2" > /dev/null; c2=$(json_str content "$S/dc-img2.json")
local code3; code3=$(curl -sS -m 300 -o "$S/dc-img3.json" -w '%{http_code}' -H 'Content-Type: application/json' -d @"$S/dc-img-body.json" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err")
ans2=$(json_str content "$S/dc-img3.json")
dc_term
ncopy=$(grep -c 'disk cache: copied' "$S/dc-img.log")
local nf; nf=$(find "$S/dc-img" -name '*.lpcd' | wc -l)
check diskcache/mmproj-image $([ "$code" = 200 ] && [ "$code3" = 200 ] && [ "$nres1" -eq 0 ] && [ "$(grep -c 'disk cache: restored' "$S/dc-img.log")" -eq 1 ] && \
[ "$c2" = "$DC_REF2" ] && [ "$RC" -eq 0 ] && [ "$ncopy" -eq 0 ] && [ "$nf" -eq 1 ] && echo "$ans1$ans2" | grep -qi red && echo 0 || echo 1) \
"image: HTTP $code '$ans1', $nres1 restore(s); text chat after it: $(grep -c 'disk cache: restored' "$S/dc-img.log") restore(s), output $([ "$c2" = "$DC_REF2" ] && echo == || echo '!=') reference; image again: HTTP $code3 '$ans2'; SIGTERM with media in the slot: exit $RC, $ncopy state(s) copied, $nf file(s)"
else
fail diskcache/mmproj-image "server did not start, see $S/dc-img.log"
fi
fi
DC_MM=(--no-mmproj)
fi
# (1') GPU: the state is saved from VRAM (warm) and from a host shadow (cold), restored into VRAM
local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 ))
if [ $free_mb -lt 1536 ]; then