diff --git a/scripts/fork-validate.sh b/scripts/fork-validate.sh index 8e7b3d5c8..becd5d499 100644 --- a/scripts/fork-validate.sh +++ b/scripts/fork-validate.sh @@ -57,7 +57,9 @@ # restart: only the suffix is processed (prompt_n == a server that never restarted) and # the greedy answer is identical; no restore for another conversation or another model # path / n_ctx / KV type; truncated and corrupt files are removed, never used; the size -# limit evicts the oldest file; a write error (RLIMIT_FSIZE) is logged and exit stays 0. +# limit evicts the oldest file; a write error (RLIMIT_FSIZE) is logged and exit stays 0; +# with an mmproj loaded (as production runs) a text chat is restored the same way, and an +# image prompt is never restored or saved and does not crash. # # Options: # --concat-build DIR build with the GGML_VK_CONCAT_FASTPATH knob for --concat (default: --build) @@ -956,7 +958,8 @@ run_moe_exact() { # gives turn 2 (turn 1 + answer + a new question); the candidate saves turn 1 on SIGTERM, restarts # and must give the same answer with the same prompt_n (only the suffix processed). CPU only # (-dev none -ngl 0, no VRAM), plus the GPU path (-ngl 99, arbiter env, warm and cold) if VRAM allows. -DC_ARGS=(-m "$SMALL" -c 4096 -dev none -ngl 0 -fa 1 -t 4 --parallel 1 --no-warmup --reasoning off --no-mmproj) +DC_ARGS=(-m "$SMALL" -c 4096 -dev none -ngl 0 -fa 1 -t 4 --parallel 1 --no-warmup --reasoning off) +DC_MM=(--no-mmproj) # the mmproj case sets --mmproj DC_GPU=(-dev Vulkan0 -ngl 99) DC_ENV=(LLAMA_SLEEP_VRAM_ONLY=1 "LLAMA_VRAM_ARENA=$ARENA" LLAMA_SLEEP_EVICT_KV=1) DC_PID= @@ -972,7 +975,7 @@ dc_msgs() { # USER1 [ANSWER1 USER2] -> messages JSON dc_start() { local name=$1 dir=$2; shift 2 local envs=(); while [ "$1" != -- ]; do envs+=("$1"); shift; done; shift - start_server "$name" "$BUILD" "$PC" "${envs[@]}" -- "${DC_ARGS[@]}" ${dir:+--cache-disk-path "$dir"} "$@" + start_server "$name" "$BUILD" "$PC" "${envs[@]}" -- "${DC_ARGS[@]}" "${DC_MM[@]}" ${dir:+--cache-disk-path "$dir"} "$@" DC_PID=$SPID wait_health "$PC" "$DC_PID" 180 } @@ -1119,6 +1122,41 @@ run_diskcache() { "exit $RC in ${DC_EXIT_MS} ms, '$(grep -h 'disk cache: failed to write' "$S/dc-full.log" | sed 's/.*: \([^:]*\), not saved/\1/' | head -1)', $left file(s) left, next start HTTP $code" stop_server "$DC_PID" + # (4) with an mmproj loaded, as the production 35B and gemma-26B run: a text-only chat is saved and + # restored (server_tokens::get_tokens() asserts in that mode); a prompt with an image is never + # restored or saved and does not crash. Own reference: the CPU references above stay as they are. + if [ ! -f "$MMPROJ" ]; then + skip diskcache/mmproj "mmproj missing: $MMPROJ" + else + DC_MM=(--mmproj "$MMPROJ" --no-mmproj-offload) + local DC_ENV_MM=("${DC_ENV[@]}" LLAMA_MMPROJ_ONDEMAND=1) + if dc_ref dc-ref-mm; then + dc_restore restore-mmproj "$S/dc-mm" 0 "${DC_ENV_MM[@]}" -- + local img ans1 ans2 c2 nres1 ncopy + img=$(make_red_ppm) + printf '{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/x-portable-pixmap;base64,%s"}},{"type":"text","text":"What is the main color of this image? Answer with one word."}]}],"max_tokens":16,%s}' \ + "$img" "$GREEDY" > "$S/dc-img-body.json" + rm -rf "$S/dc-img"; mkdir -p "$S/dc-img"; cp "$S/restore-mmproj-turn1.lpcd" "$S/dc-img/" 2>/dev/null + if dc_start dc-img "$S/dc-img" "${DC_ENV_MM[@]}" --; then + # image first (no restore), then the text chat (restored), then an image again: the slot holds media at the SIGTERM + code=$(curl -sS -m 300 -o "$S/dc-img1.json" -w '%{http_code}' -H 'Content-Type: application/json' -d @"$S/dc-img-body.json" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err") + ans1=$(json_str content "$S/dc-img1.json"); nres1=$(grep -c 'disk cache: restored' "$S/dc-img.log") + dc_chat "$PC" "$S/dc-img2.json" "$DC_M2" > /dev/null; c2=$(json_str content "$S/dc-img2.json") + local code3; code3=$(curl -sS -m 300 -o "$S/dc-img3.json" -w '%{http_code}' -H 'Content-Type: application/json' -d @"$S/dc-img-body.json" "http://127.0.0.1:$PC/v1/chat/completions" 2>>"$S/curl.err") + ans2=$(json_str content "$S/dc-img3.json") + dc_term + ncopy=$(grep -c 'disk cache: copied' "$S/dc-img.log") + local nf; nf=$(find "$S/dc-img" -name '*.lpcd' | wc -l) + check diskcache/mmproj-image $([ "$code" = 200 ] && [ "$code3" = 200 ] && [ "$nres1" -eq 0 ] && [ "$(grep -c 'disk cache: restored' "$S/dc-img.log")" -eq 1 ] && \ + [ "$c2" = "$DC_REF2" ] && [ "$RC" -eq 0 ] && [ "$ncopy" -eq 0 ] && [ "$nf" -eq 1 ] && echo "$ans1$ans2" | grep -qi red && echo 0 || echo 1) \ + "image: HTTP $code '$ans1', $nres1 restore(s); text chat after it: $(grep -c 'disk cache: restored' "$S/dc-img.log") restore(s), output $([ "$c2" = "$DC_REF2" ] && echo == || echo '!=') reference; image again: HTTP $code3 '$ans2'; SIGTERM with media in the slot: exit $RC, $ncopy state(s) copied, $nf file(s)" + else + fail diskcache/mmproj-image "server did not start, see $S/dc-img.log" + fi + fi + DC_MM=(--no-mmproj) + fi + # (1') GPU: the state is saved from VRAM (warm) and from a host shadow (cold), restored into VRAM local free_mb=$(( ( $(cat "$VRAM_TOTAL") - $(cat "$VRAM_USED") ) / 1048576 )) if [ $free_mb -lt 1536 ]; then