The llama-bench sweep measured decode as tg32 at the default depth and the README called it the continuity metric, while tg32 at n_ctx 32 is launch / barrier bound on this card and is not a valid decode metric (it once read -36% between builds at parity at tg256). lbsweep.sh now runs prefill (-p $LB_P -n 0, depth 0) and decode (-p 0 -n 256 -d 4096, LB_N / LB_D) as two invocations and labels depth cells (tg256@d4096). summarize.py marks old depth-less tg rows as invalid-metric history. The README examples and the reference values are updated to the 2026-10-24 production numbers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
96 lines
3.4 KiB
Bash
Executable File
96 lines
3.4 KiB
Bash
Executable File
#!/bin/sh
|
|
#
|
|
# llama-bench sweep, the controlled cross-check for the corpus harness.
|
|
# Runs ON THE POD. Normally invoked by bench.sh.
|
|
#
|
|
# Usage: lbsweep.sh <build_dir> <label>
|
|
# Env overrides: RESULTS, MODEL, LB_P, LB_N, LB_D, LB_R
|
|
#
|
|
# Two llama-bench invocations per call:
|
|
# prefill -p $LB_P -n 0 at depth 0 (default pp2048,pp8192)
|
|
# decode -p 0 -n $LB_N -d $LB_D (default tg256 @ d4096)
|
|
# LB_N=0 skips the decode invocation.
|
|
#
|
|
# Everything that must be compared lives inside ONE llama-bench invocation
|
|
# (-p takes a comma-separated list), because cross-invocation variance on this
|
|
# box is about 7 percent while the within-run error bars are about 0.3 percent.
|
|
# Prefill and decode cells are never compared with each other, so they may
|
|
# live in separate invocations.
|
|
#
|
|
# Decode is measured at an explicit depth on purpose: tg32 at the default
|
|
# depth (n_ctx 32) is launch/barrier bound on this card and is NOT a valid
|
|
# decode metric (it once read -36% between two builds that were at parity at
|
|
# tg256). Decode rows are labelled with their depth (tg256@d4096); old tg32
|
|
# rows in results.txt are invalid-metric history.
|
|
#
|
|
# Flag traps, learned the hard way:
|
|
# * -tb is NOT a llama-bench flag. It exists on llama-server only.
|
|
# * -ncmoe is the llama-bench spelling of --n-cpu-moe.
|
|
# * -lm none is the load mode the server runs with. -mmp was removed upstream
|
|
# (#28334) and, like any bad flag, now makes the sweep silently empty.
|
|
# * A bad flag makes llama-bench print usage and exit silently, which looks
|
|
# exactly like a sweep that produced no rows. Always check row count.
|
|
# * A ubatch larger than -p never fills, so keep -p a multiple of -ub.
|
|
#
|
|
set -e
|
|
BUILD_DIR="$1"
|
|
LABEL="$2"
|
|
if [ -z "$BUILD_DIR" ] || [ -z "$LABEL" ]; then
|
|
echo "usage: $0 <build_dir> <label>" >&2
|
|
exit 2
|
|
fi
|
|
|
|
RESULTS="${RESULTS:-/root/bench/results.txt}"
|
|
MODEL="${MODEL:-/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf}"
|
|
LB_P="${LB_P:-2048,8192}"
|
|
LB_N="${LB_N:-256}"
|
|
LB_D="${LB_D:-4096}"
|
|
LB_R="${LB_R:-3}"
|
|
|
|
STAMP=$(date -u +%Y-%m-%dT%H:%M:%S)
|
|
export LD_LIBRARY_PATH="$BUILD_DIR/bin"
|
|
export LABEL BUILD_DIR STAMP RESULTS
|
|
|
|
lb() {
|
|
"$BUILD_DIR/bin/llama-bench" \
|
|
-m "$MODEL" \
|
|
-t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none \
|
|
-r "$LB_R" -o json "$@" 2>/dev/null
|
|
}
|
|
|
|
record() { # $1 = what, stdin = llama-bench JSON
|
|
WHAT="$1" python3 -c '
|
|
import json, sys, os
|
|
label = os.environ["LABEL"]
|
|
build = os.environ["BUILD_DIR"]
|
|
stamp = os.environ["STAMP"]
|
|
results = os.environ["RESULTS"]
|
|
what = os.environ["WHAT"]
|
|
text = sys.stdin.read()
|
|
if not text.strip():
|
|
sys.stderr.write("ERROR: llama-bench produced no output for %s %s (bad flag?)\n" % (label, what))
|
|
sys.exit(3)
|
|
rows = json.loads(text)
|
|
if not rows:
|
|
sys.stderr.write("ERROR: llama-bench returned zero rows for %s %s\n" % (label, what))
|
|
sys.exit(3)
|
|
out = open(results, "a")
|
|
for r in rows:
|
|
test = "pp%d" % r["n_prompt"] if r["n_prompt"] else "tg%d" % r["n_gen"]
|
|
if r.get("n_depth", 0):
|
|
test += "@d%d" % r["n_depth"]
|
|
line = ("LBRESULT label=%s build=%s test=%s tps=%.2f stddev=%.2f "
|
|
"samples=%s time=%s"
|
|
% (label, build, test, r["avg_ts"], r["stddev_ts"],
|
|
r["samples_ts"], stamp))
|
|
out.write(line + "\n")
|
|
print(line)
|
|
out.close()
|
|
'
|
|
}
|
|
|
|
lb -p "$LB_P" -n 0 | record prefill
|
|
if [ "$LB_N" -gt 0 ]; then
|
|
lb -p 0 -n "$LB_N" -d "$LB_D" | record decode
|
|
fi
|