Files
llama.cpp/scripts/rx580-bench/summarize.py
T
LumpiastyandClaude Opus 5.5 62d442e7bf scripts: rx580-bench - decode at tg256 depth 4096, not tg32
The llama-bench sweep measured decode as tg32 at the default depth and the
README called it the continuity metric, while tg32 at n_ctx 32 is launch /
barrier bound on this card and is not a valid decode metric (it once read
-36% between builds at parity at tg256). lbsweep.sh now runs prefill
(-p $LB_P -n 0, depth 0) and decode (-p 0 -n 256 -d 4096, LB_N / LB_D) as
two invocations and labels depth cells (tg256@d4096). summarize.py marks
old depth-less tg rows as invalid-metric history. The README examples and
the reference values are updated to the 2026-10-24 production numbers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 00:33:22 +02:00

102 lines
3.3 KiB
Python
Executable File

#!/usr/bin/env python3
"""
Summarize a results.txt produced by bench.sh into a per-label table.
Usage: summarize.py [results.txt]
Prints median and min-max of every metric for every label, and flags
cross-label deltas against the ~5 percent noise floor of this machine.
"""
import re
import sys
from statistics import median
NOISE_FLOOR_PCT = 5.0
RE_CORPUS = re.compile(
r"^RESULT corpus label=(\S+) build=\S+ target=(\d+) chars=\d+ "
r"prompt_n=(\d+) rep=\d+ prompt_ms=([\d.]+) tps=([\d.]+)")
RE_LB = re.compile(
r"^LBRESULT label=(\S+) build=\S+ test=(\S+) tps=([\d.]+)")
RE_TG_NODEPTH = re.compile(r"^tg\d+-llama-bench$")
def main():
path = sys.argv[1] if len(sys.argv) > 1 else "/root/bench/results.txt"
# data[metric][label] = list of tps
data = {}
order = []
labels = []
for line in open(path):
m = RE_CORPUS.match(line)
if m:
label, target, _pn, _ms, tps = m.groups()
metric = "pp%s-corpus" % target
else:
m = RE_LB.match(line)
if not m:
continue
label, test, tps = m.groups()
metric = "%s-llama-bench" % test
if metric not in data:
data[metric] = {}
order.append(metric)
data[metric].setdefault(label, []).append(float(tps))
if label not in labels:
labels.append(label)
if not data:
print("no parseable results in %s" % path)
return 1
def sort_key(m):
n = re.search(r"(\d+)", m)
return (0 if "corpus" in m else 1, int(n.group(1)) if n else 0)
order.sort(key=sort_key)
w = max(len(l) for l in labels) + 2
head = "metric".ljust(26) + "".join(l.ljust(max(w, 24)) for l in labels)
print("")
print("t/s, median (min-max), n samples")
print(head)
print("-" * len(head))
for metric in order:
row = metric.ljust(26)
for label in labels:
vals = data[metric].get(label)
if not vals:
row += "-".ljust(max(w, 24))
else:
cell = "%.1f (%.1f-%.1f) n=%d" % (
median(vals), min(vals), max(vals), len(vals))
row += cell.ljust(max(w, 24))
print(row)
if any(RE_TG_NODEPTH.match(m) for m in order):
print("note: tgN rows without @d<depth> ran at n_ctx ~N (old default);"
" INVALID decode metric on this box, history only")
if len(labels) >= 2:
base = labels[0]
print("")
print("deltas vs %s (noise floor %.0f%%, anything under it is UNPROVEN)"
% (base, NOISE_FLOOR_PCT))
for other in labels[1:]:
print(" %s vs %s:" % (other, base))
for metric in order:
a = data[metric].get(base)
b = data[metric].get(other)
if not a or not b:
continue
ma, mb = median(a), median(b)
pct = (mb - ma) / ma * 100.0
verdict = "SIGNIFICANT" if abs(pct) >= NOISE_FLOOR_PCT else "unproven"
if RE_TG_NODEPTH.match(metric):
verdict = "INVALID metric (tg at n_ctx ~N), ignore"
print(" %-26s %+6.1f%% %s" % (metric, pct, verdict))
return 0
if __name__ == "__main__":
sys.exit(main())