From 54a7f534b6b4ceb0790fcf14ddd9aa6a67aa6d48 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Fri, 9 Oct 2026 00:35:00 +0200 Subject: [PATCH] scripts: rx580-bench - use -lm none, document the MoE cache option Upstream #28334 removed --mmap/--no-mmap/--mlock and llama-bench's -mmp, so the harness flags no longer parse: the server would refuse to start and llama-bench would silently produce no rows. Switch both harnesses to -lm none, the same load mode, and note --moe-cache-mib as a server-only option worth measuring. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ --- scripts/rx580-bench/README.md | 26 +++++++++++++++++++++----- scripts/rx580-bench/lbsweep.sh | 5 +++-- scripts/rx580-bench/ppbench.py | 6 +++--- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/scripts/rx580-bench/README.md b/scripts/rx580-bench/README.md index f029c6a8f..ecdb33a01 100644 --- a/scripts/rx580-bench/README.md +++ b/scripts/rx580-bench/README.md @@ -83,7 +83,7 @@ Results append to `/root/bench/results.txt` as parseable `RESULT` / Production serving config, matching `/root/config.yaml`: ``` --t 6 -ngl 99 --n-cpu-moe 40 -b 2048 -ub 2048 -fa 1 --no-mmap +-t 6 -ngl 99 --n-cpu-moe 40 -b 2048 -ub 2048 -fa 1 -lm none --ctx-size 40960 --no-warmup ``` @@ -100,7 +100,7 @@ Machine: AMD RX 580 8 GB (Polaris/GCN, RADV, no fp16 accel), PCIe 3.0 x16, The `llama-bench` equivalent of that config is: ``` --t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -mmp 0 -p 2048,8192 -n 32 -r 3 +-t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -p 2048,8192 -n 32 -r 3 ``` ## The corpus slices are constants, not recalibrated @@ -171,8 +171,24 @@ Therefore: `supervisorctl -c /root/supervisord.conf start llama-swap`. * **Do not run `llama-cli` detached with closed stdin** on this box; it loops forever. Use the server harness or `llama-bench -d` for depth timing. -* `--no-mmap` is deprecated upstream in favour of `--load-mode` / `-lm none`, - but still accepted and still what production passes. +* **`--mmap` / `--no-mmap` / `--mlock` and llama-bench's `-mmp` are gone** + upstream (#28334). Use `-lm none` (`--load-mode none`) on both + `llama-server` and `llama-bench`. The old flags now fail: the server refuses + to start, `llama-bench` silently produces no rows. `/root/config.yaml` must + be updated to `-lm none` as well before deploying a build past that commit. + +## Optional: the MoE expert cache + +Upstream #29887 added `--moe-cache-mib N` (default 0, off): a device cache of +the most recently used experts of the CPU-resident (`--n-cpu-moe`) layers. It +mostly helps decode and small batches; large prefill batches only read from it. +It is a server option only - `llama-bench` has no flag for it - so measure it +with the corpus harness, e.g. +`LLAMA_ARG_MOE_CACHE_MIB=1024 bench.sh --corpus-only --build DIR --label NAME` +(the environment reaches the spawned server). The variable applies to every +arm of the run, so an on/off comparison spans two runs and the noise floor +above applies in full. The cache takes VRAM from the same 8 GB as the KV +cache and compute buffers. ## Known-good reference values @@ -195,6 +211,6 @@ Older reference points for the same model/config: `pp2048` depth-0 about 285 t/s. The historical "tg32 about 19 t/s" figure is retained only as trivia - do not treat it as a target, and see the tg32 warning above. -Model load with `--no-mmap` takes about 26 s warm, longer cold. +Model load with `-lm none` takes about 26 s warm, longer cold. A full 3-round A/B run of both harnesses takes roughly 100 minutes; add about 16 minutes per extra arm per round. diff --git a/scripts/rx580-bench/lbsweep.sh b/scripts/rx580-bench/lbsweep.sh index ec3ab26d8..9b33ec1ad 100755 --- a/scripts/rx580-bench/lbsweep.sh +++ b/scripts/rx580-bench/lbsweep.sh @@ -13,7 +13,8 @@ # Flag traps, learned the hard way: # * -tb is NOT a llama-bench flag. It exists on llama-server only. # * -ncmoe is the llama-bench spelling of --n-cpu-moe. -# * -mmp 0 is the llama-bench equivalent of the server's --no-mmap. +# * -lm none is the load mode the server runs with. -mmp was removed upstream +# (#28334) and, like any bad flag, now makes the sweep silently empty. # * A bad flag makes llama-bench print usage and exit silently, which looks # exactly like a sweep that produced no rows. Always check row count. # * A ubatch larger than -p never fills, so keep -p a multiple of -ub. @@ -38,7 +39,7 @@ export LABEL BUILD_DIR STAMP RESULTS OUT=$("$BUILD_DIR/bin/llama-bench" \ -m "$MODEL" \ - -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -mmp 0 \ + -t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none \ -p "$LB_P" -n "$LB_N" -r "$LB_R" -o json 2>/dev/null) if [ -z "$OUT" ]; then diff --git a/scripts/rx580-bench/ppbench.py b/scripts/rx580-bench/ppbench.py index 6e4d5ccde..34b80c47d 100755 --- a/scripts/rx580-bench/ppbench.py +++ b/scripts/rx580-bench/ppbench.py @@ -43,10 +43,10 @@ BASE = "http://127.0.0.1:%d" % PORT # Traps encoded here on purpose: # -b / -ub 2048 : a ubatch larger than the prompt never fills, so every # prompt length below must stay a multiple of 2048. -# --no-mmap : deprecated upstream in favour of --load-mode / -lm none, -# but still accepted; it is what production passes today. +# -lm none : what --no-mmap used to be; --mmap/--no-mmap/--mlock were +# removed upstream (#28334), so the old spelling now fails. SERVER_ARGS = ["-t", "6", "-ngl", "99", "--n-cpu-moe", "40", - "-b", "2048", "-ub", "2048", "-fa", "1", "--no-mmap", + "-b", "2048", "-ub", "2048", "-fa", "1", "-lm", "none", "--ctx-size", "40960", "--no-warmup", "--port", str(PORT), "-m", MODEL]