Compare commits
50
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
adc2aa3ecb | ||
|
|
fe57bc93a0 | ||
|
|
d9ca3278bf | ||
|
|
cb48bf2240 | ||
|
|
18691e0090 | ||
|
|
2c6c517ba5 | ||
|
|
2874dcb107 | ||
|
|
dd851fd0c9 | ||
|
|
f24c3582c2 | ||
|
|
71d4cbe195 | ||
|
|
2fc9f9c351 | ||
|
|
cce0ac1f09 | ||
|
|
d7b3510499 | ||
|
|
47b892271c | ||
|
|
36514dba27 | ||
|
|
e5fc474b87 | ||
|
|
ceab1a91f8 | ||
|
|
80e662a2a1 | ||
|
|
2512290f94 | ||
|
|
62d442e7bf | ||
|
|
f664b0f20e | ||
|
|
f5c0d8f28a | ||
|
|
bfbaf15b2a | ||
|
|
149a8cb0ae | ||
|
|
e085c347d2 | ||
|
|
6c480f7e70 | ||
|
|
11044b35e3 | ||
|
|
54a7f534b6 | ||
|
|
7ac2014a80 | ||
|
|
53f9077719 | ||
|
|
878d1338f4 | ||
|
|
4c4486f8e9 | ||
|
|
5e429d4a9c | ||
|
|
b7c131219c | ||
|
|
5e99ac1982 | ||
|
|
0a0f3ab6d4 | ||
|
|
ad27d49218 | ||
|
|
9b555f1aad | ||
|
|
9857a45f0c | ||
|
|
bf2c1a6992 | ||
|
|
2feb6aeb27 | ||
|
|
8d80f0673f | ||
|
|
b9df854de8 | ||
|
|
958257cd60 | ||
|
|
17165fe2a7 | ||
|
|
481be8b284 | ||
|
|
3df719e1e8 | ||
|
|
ff2a96d2d4 | ||
|
|
fb6a8a0761 | ||
|
|
c72cb424e1 |
No files matched your search
@@ -17,6 +17,32 @@
|
|||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
## This fork - RX 580 (Polaris / GCN) serving of large MoE models
|
||||||
|
|
||||||
|
A private fork maintained for one machine: an AMD RX 580 8 GB (RADV Vulkan) serving Qwen3.6-35B-A3B,
|
||||||
|
gemma-4-26B-A4B and smaller models under llama-swap, with the MoE experts in host RAM
|
||||||
|
(`--n-cpu-moe`). **[FORK.md](FORK.md) is the source of truth**: every fork change with its
|
||||||
|
invariants, rebase hazards and validation, the measured dead ends, the production configuration and
|
||||||
|
the measurement traps. The rebase and deploy procedure is
|
||||||
|
[docs/fork-rebase-runbook.md](docs/fork-rebase-runbook.md); `scripts/fork-validate.sh` validates a
|
||||||
|
build against a reference; `scripts/rx580-bench/` is the benchmark harness.
|
||||||
|
|
||||||
|
In short, the fork adds:
|
||||||
|
|
||||||
|
- three Polaris prefill wins, on without a flag and token-identical to mainline: the MoE routing-ids
|
||||||
|
readback skip in `sched_copy_experts`, flash-attention `mask_opt` for GCN at head size >= 256, and
|
||||||
|
a tiled-transpose fast path for the delta-net conv-state concat;
|
||||||
|
- opt-in VRAM time-sharing between several llama-server processes (`LLAMA_SLEEP_VRAM_ONLY`,
|
||||||
|
`LLAMA_SLEEP_EVICT_KV`, `LLAMA_MMPROJ_ONDEMAND`, `LLAMA_VRAM_ARENA`): a model releases everything it
|
||||||
|
holds in VRAM and gets it back without a restart, so its RAM prompt cache survives;
|
||||||
|
- fixes to the server's sleep, wake and shutdown paths and to cold teardown that the sharing exposed;
|
||||||
|
- validation-only env knobs (`GGML_VK_CONCAT_FASTPATH`, `LLAMA_MOE_IDS_SKIP`), unset in production.
|
||||||
|
|
||||||
|
Production settings for the 35B are in FORK.md section 5 (`-lm none -t 6 -ngl 99 --n-cpu-moe 40
|
||||||
|
-b 2048 -ub 2048 -fa 1 --parallel 1`, ctx 131072). Keep all experts on the host (`--n-cpu-moe 40`)
|
||||||
|
for long contexts: lower values are faster only at short context and run out of VRAM as the KV grows
|
||||||
|
(FORK.md section 4).
|
||||||
|
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
A few options to get `llama.cpp` installed on your machine:
|
A few options to get `llama.cpp` installed on your machine:
|
||||||
|
|||||||
@@ -1732,6 +1732,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||||||
params.cache_ram_explicit = true;
|
params.cache_ram_explicit = true;
|
||||||
}
|
}
|
||||||
).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
|
).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
|
||||||
|
add_opt(common_arg(
|
||||||
|
{"--cache-disk-path"}, "PATH",
|
||||||
|
"save the state of each slot with a prompt to this directory on shutdown, and restore it for a prompt that starts with it after a restart (default: disabled)",
|
||||||
|
[](common_params & params, const std::string & value) {
|
||||||
|
params.cache_disk_path = value;
|
||||||
|
}
|
||||||
|
).set_env("LLAMA_ARG_CACHE_DISK_PATH").set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||||
|
add_opt(common_arg(
|
||||||
|
{"--cache-disk-size"}, "N",
|
||||||
|
string_format("size limit of the --cache-disk-path directory in MiB, the least recently used files are removed (default: %d, 0 - no limit)", params.cache_disk_mib),
|
||||||
|
[](common_params & params, int value) {
|
||||||
|
if (value < 0) {
|
||||||
|
throw std::invalid_argument("cache-disk-size must be non-negative");
|
||||||
|
}
|
||||||
|
params.cache_disk_mib = value;
|
||||||
|
}
|
||||||
|
).set_env("LLAMA_ARG_CACHE_DISK_SIZE").set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||||
add_opt(common_arg(
|
add_opt(common_arg(
|
||||||
{"-kvu", "--kv-unified"},
|
{"-kvu", "--kv-unified"},
|
||||||
{"-no-kvu", "--no-kv-unified"},
|
{"-no-kvu", "--no-kv-unified"},
|
||||||
|
|||||||
@@ -641,6 +641,8 @@ struct common_params {
|
|||||||
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
|
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
|
||||||
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
|
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
|
||||||
bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise
|
bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise
|
||||||
|
std::string cache_disk_path; // directory of the disk prompt cache, empty = disabled
|
||||||
|
int32_t cache_disk_mib = 4096; // size limit of cache_disk_path in MiB, 0 = no limit
|
||||||
|
|
||||||
// llama-connect params
|
// llama-connect params
|
||||||
bool server_connect = false;
|
bool server_connect = false;
|
||||||
|
|||||||
@@ -0,0 +1,559 @@
|
|||||||
|
# Fork rebase and deploy runbook (RX 580 fork)
|
||||||
|
|
||||||
|
Step-by-step procedure for moving the fork onto a new upstream and putting it into production, as it
|
||||||
|
has evolved over the 2026-09-09, 2026-10-09 and 2026-10-24 rebases. Read [FORK.md](../FORK.md) first:
|
||||||
|
it lists every fork feature with its invariants and rebase hazards, and this runbook refers to it.
|
||||||
|
|
||||||
|
Ground rules:
|
||||||
|
|
||||||
|
- **Never build on the workstation.** All builds run on the RX 580 pod. Locally, rely on careful reading.
|
||||||
|
- **Never resolve a conflict with `-X theirs` or `-X ours`.** Resolve every hunk by hand.
|
||||||
|
- **"Applied cleanly" is not "correct".** Re-read every clean pick against the new upstream.
|
||||||
|
- **Never `pkill -f` on the pod.** It matches the `kubectl exec` shell and kills it. Kill by explicit PID.
|
||||||
|
- **Never leave the pod not serving.** Anything that stops llama-swap must restart it from a trap.
|
||||||
|
- Keep the previous production branch untouched as the safety net. Do not push for the owner; the owner
|
||||||
|
reviews and pushes.
|
||||||
|
|
||||||
|
Contents: 1. Survey; 2. Trial rebase; 3. The real rebase; 4. Checks without a compiler; 5. Build on
|
||||||
|
the pod; 6. Validate; 7. Deploy; 8. Post-deploy checks; 9. Rollback; 10. Recovering a hung GPU;
|
||||||
|
11. Operations reference.
|
||||||
|
|
||||||
|
**Redeploying a new tip on the same upstream base (no rebase)**, e.g. new fork commits on the
|
||||||
|
production branch (2026-10-25: `6c480f7e7` -> `2fc9f9c35`, same branch, same base):
|
||||||
|
- Skip sections 1-3. Of section 4 only the ASCII check (on `git diff <prod>..<tip>`) and a careful
|
||||||
|
read of that diff apply; `<prod>` is `git -C /root/llama.cpp rev-parse HEAD` on the pod. There is
|
||||||
|
nothing to range-diff.
|
||||||
|
- Do sections 5-9 as written: build the tip in a scratch worktree, validate it with `--ref` = the
|
||||||
|
production build, then deploy.
|
||||||
|
- The branch name stays the same, so deploy step 2's `git checkout -B <same branch> <tip>` moves the
|
||||||
|
production branch on the pod to the tip. After that the rollback pointer file is the only record
|
||||||
|
of the old production commit on the pod, so write it first (deploy step 1).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Survey upstream
|
||||||
|
|
||||||
|
Local remotes: `origin` is upstream (github.com/ggerganov/llama.cpp), `gitea` is the fork
|
||||||
|
(git@gitea.lumpiasty.xyz:Lumpiasty/llama.cpp.git, SSH push from the workstation). On the pod,
|
||||||
|
`origin` is the same gitea repository over HTTPS (fetch only).
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git fetch origin
|
||||||
|
OLD_BASE=$(git merge-base origin/master HEAD) # the old upstream base (still an ancestor of the new master)
|
||||||
|
OLD_TIP=$(git rev-parse HEAD)
|
||||||
|
git log --oneline "$OLD_BASE"..HEAD # the fork commits, oldest last
|
||||||
|
git log --oneline "$OLD_BASE"..origin/master | wc -l # size of the upstream delta
|
||||||
|
```
|
||||||
|
|
||||||
|
Then look at upstream changes in the areas FORK.md lists as hazards:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git log --oneline "$OLD_BASE"..origin/master -- \
|
||||||
|
src/llama-context.cpp src/llama-context.h src/llama-model.cpp src/llama-model.h \
|
||||||
|
src/llama-kv-cache.cpp src/llama-kv-cache.h src/llama-kv-cache-iswa.cpp \
|
||||||
|
src/llama-memory.h src/llama-memory-recurrent.cpp src/llama-memory-hybrid.cpp \
|
||||||
|
src/llama-moe-cache.cpp src/llama-moe-cache.h src/models/delta-net-base.cpp \
|
||||||
|
ggml/src/ggml-vulkan/ ggml/src/ggml-backend.cpp ggml/src/ggml-backend-impl.h ggml/include/ggml-backend.h \
|
||||||
|
tools/server/server-queue.cpp tools/server/server-queue.h tools/server/server-context.cpp tools/server/server.cpp \
|
||||||
|
tools/mtmd/clip.cpp tools/mtmd/mtmd.cpp common/arg.cpp tools/llama-bench/
|
||||||
|
```
|
||||||
|
|
||||||
|
For each fork commit decide: still needed, needs a semantic merge, or obsolete. Examples of past
|
||||||
|
decisions:
|
||||||
|
- 2026-09-09: dropped the weight-pinning chain, the expert prefetch and fractional `-ncmoe` (all measured
|
||||||
|
losses or no win on this card, FORK.md section 4).
|
||||||
|
- 2026-10-09: the routing-ids readback skip had nothing to apply to (upstream #29943 deleted its block
|
||||||
|
in `ggml-backend.cpp`); it was re-authored in `llama_context::sched_copy_experts`. Upstream's new MoE
|
||||||
|
cache (#29887) needed a new fork commit so the VRAM arbiter releases and invalidates it.
|
||||||
|
- Also check CLI changes that break production config or the scripts (upstream #28334 removed
|
||||||
|
`--no-mmap`/`--mmap`/`--mlock` and llama-bench `-mmp`; the default `--port` changed to 9931).
|
||||||
|
|
||||||
|
## 2. Trial rebase in a scratch worktree
|
||||||
|
|
||||||
|
Measure the conflicts before doing the real thing:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git worktree add /tmp/trial-rebase origin/master
|
||||||
|
cd /tmp/trial-rebase
|
||||||
|
git cherry-pick "$OLD_BASE".."$OLD_TIP" # stop at each conflict, note file + hunk, then:
|
||||||
|
git cherry-pick --abort # or resolve roughly to see the next one
|
||||||
|
cd - && git worktree remove --force /tmp/trial-rebase
|
||||||
|
```
|
||||||
|
|
||||||
|
Write down, per commit: clean / conflict (which file, which hunk, what both sides did). That list
|
||||||
|
becomes the brief for the real rebase.
|
||||||
|
|
||||||
|
## 3. The real rebase
|
||||||
|
|
||||||
|
```sh
|
||||||
|
NEW=fable5/vram-ondemand-rebase-$(date +%Y%m%d)
|
||||||
|
git switch -c "$NEW" origin/master
|
||||||
|
git cherry-pick <first-fork-commit> # one at a time, in the original order
|
||||||
|
```
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- Cherry-pick in the original order, keep authors and messages. New commits you author get the
|
||||||
|
fork's trailer (`Co-Authored-By: Claude ...` for agent-written commits, FORK.md status block),
|
||||||
|
unless the owner asks for something else.
|
||||||
|
- Resolve every conflict by hand. Union both sides where both added code; keep upstream's new terms.
|
||||||
|
- Fold each fix into the commit it belongs to so every commit builds on its own:
|
||||||
|
`git commit --fixup=<sha>` then `GIT_SEQUENCE_EDITOR=: git rebase -i --autosquash origin/master`.
|
||||||
|
- Leave the previous production branch untouched.
|
||||||
|
|
||||||
|
**What `-X theirs` silently broke in a trial (2026-10-09).** It "applied" every commit and produced
|
||||||
|
broken code:
|
||||||
|
- resurrected the scheduler readback block upstream had deleted from `ggml-backend.cpp`;
|
||||||
|
- dropped `!use_sparse` from the GCN `use_mask_opt` condition;
|
||||||
|
- dropped upstream's `copy_experts_info` (compile error);
|
||||||
|
- dropped upstream's `is_empty()` in the recurrent memory;
|
||||||
|
- dropped upstream's MoE-cache memory breakdown block;
|
||||||
|
- left a static/extern mismatch with the new `ggml-vulkan-common.h` declarations (compile error).
|
||||||
|
|
||||||
|
The correct resolutions in that rebase: keep upstream's `n_embd` width check plus our decode
|
||||||
|
auto-restore; `copy_experts_info` plus our `kv_device_evicted`; `is_empty()` plus our `dev_shadows`;
|
||||||
|
upstream's `if (moe_cache)` breakdown block outside our `if (sched)` guard; make `ggml_vk_concat` and
|
||||||
|
`ggml_backend_vk_free` non-static to match the new header while our helpers stay static.
|
||||||
|
|
||||||
|
**Clean picks that were wrong (2026-10-24).**
|
||||||
|
- Concat fast path (3df719e1e): applied cleanly, did not compile. Upstream `733f57a99` renamed
|
||||||
|
`pipeline_cpy_f32_f32`/`pipeline_cpy_f16_f16` to `pipeline_cpy_32`/`pipeline_cpy_16` (bitwise copies,
|
||||||
|
same push constants).
|
||||||
|
- MoE cache release/restore (7ac2014a8): applied cleanly, did not compile (upstream `33d6135cb` changed
|
||||||
|
LRU to SLRU, `g.lru` -> `g.slru`) and had a runtime bug: `moe_cache_slru::init()` resets only the
|
||||||
|
probation segment, so re-initialising a used cache corrupts its lists. `invalidate()` now assigns a
|
||||||
|
fresh `moe_cache_slru()` first.
|
||||||
|
- VRAM sharing core (481be8b28): besides its one conflict, upstream's new `load_tensors_hadamard()`
|
||||||
|
pushes two more entries into `pimpl->ctxs_bufs`. Without matching `weight_release.emplace_back()`
|
||||||
|
calls, `release_device_weights()` reads past the end of `weight_release`. Nothing conflicted.
|
||||||
|
- Conflicts that round: `ctx_buf_maps.emplace_back` replaced upstream by `ctx_bufs.emplace_back(ctx,
|
||||||
|
std::move(buf_ptrs))` (keep upstream's line, our release-slot block before it); `llama-context.h`
|
||||||
|
(keep upstream's `n_reallocs_same/diff` beside our `kv_device_evicted`); `recv_with_timeout`
|
||||||
|
(upstream `4e201388c` uses a fixed deadline: keep our order "check `!running`, return null", then
|
||||||
|
`wait_until(lock, deadline)`).
|
||||||
|
|
||||||
|
**Semantic re-checks to do for every rebase, clean or not** (details in FORK.md):
|
||||||
|
- F1: `sched_copy_experts` and its `st` struct unchanged in meaning.
|
||||||
|
- F2: `use_mask_opt` still contains upstream's terms plus our `>= 256`.
|
||||||
|
- F3: `build_conv_state` still emits `ggml_concat(conv_states, ggml_transpose(qkv_mixed), 0)` with
|
||||||
|
`qkv_mixed` contiguous `[ch, n_seq_tokens, n_seqs]`; pipeline names; `types_ok` still excludes
|
||||||
|
quantized types.
|
||||||
|
- F4: every `ctxs_bufs` push in `src/llama-model.cpp` has a `weight_release` push
|
||||||
|
(`git grep -n "ctxs_bufs.emplace_back\|weight_release" src/llama-model.cpp`); list memory types
|
||||||
|
(`git grep -n "public llama_memory_i" src/`) and note any without a release override; MoE-cache
|
||||||
|
internals; `sched_reserve()` still goes through `sched_new()` and re-registers callbacks;
|
||||||
|
`free_scratch` still the last `ggml_backend_i` member; Vulkan prealloc fields match `ggml_vk_cleanup`.
|
||||||
|
- F6/F7: `server_queue::start_loop` still evaluates `should_sleep()` only from the idle branch, never
|
||||||
|
inside `yield_to_queue()`; `handle_sleeping_state`, `server_res_generator`, `load_model` ordering;
|
||||||
|
signal setup in `server.cpp`.
|
||||||
|
- F8: new iterators over `ctxs_bufs` or users of `sched` on teardown paths.
|
||||||
|
- Scripts: flag spellings used by `scripts/rx580-bench/*` and `scripts/fork-validate.sh`, and the log
|
||||||
|
lines `fork-validate.sh` greps for (FORK.md F9).
|
||||||
|
- Production config: every flag in `/root/config.yaml` still exists in `common/arg.cpp`.
|
||||||
|
|
||||||
|
## 4. Checks without a compiler
|
||||||
|
|
||||||
|
Before handing the branch over for a build:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git diff origin/master..HEAD | grep -n '^[+-]\(<<<<<<<\|=======\|>>>>>>>\)' # no conflict markers
|
||||||
|
git diff origin/master..HEAD | grep -nP '^\+.*[^\x00-\x7F]' # ASCII only in added lines
|
||||||
|
git range-diff "$OLD_BASE".."$OLD_TIP" origin/master..HEAD # patch vs patch
|
||||||
|
```
|
||||||
|
|
||||||
|
- The range-diff must show only the edits you made on purpose. In 2026-10-24 the whole patch was
|
||||||
|
identical to the old one except for 4 deliberate edits.
|
||||||
|
- Brace balance per changed file: the change in `{`/`}` balance from new base to new tip should equal
|
||||||
|
the change from old base to old tip.
|
||||||
|
- Every identifier in added lines must exist on the new master or be defined by the fork. This catches
|
||||||
|
a renamed function but not a renamed member (like `lru` -> `slru`); re-read member accesses by hand.
|
||||||
|
- Write down every spot where you would normally lean on the compiler and hand that list to whoever
|
||||||
|
builds.
|
||||||
|
|
||||||
|
Then the owner pushes the branch to gitea.
|
||||||
|
|
||||||
|
## 5. Build on the pod, in a scratch worktree
|
||||||
|
|
||||||
|
Production keeps serving during this step.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export KUBECONFIG=/home/user/Projects/klaster/talos/generated/kubeconfig
|
||||||
|
kubectl -n llama exec -i deploy/supervisord -- sh -c '
|
||||||
|
set -e
|
||||||
|
B=fable5/vram-ondemand-rebase-YYYYMMDD
|
||||||
|
W=/root/arms/rebYYYYMMDD
|
||||||
|
git -C /root/llama.cpp fetch origin "$B"
|
||||||
|
git -C /root/llama.cpp worktree add "$W" FETCH_HEAD
|
||||||
|
cd "$W"
|
||||||
|
cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_VULKAN=1 -DGGML_NATIVE=ON -DLLAMA_BUILD_UI=OFF
|
||||||
|
nice -n 19 cmake --build build --target llama-server llama-bench test-backend-ops -j 8
|
||||||
|
'
|
||||||
|
```
|
||||||
|
|
||||||
|
- Build only the targets you need; the full `all` target fails on the UI assets (needs npm/network).
|
||||||
|
If a tree has a stale `tools/ui/dist` with a stub `index.html`, remove it (it breaks llama-ui-embed).
|
||||||
|
- `nice -n 19 -j 8` leaves the serving models CPU. A full build takes about 3.5 minutes.
|
||||||
|
- Report compile errors verbatim against the list from step 4.
|
||||||
|
- Do not overlap a build with a large benchmark (host OOM). Disk is tight: `/root` had about 13 GB
|
||||||
|
free (97% full) on 2026-10-25, so check `df -h /root` first. Each build tree is about 700 MB;
|
||||||
|
remove scratch worktrees you no longer need with `git -C /root/llama.cpp worktree remove <dir>`.
|
||||||
|
- The build prints `UI: download dist.tar.gz.sha256 ... failed` (no UI assets download). It is
|
||||||
|
harmless with `-DLLAMA_BUILD_UI=OFF`; the listed targets still build.
|
||||||
|
- Long runs: launch detached (`setsid nohup sh X > X.log 2>&1 </dev/null &`) and poll, rather than
|
||||||
|
holding a `kubectl exec` open for hours.
|
||||||
|
- The validated build must contain all three targets: `llama-server`, `llama-bench` and
|
||||||
|
`test-backend-ops` (`--perf` and `--ops` run the last two from `--build`). A build script that
|
||||||
|
makes only `llama-server` (as the 2026-10-25 `diskcache-build.sh` first did) is not a validated
|
||||||
|
build.
|
||||||
|
- Detach only one script. Upload it first (`kubectl exec -i ... sh -c 'cat > /root/arms/X.sh' <
|
||||||
|
X.sh`), then start it alone: `setsid nohup sh /root/arms/X.sh > X.log 2>&1 </dev/null &`. Do not
|
||||||
|
background a chain such as `cat > X && sh -n X && setsid ... &` inside the `kubectl exec` shell. The
|
||||||
|
chain is a child of that session, and it can die with the session before `setsid` runs.
|
||||||
|
|
||||||
|
## 6. Validate with `scripts/fork-validate.sh`
|
||||||
|
|
||||||
|
Copy the script from the candidate tree (it is in the repo) and run it in two phases. The header of
|
||||||
|
the script describes each check.
|
||||||
|
|
||||||
|
Rule: any test of a production model must use production's exact command line: the `cmd` of its
|
||||||
|
`/root/config.yaml` entry with the macros expanded, including the mmproj that `-hf` pulls (production
|
||||||
|
never passes `--no-mmproj` to the 35B or gemma), and supervisord's `environment=`. Change only
|
||||||
|
`--port`, the arena and scratch paths. The 2026-10-25 deploy of the disk prompt cache aborted on the
|
||||||
|
first request: every test had used `--no-mmproj`, which hid a `GGML_ASSERT(!has_mtmd)`, and it was
|
||||||
|
rolled back (FORK.md F11).
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# phase 1, beside production
|
||||||
|
bash /root/arms/rebYYYYMMDD/scripts/fork-validate.sh \
|
||||||
|
--build /root/arms/rebYYYYMMDD/build --ref /root/llama.cpp/build --ops --concat
|
||||||
|
```
|
||||||
|
|
||||||
|
Add `--arbiter` to phase 1 only if the card has >= 1536 MiB free (8192 MiB minus the bytes in
|
||||||
|
`cat /sys/class/drm/card*/device/mem_info_vram_used`); below that it FAILs at once. With gemma-26B
|
||||||
|
warm in production only about 1.2 GiB is free and nothing frees it without downtime; then run
|
||||||
|
`--arbiter` in the phase-2 swapdown window instead (about 2 minutes more downtime), as below.
|
||||||
|
|
||||||
|
**Page cache before and after phase 2.** The `--ref` build may lack F10 (FORK.md), and `--perf`
|
||||||
|
loads the 35B many times: during a reference arm on 2026-10-25 MemAvailable fell to 27 GB. Run the
|
||||||
|
eviction snippet of deploy step 3 before phase 2 and again after it (swapdown does not), and check
|
||||||
|
`/proc/meminfo`.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# phase 2, with llama-swap stopped: run it under the swapdown wrapper, whose trap restarts llama-swap
|
||||||
|
cat > /root/arms/fv-perf.sh <<'EOF'
|
||||||
|
HUB=/root/.cache/huggingface/hub
|
||||||
|
Q35=$HUB/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
||||||
|
G26=$HUB/models--SC117--gemma-4-26B-A4B-it-qat-heretic-GGUF/snapshots/e6759b29958180a998f62f991e4c741fe9b4d452/gemma-4-26B-A4B-it-qat-heretic-UD-Q4_K_XL.gguf
|
||||||
|
bash /root/arms/rebYYYYMMDD/scripts/fork-validate.sh \
|
||||||
|
--build /root/arms/rebYYYYMMDD/build --ref /root/llama.cpp/build \
|
||||||
|
--arbiter --perf --moe-exact "$Q35" --moecache "$G26" --keep
|
||||||
|
EOF
|
||||||
|
setsid nohup sh /root/bench/moe/swapdown.sh /root/arms/fv-perf.sh > /root/arms/fv-perf.log 2>&1 </dev/null &
|
||||||
|
```
|
||||||
|
|
||||||
|
- `G26` is the production SC117 gemma-26B, the `refs/main` snapshot (`e6759b29...` on 2026-10-25;
|
||||||
|
an older snapshot `82c751632...` is also on disk). If it moves, read the snapshot name from
|
||||||
|
`$HUB/models--SC117--gemma-4-26B-A4B-it-qat-heretic-GGUF/refs/main`.
|
||||||
|
- `/root/bench/moe/swapdown.sh <script>` (pod-only helper, not in the repo) stops llama-swap, waits for
|
||||||
|
an idle GPU, runs the script, and restarts llama-swap from an EXIT/INT/TERM/HUP trap. It prints
|
||||||
|
`SWAPDOWN_DONE` at the end. Afterwards confirm llama-swap is RUNNING and `/v1/models` returns 200.
|
||||||
|
- The `--concat` check needs a build with the `GGML_VK_CONCAT_FASTPATH` knob (in the fork since
|
||||||
|
e085c347d, so a rebased candidate has it; older candidates need `--concat-build DIR`).
|
||||||
|
- `--moecache` needs a cache that actually hits (default 2048 MiB; at 256 MiB gemma-26B had 0 hits,
|
||||||
|
which proves nothing).
|
||||||
|
- `--arbiter` also checks that a cold server really frees its VRAM (`v-vram-release`,
|
||||||
|
`v-vram-restore`; SKIPs if production's VRAM moved during the measurement, then rerun) and A/Bs
|
||||||
|
ordinary handoff latency against `--ref` (`a-handoff-latency`). `--ref` must be the build currently
|
||||||
|
in production, so both comparisons are against what serves today.
|
||||||
|
- Phase 2 downtime, measured 2026-10-25: about 32 minutes in all (`--arbiter` about 2 min, `--perf`
|
||||||
|
about 28 min, `--moecache` under 30 s, `--moe-exact` 78 s). `--moe-exact` proves the routing-ids
|
||||||
|
readback skip exact (F1): the candidate's greedy output equals the reference's and, with the
|
||||||
|
`LLAMA_MOE_IDS_SKIP` knob (any build of this branch), the candidate's with the skip forced off.
|
||||||
|
- `--perf` gates on medians of all repetition samples (since 2512290f9); the samples are in
|
||||||
|
`<scratch>/perf.tsv`, and `fork-validate.sh --perf-report <file>` re-aggregates them offline. A pod
|
||||||
|
restart empties `/tmp`, so copy a scratch dir you want to keep to `/root/arms/` (the 2026-10-24
|
||||||
|
scratch dirs were lost that way).
|
||||||
|
- Last full run (2026-10-25, candidate `2fc9f9c35` vs reference `6c480f7e7`, logs
|
||||||
|
`/root/arms/dep-2fc9f9c35-fv1.log` and `-fv2.log`): phase 1 `--ops --concat` 8 PASS, phase 2
|
||||||
|
`--arbiter --perf --moe-exact --moecache` 29 PASS; 37 PASS, 0 FAIL. Perf medians ref/cand: pp2048
|
||||||
|
274.53/273.82, d16384 200.52/200.37, d32768 157.60/157.57, tg256@d4096 16.82/16.66 t/s. Handoff
|
||||||
|
median ref 693.5 / cand 678 ms. `--moe-exact`: 39 skips logged, identical to the reference and to
|
||||||
|
the skip forced off. Peak VRAM per arm ref 5476, cand 7062, cand 5767, ref 5476 MiB: the ~1.6 GiB
|
||||||
|
spike in the first candidate arm has now been seen twice (open question, FORK.md F9).
|
||||||
|
- Any arbiter change: `a-handoff-latency` is the ordinary-handoff A/B that two lost deploys showed
|
||||||
|
is needed. For a change aimed at latency itself, also run more blocks by hand; 25%/200 ms is a
|
||||||
|
regression gate, not a measurement.
|
||||||
|
- Disk prompt cache (FORK.md F11, from `fable5/disk-prompt-cache`): add `--diskcache` to phase 1. It
|
||||||
|
is CPU only and runs beside production (about 2 minutes). Its GPU leg SKIPs below 1536 MiB of free
|
||||||
|
VRAM; then run `--diskcache` again in the phase-2 window. A 35B measurement driver is on the pod:
|
||||||
|
`/root/arms/diskcache-test/q35.py [CHARS]` (scratch port 8095, scratch arena and cache dir, the
|
||||||
|
production 35B flags, a pan-tadeusz conversation; about 6 minutes). The 2026-10-25 window script
|
||||||
|
`/root/arms/diskcache-window.sh` (page-cache evictions, `--diskcache --arbiter`, `q35.py`) runs
|
||||||
|
under swapdown (run 2026-10-25: 29 PASS). `/root/arms/diskcache-window3.sh` runs `q35.py`
|
||||||
|
with production's exact 35B command line (mmproj loaded): save/restore, cold save, the
|
||||||
|
same-reuse-point identity and an image request (FORK.md F11). Not yet run; it supersedes
|
||||||
|
`diskcache-window2.sh` (`cmp.py` runs with `--no-mmproj`).
|
||||||
|
|
||||||
|
On any FAIL: stop and report; the scratch dir (`/tmp/fork-validate.*`) is kept with all logs.
|
||||||
|
|
||||||
|
## 7. Deploy
|
||||||
|
|
||||||
|
Deploy exactly the validated commit. Downtime: 6 minutes for the 2026-10-24 redeploy. The deploy
|
||||||
|
rebuilds the production tree in place (step 6) from the same commit and with the same cmake flags as
|
||||||
|
the validated worktree build, so nothing is copied over; compare
|
||||||
|
`grep -E 'GGML_NATIVE|LLAMA_BUILD_UI|BUILD_SHARED_LIBS|CMAKE_BUILD_TYPE' build/CMakeCache.txt` in both
|
||||||
|
trees if in doubt.
|
||||||
|
|
||||||
|
Run the stop, build and start (steps 4-7) as one script on the pod, started with `setsid nohup sh
|
||||||
|
<script> > <log> 2>&1 </dev/null &`, never as a backgrounded `cat && sh -n && setsid` chain from the
|
||||||
|
`kubectl exec` shell (see section 9).
|
||||||
|
|
||||||
|
1. **Rollback pointer.** Record the current production commit in the next free pointer file, as the
|
||||||
|
full SHA alone on line 1 (older files mix full SHAs, short SHAs and extra text; see section 9):
|
||||||
|
```sh
|
||||||
|
ls /root/DEPLOY-ROLLBACK-commit-*.txt # last used: 10 (holds 6c480f7e7)
|
||||||
|
git -C /root/llama.cpp rev-parse HEAD > /root/DEPLOY-ROLLBACK-commit-11.txt
|
||||||
|
git -C /root/llama.cpp status --short # must be clean
|
||||||
|
```
|
||||||
|
2. **Check out the candidate** in the production tree (fetch first; the branch must be on gitea):
|
||||||
|
```sh
|
||||||
|
cd /root/llama.cpp
|
||||||
|
git fetch origin fable5/vram-ondemand-rebase-YYYYMMDD
|
||||||
|
git checkout -B fable5/vram-ondemand-rebase-YYYYMMDD <validated-commit>
|
||||||
|
```
|
||||||
|
On a redeploy with the same branch name this moves the production branch to the new commit.
|
||||||
|
3. **Evict the page cache.** The pod is BestEffort QoS on a shared 62 GB node, so it is the first OOM
|
||||||
|
victim, and benchmarks, mmap loads and builds without F10 leave tens of GB of model files cached.
|
||||||
|
During the first 2026-10-24 deploy the Talos OOM controller killed the whole pod 17 s into
|
||||||
|
verification (cause likely, not proven, memory pressure). Evict while production still runs
|
||||||
|
(unprivileged and safe: `-lm none` servers hold their weights in anonymous memory, and pages
|
||||||
|
mapped by an mmap model are not dropped). The same snippet is used before and after validation
|
||||||
|
phase 2:
|
||||||
|
```sh
|
||||||
|
python3 - <<'EOF'
|
||||||
|
import glob, os
|
||||||
|
n = 0
|
||||||
|
for root in ('/root/.cache/huggingface/hub', '/root/models'):
|
||||||
|
for p in glob.glob(root + '/**/*.gguf', recursive=True):
|
||||||
|
fd = os.open(p, os.O_RDONLY)
|
||||||
|
try:
|
||||||
|
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
|
||||||
|
n += 1
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
print('evicted', n, 'files')
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
F10 (FORK.md) drops the cache of every `-lm none` load by itself (verified in production on a 35B
|
||||||
|
GPU load: node page cache stayed at 4.2-5.3 GB instead of growing by about 20 GB), but files cached
|
||||||
|
by mmap loads, benchmarks or older builds still need this step. `/sys/fs/cgroup/memory.stat`
|
||||||
|
inside the container is the HOST root cgroup; the pod's own cgroup is under
|
||||||
|
`/sys/fs/cgroup/kubepods/besteffort/pod<uid>/`.
|
||||||
|
4. **Stop llama-swap and reap orphans.**
|
||||||
|
```sh
|
||||||
|
supervisorctl -c /root/supervisord.conf stop llama-swap
|
||||||
|
ps -eo pid,ppid,etimes,args | grep '[l]lama-server'
|
||||||
|
kill -TERM <pid> <pid> # explicit PIDs only, never pkill -f
|
||||||
|
ps -eo pid,args | grep '[l]lama-server' # must be empty
|
||||||
|
```
|
||||||
|
`supervisorctl stop llama-swap` has left its llama-server children running as orphans (ppid 1),
|
||||||
|
one still generating for a client for a minute. They exit within about 1-2 s of a SIGTERM. Keep
|
||||||
|
this check even when it finds nothing (2026-10-25: no orphans).
|
||||||
|
5. **Check memory**, now that the running models are gone: `grep -E 'MemAvailable|MemFree'
|
||||||
|
/proc/meminfo`. Proceed only with MemAvailable >= 25 GB. In 2026-10-24 the eviction took node-wide
|
||||||
|
file cache from 15.6 to 4.9 GB and MemAvailable to 45.8 GB.
|
||||||
|
6. **Build in place**, now that nothing runs from `/root/llama.cpp/build/bin` (llama-swap's
|
||||||
|
`LD_LIBRARY_PATH` points there, so do not rebuild it under running servers):
|
||||||
|
```sh
|
||||||
|
cd /root/llama.cpp && cmake --build build --target llama-server llama-bench test-backend-ops -j 12
|
||||||
|
ls -lL build/bin/llama-server build/bin/libllama-server-impl.so build/bin/libllama-common.so \
|
||||||
|
build/bin/libllama.so build/bin/libmtmd.so build/bin/libggml.so build/bin/libggml-base.so \
|
||||||
|
build/bin/libggml-cpu.so build/bin/libggml-vulkan.so
|
||||||
|
```
|
||||||
|
All must have fresh mtimes. Use `-L` (follow the symlinks): `build/bin` keeps every old versioned
|
||||||
|
`libggml-*.so.0.x` and `libmtmd.so.0.x`, so a plain `ls` lists stale files beside the new ones.
|
||||||
|
`llama-server` is a 17 KB stub. The server code is in `libllama-server-impl.so`, the common code
|
||||||
|
(arguments, sampling) in `libllama-common.so`; a server change shows only in their mtimes.
|
||||||
|
7. **Start and probe.**
|
||||||
|
```sh
|
||||||
|
supervisorctl -c /root/supervisord.conf start llama-swap
|
||||||
|
curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/v1/models # 200
|
||||||
|
```
|
||||||
|
Sample `/proc/meminfo` every few seconds during the first model loads. Confirm the running commit
|
||||||
|
from the first chat response (section 8): `system_fingerprint` is `b<build>-<short sha>`
|
||||||
|
(2026-10-25: `b11906-2fc9f9c35`). The first 35B request after that deploy took 34 s including
|
||||||
|
the model load and the answer.
|
||||||
|
8. **Disk prompt cache** (FORK.md F11), only once `--cache-disk-path` is in `/root/config.yaml`:
|
||||||
|
- Every deploy empties the cache: the key holds the build commit, so the previous build's files are
|
||||||
|
never restored. They stay in the directory as "other files" until the size limit evicts them.
|
||||||
|
To free the space at once, `rm /root/prompt-cache/*.lpcd` while llama-swap is stopped (step 4).
|
||||||
|
- Check `/root` free space (`df -h /root`; 11-13 GB on 2026-10-25). The cap (`--cache-disk-size`)
|
||||||
|
plus a 1 GiB floor bounds the cache, but builds and model downloads share the PVC.
|
||||||
|
- Stopping llama-swap takes longer: each large model writes its slot state on SIGTERM, about 1 s
|
||||||
|
per GB of state, before it exits (FORK.md F7, F11).
|
||||||
|
|
||||||
|
## 8. Post-deploy checks
|
||||||
|
|
||||||
|
All through llama-swap on :8080. A request helper:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
ask() { curl -s -m 300 http://127.0.0.1:8080/v1/chat/completions -H 'Content-Type: application/json' \
|
||||||
|
-d "{\"model\":\"$1\",\"messages\":[{\"role\":\"user\",\"content\":\"$2\"}],\"max_tokens\":32,\"temperature\":0}"; echo; }
|
||||||
|
holder() { ino=$(stat -c %i /dev/shm/llama-vram/token.lock); grep "FLOCK.* WRITE .*:$ino " /proc/locks; }
|
||||||
|
# image request body: a 64x64 solid red binary PPM, no image tools needed (like fork-validate.sh)
|
||||||
|
img() { python3 - "$1" <<'EOF'
|
||||||
|
import base64, json, sys
|
||||||
|
ppm = b'P6\n64 64\n255\n' + b'\xff\x00\x00' * (64 * 64)
|
||||||
|
url = 'data:image/x-portable-pixmap;base64,' + base64.b64encode(ppm).decode()
|
||||||
|
print(json.dumps({'model': sys.argv[1], 'max_tokens': 32, 'temperature': 0, 'messages': [{'role': 'user',
|
||||||
|
'content': [{'type': 'image_url', 'image_url': {'url': url}},
|
||||||
|
{'type': 'text', 'text': 'What is the main colour of this image? One word.'}]}]}))
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
# img MODEL | curl -s -m 300 http://127.0.0.1:8080/v1/chat/completions -H 'Content-Type: application/json' -d @-
|
||||||
|
```
|
||||||
|
|
||||||
|
Use the `-nothink` aliases with these `max_tokens: 32` helpers. A thinking alias spends the 32 tokens
|
||||||
|
on `reasoning_content` and returns an empty `content`; for one, raise `max_tokens` to >= 512.
|
||||||
|
|
||||||
|
1. `ask 'unsloth/Qwen3.6-35B-A3B-GGUF-nothink:UD-Q4_K_XL' 'Capital of France? One word.'` answers;
|
||||||
|
`holder` shows the 35B's PID (check with `ps -o args -p <pid>`).
|
||||||
|
2. Handoffs: alternate `Qwen3.5-4B-task:Q4_K_M` and the 35B two or three times. Each answer is
|
||||||
|
correct and `holder` shows the PID of the model that just answered. Expect a few seconds per switch
|
||||||
|
(2026-10-24: 3.1-5.6 s).
|
||||||
|
3. Multimodal: one image request to `SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-nothink:UD-Q4_K_XL`
|
||||||
|
with the `img` helper; the answer names red. This also forces an eviction in the llama-swap matrix: the
|
||||||
|
evicted process must exit by itself, logging `VRAM arbiter: loop stopped, device memory freed,
|
||||||
|
releasing the GPU token`, with no kill escalation.
|
||||||
|
4. Text to gemma and to the 4B after the eviction; `holder` correct each time.
|
||||||
|
With the disk prompt cache on (F11): send the 35B a chat of at least 256 tokens before step 3.
|
||||||
|
Its eviction must log `disk cache: saved the prompt of N tokens`. Then send the same chat plus one
|
||||||
|
new message to the 35B again: the log shows `disk cache: restored`, and `timings.prompt_n` is
|
||||||
|
only the new suffix.
|
||||||
|
5. Ports: the children bind the `--port ${PORT}` values llama-swap passes (not upstream's default 9931).
|
||||||
|
6. Log scan of the new llama-swap log lines (`curl -s http://127.0.0.1:8080/logs`) for
|
||||||
|
`GGML_ASSERT|Assertion|terminate called|Segmentation fault|ErrorDeviceLost|ErrorOutOfDeviceMemory|DEVICE_LOST`: none.
|
||||||
|
7. `dmesg` clean (no `ring ... timeout`, no `GPU_HUNG`). No pod restart and no OOM kill:
|
||||||
|
- Restarts, from the workstation: `kubectl get pods -n llama | grep supervisord`; RESTARTS of the
|
||||||
|
Running pod must equal the value you noted before the deploy. A stale `supervisord-...-pbmjq`
|
||||||
|
in state `Unknown` is also listed; it is an old leftover, ignore it.
|
||||||
|
- OOM kills, in the pod: `dmesg | grep -E 'OOM controller|Sending SIGKILL to cgroup'`. The
|
||||||
|
timestamps are seconds since boot; compare them with the first field of `/proc/uptime` to see
|
||||||
|
whether any falls in the deploy window (2026-10-25: the newest was about 19 h old).
|
||||||
|
|
||||||
|
A wedged arbiter looks like this: requests hang while `/health` still returns 200, and `/proc/locks`
|
||||||
|
shows a process holding WRITE on `token.lock` that is not generating. llama-swap restarts a child
|
||||||
|
that exits (on the next request for its model, FORK.md F7) but never one that wedges. Inspect it with
|
||||||
|
`gdb -p <pid> -batch -ex 'thread apply all bt'` before killing it. gdb is not in
|
||||||
|
the pod image: it was installed by hand and was gone after the 2026-10-24 pod restart, so check
|
||||||
|
`command -v gdb` and reinstall it first if needed.
|
||||||
|
|
||||||
|
## 9. Rollback
|
||||||
|
|
||||||
|
```sh
|
||||||
|
supervisorctl -c /root/supervisord.conf stop llama-swap
|
||||||
|
ps -eo pid,args | grep '[l]lama-server' # SIGTERM any orphan by PID
|
||||||
|
cd /root/llama.cpp
|
||||||
|
git checkout -B <previous-production-branch> "$(head -1 /root/DEPLOY-ROLLBACK-commit-N.txt | cut -d' ' -f1)"
|
||||||
|
cmake --build build --target llama-server llama-bench -j 12
|
||||||
|
supervisorctl -c /root/supervisord.conf start llama-swap
|
||||||
|
curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/v1/models
|
||||||
|
```
|
||||||
|
|
||||||
|
Then run the post-deploy checks again. Roll back on any failed check; report rather than debug in
|
||||||
|
production. Keep the candidate worktree for a later redeploy.
|
||||||
|
|
||||||
|
If you run the rollback (or the deploy steps 4-7) from a workstation through `kubectl exec`, put the
|
||||||
|
whole sequence in one script on the pod and start it as `setsid nohup sh /root/arms/rollback.sh >
|
||||||
|
/root/arms/rollback.log 2>&1 </dev/null &`. Do not background a `cat > X && sh -n X && setsid ...`
|
||||||
|
chain in the exec shell: it can die with the exec session halfway, leaving llama-swap stopped (section
|
||||||
|
5). A rollback that also undoes a config change (for example removing `--cache-disk-path` from
|
||||||
|
`/root/config.yaml`, F11) needs the config restored first, from its `/root/config.yaml.bak-*`.
|
||||||
|
|
||||||
|
Pointer files 2 to 10 exist; for example commit-8 holds `64543a435`, commit-9 `5751c977b`, and
|
||||||
|
commit-10 `6c480f7e7`, the rollback target for the current production `2fc9f9c35`. The formats vary
|
||||||
|
(commit-4 to 6 hold short SHAs, commit-6 has branch and date after the SHA, commit-9 a subject on
|
||||||
|
line 2), hence `head -1 | cut -d' ' -f1`. New pointers: the full SHA alone on line 1.
|
||||||
|
|
||||||
|
## 10. Recovering a hung GPU
|
||||||
|
|
||||||
|
Symptoms: a process that survives SIGKILL (stuck in the driver); `ring sdma1 timeout` or
|
||||||
|
`ring comp_... timeout` in the kernel log; after a reboot the supervisord pod sits in FailedMount with
|
||||||
|
`/dev/kfd is not a character device` / `/dev/dri is not a directory`, and
|
||||||
|
`talosctl get pcidevices` lists no VGA device (the RX 580 and its HDMI audio function are missing).
|
||||||
|
A warm reboot does not bring the card back (classic Polaris failed reset).
|
||||||
|
|
||||||
|
The node (`anapistula-delrosalae`, Talos, bare metal) has a NanoKVM at http://192.168.255.8.
|
||||||
|
Credentials: ask the owner. Procedure (owner's instructions, follow exactly):
|
||||||
|
|
||||||
|
1. Soft shutdown: **one short press** of the power button through the NanoKVM.
|
||||||
|
2. **Never use the reset button.** It does not cut GPU power and is useless here.
|
||||||
|
3. Wait until the power LED goes off.
|
||||||
|
4. Press the power button once more to boot.
|
||||||
|
5. Only if the LED is still on 10+ minutes after the short press: one **long press of 8 seconds** to
|
||||||
|
force power off, wait for the LED to go off, then a short press to boot. With the GPU off the bus the
|
||||||
|
soft shutdown has hung before (Talos services stopped, LED stayed on), and the long press was needed.
|
||||||
|
|
||||||
|
Talos access from the workstation:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export TALOSCONFIG=/home/user/Projects/klaster/talos/generated/talosconfig
|
||||||
|
nix run nixpkgs#talosctl -- -n anapistula-delrosalae get pcidevices
|
||||||
|
nix run nixpkgs#talosctl -- -n anapistula-delrosalae dmesg
|
||||||
|
```
|
||||||
|
|
||||||
|
Afterwards: pod Running, RX 580 listed, llama-swap RUNNING, `/v1/models` 200, and the 120 W cap active:
|
||||||
|
`cat /sys/class/drm/card*/device/hwmon/hwmon*/power1_cap` reads `120000000` (the
|
||||||
|
`power1_cap ... -> 120000000` line in `/tmp/fan-control.log` may be missing after a pod restart). The two 2026-10-09 hangs both came at the
|
||||||
|
first 4096-token prefill right after a decode phase, before the cap was fixed at 120 W; both a fork
|
||||||
|
and a clean-upstream build hung, so they were treated as hardware/power, not code.
|
||||||
|
|
||||||
|
## 11. Operations reference
|
||||||
|
|
||||||
|
Pod access:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export KUBECONFIG=/home/user/Projects/klaster/talos/generated/kubeconfig
|
||||||
|
kubectl -n llama exec -i deploy/supervisord -- sh -c '<command>'
|
||||||
|
kubectl get pods -n llama -l app=llama-swap -o name # the pod name changes across restarts
|
||||||
|
```
|
||||||
|
|
||||||
|
Wiring: k8s service `llama:11434` -> pod port 8080 (selector `app=llama-swap`) -> llama-swap `:8080`;
|
||||||
|
a Caddy proxy fronts `llama.lumpiasty.xyz`. llama-swap must listen on `:8080`.
|
||||||
|
|
||||||
|
Files (all under `/root`, a persistent volume):
|
||||||
|
- `/root/llama.cpp`: production tree, built in `build/` (shared libraries, so the real code is in
|
||||||
|
`build/bin/libllama.so` and `libggml-*.so`). Scratch worktrees go in `/root/arms/`.
|
||||||
|
- `/root/supervisord.conf`: supervisord (PID 1). `[program:llama-swap]` runs
|
||||||
|
`/root/llama-swap -config /root/config.yaml -listen :8080 -watch-config`; its `environment=` sets
|
||||||
|
`PATH` and `LD_LIBRARY_PATH` to `/root/llama.cpp/build/bin` and the arbiter variables (FORK.md section 5).
|
||||||
|
- `/root/config.yaml`: llama-swap models, macros (`moe_offload_args`, `moe_offload_gemma`,
|
||||||
|
`common_args`), matrix/exclusivity sets, preload. Backups `/root/config.yaml.bak-*`. Edit only with
|
||||||
|
the owner's approval.
|
||||||
|
- `/root/fan-control.sh`: fan curve and the fixed 120 W cap (separate supervisord program).
|
||||||
|
- `/root/bench/`: benchmark results and pod-side helpers (`/root/bench/moe/swapdown.sh`).
|
||||||
|
- `/root/DEPLOY-ROLLBACK-commit-N.txt`: rollback pointers.
|
||||||
|
- `/root/prompt-cache/` (recommended, not yet configured): disk prompt cache files `*.lpcd` of every
|
||||||
|
model with `--cache-disk-path` (FORK.md F11). Safe to empty while llama-swap is stopped.
|
||||||
|
- `/dev/shm/llama-vram/`: production arbiter arena (`token.lock`, `want.lock`; an inert `doorbell/`
|
||||||
|
directory from the old design may remain). Never point a test at it.
|
||||||
|
|
||||||
|
Reloads: a `config.yaml` change is picked up by `-watch-config`, but a running model keeps its old
|
||||||
|
arguments until it restarts (`supervisorctl -c /root/supervisord.conf restart llama-swap`). An
|
||||||
|
`environment=` change needs `supervisorctl -c /root/supervisord.conf reread && supervisorctl -c
|
||||||
|
/root/supervisord.conf update`.
|
||||||
|
|
||||||
|
Useful commands:
|
||||||
|
- VRAM in use: `cat /sys/class/drm/card*/device/mem_info_vram_used`.
|
||||||
|
- Token holder and waiters: `/proc/locks` filtered by the inode of `token.lock` / `want.lock`
|
||||||
|
(WRITE = holder on token.lock, READ = registered waiters on want.lock).
|
||||||
|
- 120 W cap active: `cat /sys/class/drm/card*/device/hwmon/hwmon*/power1_cap` = `120000000`.
|
||||||
|
- The pod has `curl`, `python3`, `flock`; no `jq`, and no `gdb` unless reinstalled by hand since the
|
||||||
|
last pod restart (`fork-validate.sh` then runs its SIGTERM-in-release case by polling the log).
|
||||||
|
|
||||||
|
Second test box (optional): `ssh user@gaming-pc`, the owner's NixOS desktop with an RX 6600 (RDNA2).
|
||||||
|
Pod-built binaries can run there through the host glibc loader without compiling. Numbers from it do
|
||||||
|
not transfer: about 2.25x the RX 580's prefill, 13% slower decode, and the GCN-only mask_opt change does
|
||||||
|
not apply. Do not change its system config.
|
||||||
|
|
||||||
|
Not a fork problem: OpenWebUI's stop button does not cancel generation (its Redis task-command listener
|
||||||
|
never handles the stop); llama-server and llama-swap both cancel correctly on client disconnect.
|
||||||
@@ -106,6 +106,11 @@ extern "C" {
|
|||||||
GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
||||||
GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
||||||
|
|
||||||
|
// Free transient/scratch device memory the backend holds outside of any allocated buffer
|
||||||
|
// (compute preallocations, staging buffers). No-op if the backend does not implement it.
|
||||||
|
// The backend remains usable; scratch is reallocated lazily on the next compute.
|
||||||
|
GGML_API void ggml_backend_free_scratch(ggml_backend_t backend);
|
||||||
|
|
||||||
// NOTE: will be removed, use device version instead
|
// NOTE: will be removed, use device version instead
|
||||||
GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
||||||
GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
|
GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
|
||||||
|
|||||||
@@ -154,6 +154,11 @@ extern "C" {
|
|||||||
|
|
||||||
// (optional) sort/optimize the nodes in the graph
|
// (optional) sort/optimize the nodes in the graph
|
||||||
void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params);
|
void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params);
|
||||||
|
|
||||||
|
// (optional) free transient/scratch device memory the backend holds outside of any buffer
|
||||||
|
// (e.g. compute preallocations and staging buffers). The backend stays usable; the scratch
|
||||||
|
// is reallocated lazily on the next compute. Used to shrink an idle model's device footprint.
|
||||||
|
void (*free_scratch) (ggml_backend_t backend);
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ggml_backend {
|
struct ggml_backend {
|
||||||
|
|||||||
@@ -571,6 +571,15 @@ void ggml_backend_synchronize(ggml_backend_t backend) {
|
|||||||
backend->iface.synchronize(backend);
|
backend->iface.synchronize(backend);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ggml_backend_free_scratch(ggml_backend_t backend) {
|
||||||
|
GGML_ASSERT(backend);
|
||||||
|
if (backend->iface.free_scratch == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
backend->iface.free_scratch(backend);
|
||||||
|
}
|
||||||
|
|
||||||
ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
|
ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
|
||||||
GGML_ASSERT(backend);
|
GGML_ASSERT(backend);
|
||||||
GGML_ASSERT(backend->iface.graph_plan_create != NULL);
|
GGML_ASSERT(backend->iface.graph_plan_create != NULL);
|
||||||
|
|||||||
@@ -1247,6 +1247,10 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device,
|
|||||||
|
|
||||||
result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4);
|
result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4);
|
||||||
|
|
||||||
|
// Staging K/V through shared memory is a loss on AMD GCN: measured -6.7% at depth 16k and
|
||||||
|
// -7.4% at 32k on Polaris (head size 128). The kvsh stride of D/4+1 dwords is 4 mod 32, which
|
||||||
|
// costs an 8-way LDS bank conflict on wave64, and the extra shared memory cuts into occupancy
|
||||||
|
// that this shader is already short of. Keep it to NVIDIA.
|
||||||
result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0;
|
result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0;
|
||||||
|
|
||||||
if (!reduce_block_rows && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
if (!reduce_block_rows && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
||||||
@@ -8278,8 +8282,10 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Only use mask opt when the mask is fairly large. This hasn't been tuned extensively.
|
// Only use mask opt when the mask is fairly large. This hasn't been tuned extensively.
|
||||||
|
// GCN with large head size (>= 256) benefits in high-context prefill (skipping the
|
||||||
|
// per-block mask add on fully-visible blocks dominates), so enable it there too.
|
||||||
bool use_mask_opt = mask && !use_sparse && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16
|
bool use_mask_opt = mask && !use_sparse && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16
|
||||||
&& (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256);
|
&& (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK >= 256 || HSV >= 256);
|
||||||
vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc,
|
vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc,
|
||||||
mask != nullptr, use_mask_opt, logit_softcap != 0, use_sparse, k_type_eff, v_type_eff);
|
mask != nullptr, use_mask_opt, logit_softcap != 0, use_sparse, k_type_eff, v_type_eff);
|
||||||
|
|
||||||
@@ -10531,9 +10537,120 @@ void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, con
|
|||||||
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_OPT_STEP_SGD, { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f });
|
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_OPT_STEP_SGD, { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fast path for concat along dim 0 where one source is stored "transposed"
|
||||||
|
// (nb[1] == type_size, dim1 innermost) and the other source + dst are
|
||||||
|
// contiguous along dim 0. The generic concat shader reads the transposed
|
||||||
|
// source with a catastrophic uncoalesced stride; here we instead copy the
|
||||||
|
// contiguous source with copy.comp and transpose the other source into the
|
||||||
|
// matching dst sub-region with the tiled copy_transpose shader (shared-memory
|
||||||
|
// transpose, coalesced read+write). Mirrors the precedent in
|
||||||
|
// ggml_vk_cpy_to_contiguous: direct dispatch with custom push constants.
|
||||||
|
static void ggml_vk_concat_transpose_fastpath(ggml_backend_vk_context * ctx, vk_context& subctx,
|
||||||
|
const ggml_tensor * ctg, const ggml_tensor * trp,
|
||||||
|
ggml_tensor * dst, uint32_t off_ctg, uint32_t off_trp) {
|
||||||
|
const uint32_t ts = ggml_type_size(dst->type);
|
||||||
|
|
||||||
|
vk_pipeline pipeline_cpy = (ts == 4) ? ctx->device->pipeline_cpy_32
|
||||||
|
: ctx->device->pipeline_cpy_16;
|
||||||
|
vk_pipeline pipeline_trp = (ts == 4) ? ctx->device->pipeline_cpy_transpose_32
|
||||||
|
: ctx->device->pipeline_cpy_transpose_16;
|
||||||
|
|
||||||
|
ggml_pipeline_request_descriptor_sets(ctx, pipeline_cpy, 1);
|
||||||
|
ggml_pipeline_request_descriptor_sets(ctx, pipeline_trp, 1);
|
||||||
|
|
||||||
|
vk_subbuffer ctg_buf = ggml_vk_tensor_subbuffer(ctx, ctg, true);
|
||||||
|
vk_subbuffer trp_buf = ggml_vk_tensor_subbuffer(ctx, trp, true);
|
||||||
|
vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true);
|
||||||
|
|
||||||
|
const uint32_t a_misalign_ctg = get_misalign_bytes(ctx, ctg) / ts;
|
||||||
|
const uint32_t a_misalign_trp = get_misalign_bytes(ctx, trp) / ts;
|
||||||
|
const uint32_t d_misalign = get_misalign_bytes(ctx, dst) / ts;
|
||||||
|
|
||||||
|
// Dispatch A: contiguous copy of `ctg` into dst[off_ctg : off_ctg + ctg->ne[0], :]
|
||||||
|
if (ctg->ne[0] > 0) {
|
||||||
|
const uint32_t ne_ctg = (uint32_t) ggml_nelements(ctg);
|
||||||
|
vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(ctg, dst, ne_ctg);
|
||||||
|
pc.ne10 = (uint32_t) ctg->ne[0]; // only ctg's columns in dst
|
||||||
|
pc.misalign_offsets = (a_misalign_ctg << 16) | (d_misalign + off_ctg);
|
||||||
|
init_pushconst_fastdiv(pc);
|
||||||
|
std::array<uint32_t, 3> el = ne_ctg > 262144 ? std::array<uint32_t,3>{512, 512, CEIL_DIV(ne_ctg, 262144)}
|
||||||
|
: ne_ctg > 512 ? std::array<uint32_t,3>{512, CEIL_DIV(ne_ctg, 512), 1}
|
||||||
|
: std::array<uint32_t,3>{ne_ctg, 1, 1};
|
||||||
|
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline_cpy, { ctg_buf, dst_buf }, pc, el);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dispatch B: tiled transpose of `trp` into dst[off_trp : off_trp + trp->ne[0], :]
|
||||||
|
if (trp->ne[0] > 0) {
|
||||||
|
vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(trp, dst, ggml_nelements(trp));
|
||||||
|
pc.ne10 = (uint32_t) trp->ne[0]; // dst bound = trp columns (NOT dst->ne[0])
|
||||||
|
pc.misalign_offsets = (a_misalign_trp << 16) | (d_misalign + off_trp);
|
||||||
|
init_pushconst_fastdiv(pc);
|
||||||
|
std::array<uint32_t, 3> el = {
|
||||||
|
(uint32_t) CEIL_DIV(trp->ne[0], 32),
|
||||||
|
(uint32_t) CEIL_DIV(trp->ne[1], 32),
|
||||||
|
(uint32_t) (trp->ne[2] * trp->ne[3]),
|
||||||
|
};
|
||||||
|
el[0] = std::min(el[0], (uint32_t) ctx->device->properties.limits.maxComputeWorkGroupCount[0]);
|
||||||
|
el[1] = std::min(el[1], (uint32_t) ctx->device->properties.limits.maxComputeWorkGroupCount[1]);
|
||||||
|
el[2] = std::min(el[2], (uint32_t) ctx->device->properties.limits.maxComputeWorkGroupCount[2]);
|
||||||
|
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline_trp, { trp_buf, dst_buf }, pc, el);
|
||||||
|
}
|
||||||
|
|
||||||
|
ggml_vk_sync_buffers(ctx, subctx);
|
||||||
|
}
|
||||||
|
|
||||||
void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||||
int * op_params = (int *)dst->op_params;
|
int * op_params = (int *)dst->op_params;
|
||||||
|
|
||||||
|
// Fast path: concat along dim 0 with one source "transposed" (nb[1]==type_size,
|
||||||
|
// dim1 innermost) and the other source + dst contiguous along dim 0. The generic
|
||||||
|
// shader reads the transposed source uncoalesced (~5.6ms on RX 580 vs ~130us for
|
||||||
|
// a coalesced copy); here we instead use a tiled shared-memory transpose.
|
||||||
|
auto src_transposed_2d = [&](const ggml_tensor * s) {
|
||||||
|
const uint32_t ts = ggml_type_size(s->type);
|
||||||
|
return s->nb[1] == ts // dim1 innermost
|
||||||
|
&& s->nb[0] == (size_t) s->ne[1] * ts // consistent 2D transpose
|
||||||
|
&& s->ne[0] >= 32 // at least one full transpose tile (TILE_DIM); at 1 token the
|
||||||
|
// tiled path is pure overhead, the generic copy is faster
|
||||||
|
&& s->ne[2] == 1 && s->ne[3] == 1;
|
||||||
|
};
|
||||||
|
const uint32_t dst_ts = ggml_type_size(dst->type);
|
||||||
|
const bool dim0_ok = (op_params[0] == 0) && (dst->nb[0] == dst_ts);
|
||||||
|
// the dispatches below are raw f32/f16 element copies, so quantized and integer
|
||||||
|
// types stay on the generic path
|
||||||
|
const bool types_ok = (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)
|
||||||
|
&& (src0->type == src1->type && src0->type == dst->type);
|
||||||
|
const bool shapes_ok = (src0->ne[1] == src1->ne[1] && src1->ne[1] == dst->ne[1])
|
||||||
|
&& (src0->ne[2] == src1->ne[2] && src0->ne[2] == dst->ne[2])
|
||||||
|
&& (src0->ne[3] == src1->ne[3] && src0->ne[3] == dst->ne[3])
|
||||||
|
&& (src0->ne[0] + src1->ne[0] == dst->ne[0]);
|
||||||
|
// doffset is 16-bit for unary shaders; guard against overflow
|
||||||
|
const bool off_fits = ((get_misalign_bytes(ctx, dst)/dst_ts + dst->ne[0])) < 0xFFFFu;
|
||||||
|
|
||||||
|
const bool s1_trans = dim0_ok && types_ok && shapes_ok && off_fits
|
||||||
|
&& src_transposed_2d(src1) && ggml_is_contiguous(src0);
|
||||||
|
const bool s0_trans = dim0_ok && types_ok && shapes_ok && off_fits
|
||||||
|
&& src_transposed_2d(src0) && ggml_is_contiguous(src1);
|
||||||
|
|
||||||
|
// validation/A-B knob: GGML_VK_CONCAT_FASTPATH=0 forces the generic path, =log reports every
|
||||||
|
// fast-path dispatch on stderr
|
||||||
|
static const char * fastpath_env = getenv("GGML_VK_CONCAT_FASTPATH");
|
||||||
|
static const bool fastpath_off = fastpath_env != nullptr && std::string(fastpath_env) == "0";
|
||||||
|
static const bool fastpath_log = fastpath_env != nullptr && std::string(fastpath_env) == "log";
|
||||||
|
|
||||||
|
if (!fastpath_off && (s1_trans || s0_trans)) {
|
||||||
|
const ggml_tensor * ctg = s1_trans ? src0 : src1; // contiguous source
|
||||||
|
const ggml_tensor * trp = s1_trans ? src1 : src0; // transposed source
|
||||||
|
if (fastpath_log) {
|
||||||
|
std::cerr << "ggml_vk_concat: transpose fast path, transposed ne0=" << trp->ne[0] << std::endl;
|
||||||
|
}
|
||||||
|
// dst offsets (in elements) where each source region begins
|
||||||
|
const uint32_t off_ctg = s1_trans ? 0u : (uint32_t) src1->ne[0];
|
||||||
|
const uint32_t off_trp = s1_trans ? (uint32_t) src0->ne[0] : 0u;
|
||||||
|
ggml_vk_concat_transpose_fastpath(ctx, subctx, ctg, trp, dst, off_ctg, off_trp);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);
|
const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);
|
||||||
const uint32_t units_per_block = ggml_type_size(dst->type) / unit_size;
|
const uint32_t units_per_block = ggml_type_size(dst->type) / unit_size;
|
||||||
const uint32_t block_size = ggml_blck_size(dst->type);
|
const uint32_t block_size = ggml_blck_size(dst->type);
|
||||||
@@ -13285,6 +13402,35 @@ static const char * ggml_backend_vk_name(ggml_backend_t backend) {
|
|||||||
return ctx->name.c_str();
|
return ctx->name.c_str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Free the compute-scratch device buffers (prealloc_* and the transfer staging buffer) without
|
||||||
|
// tearing down the backend (pipelines, command pools, fences and device stay alive). These buffers
|
||||||
|
// are reallocated lazily by ggml_vk_preallocate_buffers() on the next compute, so this just shrinks
|
||||||
|
// an idle model's device footprint. Mirrors the buffer-freeing subset of ggml_vk_cleanup().
|
||||||
|
static void ggml_backend_vk_free_scratch(ggml_backend_t backend) {
|
||||||
|
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
|
||||||
|
VK_LOG_DEBUG("ggml_backend_vk_free_scratch(" << ctx->name << ")");
|
||||||
|
|
||||||
|
// discard any unsubmitted command buffer and wait for in-flight work before freeing
|
||||||
|
ctx->compute_ctx.reset();
|
||||||
|
ggml_vk_synchronize(ctx);
|
||||||
|
|
||||||
|
ggml_vk_destroy_buffer(ctx->prealloc_x);
|
||||||
|
ggml_vk_destroy_buffer(ctx->prealloc_y);
|
||||||
|
ggml_vk_destroy_buffer(ctx->prealloc_split_k);
|
||||||
|
ggml_vk_destroy_buffer(ctx->prealloc_add_rms_partials);
|
||||||
|
ggml_vk_destroy_buffer(ctx->sync_staging);
|
||||||
|
|
||||||
|
ctx->prealloc_y_last_pipeline_used = nullptr;
|
||||||
|
ctx->prealloc_y_last_tensor_used = nullptr;
|
||||||
|
ctx->prealloc_y_last_k_padded = false;
|
||||||
|
|
||||||
|
ctx->prealloc_size_x = 0;
|
||||||
|
ctx->prealloc_size_y = 0;
|
||||||
|
ctx->prealloc_size_split_k = 0;
|
||||||
|
ctx->prealloc_size_add_rms_partials = 0;
|
||||||
|
ctx->prealloc_size_add_rms_partials_offset = 0;
|
||||||
|
}
|
||||||
|
|
||||||
void ggml_backend_vk_free(ggml_backend_t backend) {
|
void ggml_backend_vk_free(ggml_backend_t backend) {
|
||||||
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
|
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
|
||||||
VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
|
VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
|
||||||
@@ -15276,6 +15422,7 @@ static ggml_backend_i ggml_backend_vk_interface = {
|
|||||||
/* .event_record = */ ggml_backend_vk_event_record,
|
/* .event_record = */ ggml_backend_vk_event_record,
|
||||||
/* .event_wait = */ ggml_backend_vk_event_wait,
|
/* .event_wait = */ ggml_backend_vk_event_wait,
|
||||||
/* .graph_optimize = */ ggml_vk_graph_optimize,
|
/* .graph_optimize = */ ggml_vk_graph_optimize,
|
||||||
|
/* .free_scratch = */ ggml_backend_vk_free_scratch,
|
||||||
};
|
};
|
||||||
|
|
||||||
static ggml_guid_t ggml_backend_vk_guid() {
|
static ggml_guid_t ggml_backend_vk_guid() {
|
||||||
|
|||||||
@@ -591,6 +591,23 @@ extern "C" {
|
|||||||
|
|
||||||
LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);
|
LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);
|
||||||
LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx);
|
LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx);
|
||||||
|
|
||||||
|
// On-demand device (VRAM) residency: free the model's GPU weight buffers (keeping a host
|
||||||
|
// shadow so no reload is needed) to hand VRAM to another model, and rebuild them on demand.
|
||||||
|
// Any subsequent llama_decode auto-restores. If evict_kv is true, the KV cache device buffers
|
||||||
|
// are also evicted to a host shadow (for when the KV would not fit alongside the other model),
|
||||||
|
// at the cost of a D2H/H2D copy of the live KV each cycle; otherwise the KV stays resident.
|
||||||
|
// Intended for time-sharing a single GPU between several always-loaded models.
|
||||||
|
LLAMA_API void llama_context_release_device(struct llama_context * ctx, bool evict_kv);
|
||||||
|
LLAMA_API void llama_context_restore_device(struct llama_context * ctx);
|
||||||
|
// Free all device buffers (weights, KV, compute) for good, without host copies, e.g. to hand
|
||||||
|
// the VRAM to another process before exit. The context must not be used for compute afterwards.
|
||||||
|
LLAMA_API void llama_context_discard_device(struct llama_context * ctx);
|
||||||
|
// Before exit, after llama_context_release_device(ctx, true): point the evicted KV/recurrent state
|
||||||
|
// at its host copy (no device use, no extra copy) so llama_state_seq_get_data can read it. The
|
||||||
|
// context must not be used for compute afterwards.
|
||||||
|
LLAMA_API bool llama_context_restore_memory_host(struct llama_context * ctx);
|
||||||
|
LLAMA_API bool llama_context_weights_resident(const struct llama_context * ctx);
|
||||||
LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
|
LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
|
||||||
|
|
||||||
LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);
|
LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,247 @@
|
|||||||
|
# rx580-bench
|
||||||
|
|
||||||
|
Prompt-processing benchmark harness for the RX 580 (Polaris) serving pod.
|
||||||
|
|
||||||
|
It exists so that no future session has to rediscover how to measure this
|
||||||
|
machine. It encodes the production config, fixed corpus slices, the repeat
|
||||||
|
and median discipline, and the noise floor.
|
||||||
|
|
||||||
|
## What it measures
|
||||||
|
|
||||||
|
Two things, deliberately:
|
||||||
|
|
||||||
|
1. **Corpus prompt processing** (the number we actually care about). Real
|
||||||
|
Polish prose from `pan-tadeusz.txt` pushed through `llama-server`'s
|
||||||
|
`/completion` endpoint with `n_predict: 1`, `cache_prompt: false`. This
|
||||||
|
mimics agentic tool-result parsing: a big blob of real text arriving cold.
|
||||||
|
Reported as `timings.prompt_per_second` at prompt lengths of about 4k, 16k
|
||||||
|
and 32k tokens.
|
||||||
|
2. **`llama-bench` sweep** (the controlled cross-check). Synthetic random
|
||||||
|
tokens: prefill `pp2048`, `pp8192` at depth 0, and decode `tg256@d4096`
|
||||||
|
(256 generated tokens on top of a 4096-token context, `-p 0 -n 256
|
||||||
|
-d 4096`). The prefill cells are the continuity metric with prior work on
|
||||||
|
this box. Decode used to be `tg32` at the default depth; that is not a valid
|
||||||
|
metric here (see "Reading the numbers"), and old `tg32` rows in
|
||||||
|
`results.txt` are history only.
|
||||||
|
|
||||||
|
They do not measure the same thing and they do not agree. `llama-bench`
|
||||||
|
pp2048 is a cold depth-0 batch; the corpus numbers include the cost of a
|
||||||
|
growing KV cache, so they fall off with prompt length. Both are useful.
|
||||||
|
|
||||||
|
## How to run it
|
||||||
|
|
||||||
|
One command from the workstation:
|
||||||
|
|
||||||
|
```
|
||||||
|
./scripts/rx580-bench/run.sh --build /root/arms/new-clean/build --label new-clean
|
||||||
|
```
|
||||||
|
|
||||||
|
Interleaved A/B, which is the only valid way to compare two builds:
|
||||||
|
|
||||||
|
```
|
||||||
|
./scripts/rx580-bench/run.sh \
|
||||||
|
--build /root/arms/new-clean/build --label new-clean \
|
||||||
|
--build-b /root/arms/new-fork/build --label-b new-fork \
|
||||||
|
--rounds 3
|
||||||
|
```
|
||||||
|
|
||||||
|
Summarize whatever has accumulated:
|
||||||
|
|
||||||
|
```
|
||||||
|
./scripts/rx580-bench/run.sh --summarize
|
||||||
|
```
|
||||||
|
|
||||||
|
`run.sh` copies the harness onto the pod and execs it there. The wrapper is
|
||||||
|
thin on purpose: all the methodology lives in the pod-side scripts, so it can
|
||||||
|
also be driven directly on the pod if kubectl is inconvenient.
|
||||||
|
|
||||||
|
The run is detached on the pod and then tailed, so Ctrl-C on the wrapper does
|
||||||
|
not orphan a half-finished run. `bench.sh` restarts `llama-swap` from an EXIT
|
||||||
|
trap on success, failure and interrupt alike.
|
||||||
|
|
||||||
|
### Pod access
|
||||||
|
|
||||||
|
```
|
||||||
|
export KUBECONFIG=/home/user/Projects/klaster/talos/generated/kubeconfig
|
||||||
|
kubectl -n llama exec -i deploy/supervisord -- sh -c '<command>'
|
||||||
|
```
|
||||||
|
|
||||||
|
The pod has `curl` and `python3` but **no `jq`**; the harness uses python3 for
|
||||||
|
all JSON.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
| file | side | what |
|
||||||
|
|---|---|---|
|
||||||
|
| `run.sh` | workstation | one-command wrapper; installs and execs the rest |
|
||||||
|
| `bench.sh` | pod | orchestrator: stops llama-swap, waits for idle GPU, runs both harnesses, restarts llama-swap via trap |
|
||||||
|
| `ppbench.py` | pod | the corpus prompt-processing harness |
|
||||||
|
| `lbsweep.sh` | pod | the llama-bench sweep |
|
||||||
|
| `summarize.py` | pod | turns `results.txt` into a median/min-max table with delta significance |
|
||||||
|
|
||||||
|
Results append to `/root/bench/results.txt` as parseable `RESULT` /
|
||||||
|
`SUMMARY` / `LBRESULT` lines. Server logs land in `/root/bench/server-*.log`.
|
||||||
|
|
||||||
|
## The config it assumes
|
||||||
|
|
||||||
|
Production serving config, matching `/root/config.yaml`:
|
||||||
|
|
||||||
|
```
|
||||||
|
-t 6 -ngl 99 --n-cpu-moe 40 -b 2048 -ub 2048 -fa 1 -lm none
|
||||||
|
--ctx-size 40960 --no-warmup
|
||||||
|
```
|
||||||
|
|
||||||
|
Model:
|
||||||
|
|
||||||
|
```
|
||||||
|
/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
||||||
|
```
|
||||||
|
|
||||||
|
Qwen3.6-35B-A3B, 20.8 GiB, 40 layers, 256 experts of which 8 are used.
|
||||||
|
Machine: AMD RX 580 8 GB (Polaris/GCN, RADV, no fp16 accel), PCIe 3.0 x16,
|
||||||
|
12 CPU cores, 62 GB RAM.
|
||||||
|
|
||||||
|
The `llama-bench` equivalent of that config is:
|
||||||
|
|
||||||
|
```
|
||||||
|
-t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -p 2048,8192 -n 0 -r 3
|
||||||
|
-t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -p 0 -n 256 -d 4096 -r 3
|
||||||
|
```
|
||||||
|
|
||||||
|
`lbsweep.sh` runs these as two invocations (prefill, then decode). Override
|
||||||
|
with `LB_P` (prefill sizes), `LB_N` / `LB_D` (decode tokens / depth; `LB_N=0`
|
||||||
|
skips decode) and `LB_R` (repetitions).
|
||||||
|
|
||||||
|
## The corpus slices are constants, not recalibrated
|
||||||
|
|
||||||
|
`pan-tadeusz.txt` is 447334 chars / 482907 bytes of Polish text. Against this
|
||||||
|
model's tokenizer it runs **2.6702 chars/token**, and `/completion` reports
|
||||||
|
`prompt_n` identical to `/tokenize` (BOS offset 0).
|
||||||
|
|
||||||
|
The slices are `corpus[0:NCHARS]` and are baked into `ppbench.py`:
|
||||||
|
|
||||||
|
| target tokens | chars | utf-8 bytes |
|
||||||
|
|---|---|---|
|
||||||
|
| 4096 | 10776 | 11608 |
|
||||||
|
| 16384 | 43858 | 47175 |
|
||||||
|
| 32768 | 87165 | 93843 |
|
||||||
|
|
||||||
|
These hit the target `prompt_n` exactly. Every build is therefore measured on
|
||||||
|
byte-identical input. `ppbench.py` warns loudly if the reported `prompt_n`
|
||||||
|
ever stops matching the target, which is the signal that the table is stale.
|
||||||
|
Only recalibrate (`ppbench.py <build> <label> --calibrate`) if the model file
|
||||||
|
or the corpus changes.
|
||||||
|
|
||||||
|
## Reading the numbers
|
||||||
|
|
||||||
|
* `prompt_n` is the token count actually prefilled. It must equal the target.
|
||||||
|
* `prompt_ms` is wall-clock prefill time.
|
||||||
|
* `tps` / `prompt_per_second` is `prompt_n / prompt_ms * 1000`. Higher better.
|
||||||
|
* `pp2048` / `pp8192` are llama-bench prefill throughput at depth 0. A depth
|
||||||
|
suffix (`pp2048@d16384`) means the prompt is processed on top of a context
|
||||||
|
of that many tokens.
|
||||||
|
* `tg256@d4096` is the decode metric: 256 generated tokens at depth 4096
|
||||||
|
(`-p 0 -n 256 -d 4096`). Decode is CPU-bound with `--n-cpu-moe 40` and
|
||||||
|
swings with node CPU contention (about 60 co-tenant pods), so compare
|
||||||
|
medians across interleaved rounds and expect occasional catastrophic
|
||||||
|
samples.
|
||||||
|
* `tg32` (old rows, no depth suffix) is token generation at the DEFAULT depth
|
||||||
|
(n_ctx 32). **It is NOT a valid decode metric on this box.** At that size
|
||||||
|
decode is launch/barrier bound, not compute bound, and measures a regime
|
||||||
|
serving never reaches: it read -36% between two upstream commits that are at
|
||||||
|
parity at tg256 (the cause was the view-alias fix in
|
||||||
|
ggml_vk_graph_optimize). tg is also NOT monotonic in n_ctx here - on one
|
||||||
|
binary n_ctx 256 measured slower than both 32 and 544 - so only ever compare
|
||||||
|
same-depth cells. `summarize.py` flags any depth-less tg row.
|
||||||
|
|
||||||
|
## Noise floor: deltas under about 5 percent are unproven
|
||||||
|
|
||||||
|
This is the single most important thing in this directory.
|
||||||
|
|
||||||
|
`llama-bench`'s within-run error bars understate **cross-invocation** variance
|
||||||
|
by roughly **14x**. The same config measured `276.7 +/- 0.9` in one invocation
|
||||||
|
and `296.4 +/- 0.6` in another, 7 percent apart, with error bars that claimed
|
||||||
|
0.3 percent precision.
|
||||||
|
|
||||||
|
Therefore:
|
||||||
|
|
||||||
|
* Any comparison you want to draw a conclusion from must live **inside a
|
||||||
|
single `llama-bench` invocation**, using comma-separated sweeps.
|
||||||
|
* Two different *builds* cannot share an invocation, so run them
|
||||||
|
**interleaved A/B/A/B/A/B across at least 3 rounds** and compare medians.
|
||||||
|
That is what `--build-b` does.
|
||||||
|
* State explicitly that any cross-build delta under about 5 percent is
|
||||||
|
unproven. `summarize.py` labels them `unproven` for you.
|
||||||
|
|
||||||
|
## Traps
|
||||||
|
|
||||||
|
* **`-tb` is not a `llama-bench` flag.** It is a `llama-server` flag. Passing
|
||||||
|
it makes `llama-bench` print usage and exit silently, mid-sweep, which looks
|
||||||
|
exactly like a run that produced nothing. Always check the row count.
|
||||||
|
* **A ubatch larger than the prompt never fills.** With `-ub 2048`, prompt
|
||||||
|
lengths must be multiples of 2048 or the sweep measures nothing meaningful.
|
||||||
|
* **Nothing else may touch the GPU.** `bench.sh` stops `llama-swap` and
|
||||||
|
refuses to start if any `llama-*` process is still alive.
|
||||||
|
* **Never leave the pod not serving.** `bench.sh` restarts `llama-swap` from
|
||||||
|
an EXIT trap. If you bypass the script, restart it by hand:
|
||||||
|
`supervisorctl -c /root/supervisord.conf start llama-swap`.
|
||||||
|
* **Do not run `llama-cli` detached with closed stdin** on this box; it loops
|
||||||
|
forever. Use the server harness or `llama-bench -d` for depth timing.
|
||||||
|
* **`--mmap` / `--no-mmap` / `--mlock` and llama-bench's `-mmp` are gone**
|
||||||
|
upstream (#28334). Use `-lm none` (`--load-mode none`) on both
|
||||||
|
`llama-server` and `llama-bench`. The old flags now fail: the server refuses
|
||||||
|
to start, `llama-bench` silently produces no rows. `/root/config.yaml` must
|
||||||
|
be updated to `-lm none` as well before deploying a build past that commit.
|
||||||
|
|
||||||
|
## Optional: the MoE expert cache
|
||||||
|
|
||||||
|
Upstream #29887 added `--moe-cache-mib N` (default 0, off): a device cache of
|
||||||
|
the most recently used experts of the CPU-resident (`--n-cpu-moe`) layers. It
|
||||||
|
mostly helps decode and small batches; large prefill batches only read from it.
|
||||||
|
It is a server option only - `llama-bench` has no flag for it - so measure it
|
||||||
|
with the corpus harness, e.g.
|
||||||
|
`LLAMA_ARG_MOE_CACHE_MIB=1024 bench.sh --corpus-only --build DIR --label NAME`
|
||||||
|
(the environment reaches the spawned server). The variable applies to every
|
||||||
|
arm of the run, so an on/off comparison spans two runs and the noise floor
|
||||||
|
above applies in full. The cache takes VRAM from the same 8 GB as the KV
|
||||||
|
cache and compute buffers.
|
||||||
|
|
||||||
|
## Known-good reference values
|
||||||
|
|
||||||
|
If a fresh run disagrees with these by much more than the noise floor,
|
||||||
|
something is wrong (something else on the GPU, a thermal problem, a bad
|
||||||
|
build) before you believe you found a speedup.
|
||||||
|
|
||||||
|
Model Qwen3.6-35B-A3B-UD-Q4_K_XL, production config.
|
||||||
|
|
||||||
|
llama-bench, current production fork (20261024 rebase, `6c480f7e7`) and the
|
||||||
|
previous production fork (`5751c977b`), measured 2026-10-24 by
|
||||||
|
`fork-validate.sh --perf` (2 interleaved arms each, `-r 2` prefill, `-r 3`
|
||||||
|
decode):
|
||||||
|
|
||||||
|
| metric | about |
|
||||||
|
|---|---|
|
||||||
|
| pp2048 (depth 0) | 274-286 t/s (274 current, 286 previous; the -4% is inside the noise floor) |
|
||||||
|
| pp2048@d16384 | 201-206 t/s |
|
||||||
|
| pp2048@d32768 | 158-160 t/s |
|
||||||
|
| tg256@d4096 | 17.8 t/s |
|
||||||
|
|
||||||
|
Corpus harness, measured 2026-09-08/09 on the then-current upstream
|
||||||
|
(`f3f1a8f27`) and the fork deployed then (not re-measured since):
|
||||||
|
|
||||||
|
| metric | clean upstream | fork |
|
||||||
|
|---|---|---|
|
||||||
|
| corpus pp @ 4k | about 240 t/s | about 255 t/s |
|
||||||
|
| corpus pp @ 16k | about 206 t/s | about 225 t/s |
|
||||||
|
| corpus pp @ 32k | about 171 t/s | about 195 t/s |
|
||||||
|
| llama-bench pp8192 | about 222 t/s | about 246 t/s |
|
||||||
|
|
||||||
|
Invalid-metric history: the 2026-09 runs recorded `tg32` (7-15 t/s, swinging
|
||||||
|
by run) and older notes mention "tg32 about 19 t/s". Neither is a decode
|
||||||
|
reference; see the tg32 warning above.
|
||||||
|
|
||||||
|
Model load with `-lm none` takes about 26 s warm, longer cold.
|
||||||
|
A full 3-round A/B run of both harnesses takes roughly 100 minutes;
|
||||||
|
add about 16 minutes per extra arm per round. The tg256@d4096 decode cell
|
||||||
|
costs about 1.2 minutes per arm per round (model load, the 4096-token depth
|
||||||
|
prefill and 3 x 256 tokens), measured 2026-10-24.
|
||||||
Executable
+167
@@ -0,0 +1,167 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
#
|
||||||
|
# RX 580 prompt-processing benchmark orchestrator. Runs ON THE POD.
|
||||||
|
#
|
||||||
|
# Stops llama-swap, waits for the GPU to go idle, runs the real-corpus
|
||||||
|
# prompt-processing harness and the llama-bench sweep for every arm, appends
|
||||||
|
# parseable results, and ALWAYS restarts llama-swap on exit (success, error
|
||||||
|
# or Ctrl-C).
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# bench.sh --build DIR --label NAME
|
||||||
|
# [--build-b DIR --label-b NAME] second arm, interleaved
|
||||||
|
# [--arm DIR:LABEL ...] extra arms, repeatable (N-way)
|
||||||
|
# [--rounds N] default 3 multi-arm, 1 otherwise
|
||||||
|
# [--reps N] corpus repeats per size, default 3
|
||||||
|
# [--sizes 4096,16384,32768]
|
||||||
|
# [--corpus-only | --lb-only]
|
||||||
|
# [--results PATH] default /root/bench/results.txt
|
||||||
|
# [--keep-swap-down] do not restart llama-swap at exit
|
||||||
|
#
|
||||||
|
set -e
|
||||||
|
|
||||||
|
BENCH_DIR=/root/bench
|
||||||
|
SUPCONF=/root/supervisord.conf
|
||||||
|
RESULTS="$BENCH_DIR/results.txt"
|
||||||
|
MODEL=/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
||||||
|
|
||||||
|
ARMS="" # space separated DIR|LABEL entries
|
||||||
|
BUILD_A=""; LABEL_A=""
|
||||||
|
BUILD_B=""; LABEL_B=""
|
||||||
|
ROUNDS=""
|
||||||
|
REPS=3
|
||||||
|
SIZES=4096,16384,32768
|
||||||
|
DO_CORPUS=1
|
||||||
|
DO_LB=1
|
||||||
|
KEEP_SWAP_DOWN=0
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--build) BUILD_A="$2"; shift 2 ;;
|
||||||
|
--label) LABEL_A="$2"; shift 2 ;;
|
||||||
|
--build-b) BUILD_B="$2"; shift 2 ;;
|
||||||
|
--label-b) LABEL_B="$2"; shift 2 ;;
|
||||||
|
--arm) ARMS="$ARMS ${2%%:*}|${2#*:}"; shift 2 ;;
|
||||||
|
--rounds) ROUNDS="$2"; shift 2 ;;
|
||||||
|
--reps) REPS="$2"; shift 2 ;;
|
||||||
|
--sizes) SIZES="$2"; shift 2 ;;
|
||||||
|
--results) RESULTS="$2"; shift 2 ;;
|
||||||
|
--corpus-only) DO_LB=0; shift ;;
|
||||||
|
--lb-only) DO_CORPUS=0; shift ;;
|
||||||
|
--keep-swap-down) KEEP_SWAP_DOWN=1; shift ;;
|
||||||
|
-h|--help) sed -n '2,20p' "$0"; exit 0 ;;
|
||||||
|
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -n "$BUILD_A" ] && [ -z "$LABEL_A" ]; then
|
||||||
|
echo "error: --build needs --label" >&2; exit 2
|
||||||
|
fi
|
||||||
|
if [ -n "$BUILD_B" ] && [ -z "$LABEL_B" ]; then
|
||||||
|
echo "error: --build-b needs --label-b" >&2; exit 2
|
||||||
|
fi
|
||||||
|
# --build/--label and --build-b/--label-b are sugar for the first two arms.
|
||||||
|
if [ -n "$BUILD_A" ]; then ARMS="$BUILD_A|$LABEL_A $ARMS"; fi
|
||||||
|
if [ -n "$BUILD_B" ]; then ARMS="$ARMS $BUILD_B|$LABEL_B"; fi
|
||||||
|
ARMS=$(echo $ARMS)
|
||||||
|
if [ -z "$ARMS" ]; then
|
||||||
|
echo "error: need at least one arm (--build/--label or --arm DIR:LABEL)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
NARMS=$(echo "$ARMS" | wc -w)
|
||||||
|
if [ -z "$ROUNDS" ]; then
|
||||||
|
if [ "$NARMS" -gt 1 ]; then ROUNDS=3; else ROUNDS=1; fi
|
||||||
|
fi
|
||||||
|
for e in $ARMS; do
|
||||||
|
d=${e%%|*}
|
||||||
|
if [ ! -x "$d/bin/llama-server" ] || [ ! -x "$d/bin/llama-bench" ]; then
|
||||||
|
echo "error: $d lacks bin/llama-server or bin/llama-bench" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
cat <<'WARN'
|
||||||
|
-------------------------------------------------------------------------
|
||||||
|
RX 580 benchmark. Read this before trusting any number.
|
||||||
|
|
||||||
|
* Cross-invocation variance on this box is about 7 percent. llama-bench
|
||||||
|
within-run error bars understate it by roughly 14x. Any cross-build
|
||||||
|
delta under about 5 percent is UNPROVEN noise.
|
||||||
|
* Comparisons you care about must live inside ONE llama-bench invocation
|
||||||
|
(comma-separated sweeps), or be interleaved across at least 3 rounds.
|
||||||
|
That is what multiple arms plus --rounds does.
|
||||||
|
* Nothing else may touch the GPU while this runs. llama-swap is stopped
|
||||||
|
for the duration and restarted on exit, including on error or Ctrl-C.
|
||||||
|
* Traps: -tb is NOT a llama-bench flag. A ubatch larger than -p never
|
||||||
|
fills, so prompt lengths must be multiples of the 2048 ubatch. A bad
|
||||||
|
flag makes llama-bench print usage and exit silently mid-sweep.
|
||||||
|
-------------------------------------------------------------------------
|
||||||
|
WARN
|
||||||
|
|
||||||
|
echo "[bench] $NARMS arm(s), $ROUNDS round(s): $ARMS"
|
||||||
|
|
||||||
|
restore_swap() {
|
||||||
|
rc=$?
|
||||||
|
if [ "$KEEP_SWAP_DOWN" -eq 0 ]; then
|
||||||
|
echo ""
|
||||||
|
echo "[bench] restarting llama-swap"
|
||||||
|
supervisorctl -c "$SUPCONF" start llama-swap || true
|
||||||
|
supervisorctl -c "$SUPCONF" status llama-swap || true
|
||||||
|
else
|
||||||
|
echo "[bench] --keep-swap-down: llama-swap left STOPPED"
|
||||||
|
fi
|
||||||
|
exit $rc
|
||||||
|
}
|
||||||
|
trap restore_swap EXIT INT TERM
|
||||||
|
|
||||||
|
echo "[bench] stopping llama-swap"
|
||||||
|
supervisorctl -c "$SUPCONF" stop llama-swap || true
|
||||||
|
|
||||||
|
wait_gpu_idle() {
|
||||||
|
i=0
|
||||||
|
while [ $i -lt 60 ]; do
|
||||||
|
if ! pgrep -f 'bin/llama-server|bin/llama-bench|bin/llama-cli' >/dev/null 2>&1; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
i=$((i+1))
|
||||||
|
done
|
||||||
|
echo "error: something is still on the GPU:" >&2
|
||||||
|
pgrep -af 'bin/llama-server|bin/llama-bench|bin/llama-cli' >&2
|
||||||
|
exit 3
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "[bench] waiting for the GPU to go idle"
|
||||||
|
wait_gpu_idle
|
||||||
|
echo "[bench] GPU idle"
|
||||||
|
|
||||||
|
run_corpus() {
|
||||||
|
[ "$DO_CORPUS" -eq 1 ] || return 0
|
||||||
|
echo "===== corpus $2 ====="
|
||||||
|
wait_gpu_idle
|
||||||
|
python3 "$BENCH_DIR/ppbench.py" "$1" "$2" \
|
||||||
|
--sizes "$SIZES" --reps "$REPS" --results "$RESULTS"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_lb() {
|
||||||
|
[ "$DO_LB" -eq 1 ] || return 0
|
||||||
|
echo "===== llama-bench $2 ====="
|
||||||
|
wait_gpu_idle
|
||||||
|
RESULTS="$RESULTS" MODEL="$MODEL" sh "$BENCH_DIR/lbsweep.sh" "$1" "$2"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Interleave: every arm is measured once per round, so slow drift in the
|
||||||
|
# machine hits all arms roughly equally instead of biasing whichever ran first.
|
||||||
|
r=1
|
||||||
|
while [ "$r" -le "$ROUNDS" ]; do
|
||||||
|
echo ""
|
||||||
|
echo "########## ROUND $r / $ROUNDS ##########"
|
||||||
|
for e in $ARMS; do run_corpus "${e%%|*}" "${e#*|}"; done
|
||||||
|
for e in $ARMS; do run_lb "${e%%|*}" "${e#*|}"; done
|
||||||
|
r=$((r+1))
|
||||||
|
done
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "ALL_RUNS_DONE"
|
||||||
|
echo "[bench] raw results appended to $RESULTS"
|
||||||
|
python3 "$BENCH_DIR/summarize.py" "$RESULTS" || true
|
||||||
Executable
+95
@@ -0,0 +1,95 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
#
|
||||||
|
# llama-bench sweep, the controlled cross-check for the corpus harness.
|
||||||
|
# Runs ON THE POD. Normally invoked by bench.sh.
|
||||||
|
#
|
||||||
|
# Usage: lbsweep.sh <build_dir> <label>
|
||||||
|
# Env overrides: RESULTS, MODEL, LB_P, LB_N, LB_D, LB_R
|
||||||
|
#
|
||||||
|
# Two llama-bench invocations per call:
|
||||||
|
# prefill -p $LB_P -n 0 at depth 0 (default pp2048,pp8192)
|
||||||
|
# decode -p 0 -n $LB_N -d $LB_D (default tg256 @ d4096)
|
||||||
|
# LB_N=0 skips the decode invocation.
|
||||||
|
#
|
||||||
|
# Everything that must be compared lives inside ONE llama-bench invocation
|
||||||
|
# (-p takes a comma-separated list), because cross-invocation variance on this
|
||||||
|
# box is about 7 percent while the within-run error bars are about 0.3 percent.
|
||||||
|
# Prefill and decode cells are never compared with each other, so they may
|
||||||
|
# live in separate invocations.
|
||||||
|
#
|
||||||
|
# Decode is measured at an explicit depth on purpose: tg32 at the default
|
||||||
|
# depth (n_ctx 32) is launch/barrier bound on this card and is NOT a valid
|
||||||
|
# decode metric (it once read -36% between two builds that were at parity at
|
||||||
|
# tg256). Decode rows are labelled with their depth (tg256@d4096); old tg32
|
||||||
|
# rows in results.txt are invalid-metric history.
|
||||||
|
#
|
||||||
|
# Flag traps, learned the hard way:
|
||||||
|
# * -tb is NOT a llama-bench flag. It exists on llama-server only.
|
||||||
|
# * -ncmoe is the llama-bench spelling of --n-cpu-moe.
|
||||||
|
# * -lm none is the load mode the server runs with. -mmp was removed upstream
|
||||||
|
# (#28334) and, like any bad flag, now makes the sweep silently empty.
|
||||||
|
# * A bad flag makes llama-bench print usage and exit silently, which looks
|
||||||
|
# exactly like a sweep that produced no rows. Always check row count.
|
||||||
|
# * A ubatch larger than -p never fills, so keep -p a multiple of -ub.
|
||||||
|
#
|
||||||
|
set -e
|
||||||
|
BUILD_DIR="$1"
|
||||||
|
LABEL="$2"
|
||||||
|
if [ -z "$BUILD_DIR" ] || [ -z "$LABEL" ]; then
|
||||||
|
echo "usage: $0 <build_dir> <label>" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
RESULTS="${RESULTS:-/root/bench/results.txt}"
|
||||||
|
MODEL="${MODEL:-/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf}"
|
||||||
|
LB_P="${LB_P:-2048,8192}"
|
||||||
|
LB_N="${LB_N:-256}"
|
||||||
|
LB_D="${LB_D:-4096}"
|
||||||
|
LB_R="${LB_R:-3}"
|
||||||
|
|
||||||
|
STAMP=$(date -u +%Y-%m-%dT%H:%M:%S)
|
||||||
|
export LD_LIBRARY_PATH="$BUILD_DIR/bin"
|
||||||
|
export LABEL BUILD_DIR STAMP RESULTS
|
||||||
|
|
||||||
|
lb() {
|
||||||
|
"$BUILD_DIR/bin/llama-bench" \
|
||||||
|
-m "$MODEL" \
|
||||||
|
-t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none \
|
||||||
|
-r "$LB_R" -o json "$@" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
record() { # $1 = what, stdin = llama-bench JSON
|
||||||
|
WHAT="$1" python3 -c '
|
||||||
|
import json, sys, os
|
||||||
|
label = os.environ["LABEL"]
|
||||||
|
build = os.environ["BUILD_DIR"]
|
||||||
|
stamp = os.environ["STAMP"]
|
||||||
|
results = os.environ["RESULTS"]
|
||||||
|
what = os.environ["WHAT"]
|
||||||
|
text = sys.stdin.read()
|
||||||
|
if not text.strip():
|
||||||
|
sys.stderr.write("ERROR: llama-bench produced no output for %s %s (bad flag?)\n" % (label, what))
|
||||||
|
sys.exit(3)
|
||||||
|
rows = json.loads(text)
|
||||||
|
if not rows:
|
||||||
|
sys.stderr.write("ERROR: llama-bench returned zero rows for %s %s\n" % (label, what))
|
||||||
|
sys.exit(3)
|
||||||
|
out = open(results, "a")
|
||||||
|
for r in rows:
|
||||||
|
test = "pp%d" % r["n_prompt"] if r["n_prompt"] else "tg%d" % r["n_gen"]
|
||||||
|
if r.get("n_depth", 0):
|
||||||
|
test += "@d%d" % r["n_depth"]
|
||||||
|
line = ("LBRESULT label=%s build=%s test=%s tps=%.2f stddev=%.2f "
|
||||||
|
"samples=%s time=%s"
|
||||||
|
% (label, build, test, r["avg_ts"], r["stddev_ts"],
|
||||||
|
r["samples_ts"], stamp))
|
||||||
|
out.write(line + "\n")
|
||||||
|
print(line)
|
||||||
|
out.close()
|
||||||
|
'
|
||||||
|
}
|
||||||
|
|
||||||
|
lb -p "$LB_P" -n 0 | record prefill
|
||||||
|
if [ "$LB_N" -gt 0 ]; then
|
||||||
|
lb -p 0 -n "$LB_N" -d "$LB_D" | record decode
|
||||||
|
fi
|
||||||
Executable
+239
@@ -0,0 +1,239 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Real-corpus prompt-processing benchmark for llama-server on the RX 580 pod.
|
||||||
|
|
||||||
|
Runs ON THE POD. Normally invoked by bench.sh, not directly.
|
||||||
|
|
||||||
|
Starts llama-server from a given build directory with the production config,
|
||||||
|
waits for /health, POSTs fixed slices of the Pan Tadeusz corpus to /completion,
|
||||||
|
records timings.prompt_n / prompt_ms / prompt_per_second, then shuts the server
|
||||||
|
down cleanly.
|
||||||
|
|
||||||
|
Prompt slices are FIXED CONSTANTS (see SLICES below), calibrated once against
|
||||||
|
the Qwen3.6-35B-A3B tokenizer, so that every build is measured on byte-identical
|
||||||
|
input. Do not re-calibrate for a normal run; use --calibrate only if the model
|
||||||
|
or the corpus changes.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
ppbench.py <build_dir> <label> [--sizes 4096,16384,32768] [--reps 3]
|
||||||
|
[--results PATH] [--calibrate]
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from statistics import median
|
||||||
|
|
||||||
|
BENCH_DIR = "/root/bench"
|
||||||
|
CORPUS = os.path.join(BENCH_DIR, "pan-tadeusz.txt")
|
||||||
|
DEFAULT_RESULTS = os.path.join(BENCH_DIR, "results.txt")
|
||||||
|
|
||||||
|
MODEL = ("/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/"
|
||||||
|
"snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/"
|
||||||
|
"Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf")
|
||||||
|
|
||||||
|
PORT = 8099
|
||||||
|
BASE = "http://127.0.0.1:%d" % PORT
|
||||||
|
|
||||||
|
# Production serving config. Keep in sync with /root/config.yaml (llama-swap).
|
||||||
|
# Traps encoded here on purpose:
|
||||||
|
# -b / -ub 2048 : a ubatch larger than the prompt never fills, so every
|
||||||
|
# prompt length below must stay a multiple of 2048.
|
||||||
|
# -lm none : what --no-mmap used to be; --mmap/--no-mmap/--mlock were
|
||||||
|
# removed upstream (#28334), so the old spelling now fails.
|
||||||
|
SERVER_ARGS = ["-t", "6", "-ngl", "99", "--n-cpu-moe", "40",
|
||||||
|
"-b", "2048", "-ub", "2048", "-fa", "1", "-lm", "none",
|
||||||
|
"--ctx-size", "40960", "--no-warmup",
|
||||||
|
"--port", str(PORT), "-m", MODEL]
|
||||||
|
|
||||||
|
# Calibrated once on 2026-09-08 against Qwen3.6-35B-A3B-UD-Q4_K_XL:
|
||||||
|
# Polish text of pan-tadeusz.txt runs 2.6702 chars/token, and /completion
|
||||||
|
# reports prompt_n identical to /tokenize (BOS offset 0).
|
||||||
|
# Each slice is corpus[0:NCHARS] read as UTF-8 text; NBYTES is the resulting
|
||||||
|
# UTF-8 byte length, recorded so the slice can be reproduced with byte tools.
|
||||||
|
# These produce EXACTLY the target prompt_n. Do not edit without recalibrating.
|
||||||
|
SLICES = {
|
||||||
|
4096: {"chars": 10776, "bytes": 11608},
|
||||||
|
16384: {"chars": 43858, "bytes": 47175},
|
||||||
|
32768: {"chars": 87165, "bytes": 93843},
|
||||||
|
}
|
||||||
|
|
||||||
|
DEFAULT_SIZES = [4096, 16384, 32768]
|
||||||
|
|
||||||
|
|
||||||
|
def post(path, payload, timeout=2400):
|
||||||
|
req = urllib.request.Request(
|
||||||
|
BASE + path,
|
||||||
|
data=json.dumps(payload).encode("utf-8"),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST")
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||||
|
return json.loads(r.read().decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def health_ok():
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(BASE + "/health", timeout=5) as r:
|
||||||
|
return json.loads(r.read().decode("utf-8")).get("status") == "ok"
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def start_server(build_dir, logpath):
|
||||||
|
env = dict(os.environ)
|
||||||
|
env["LD_LIBRARY_PATH"] = os.path.join(build_dir, "bin")
|
||||||
|
binary = os.path.join(build_dir, "bin", "llama-server")
|
||||||
|
if not os.path.exists(binary):
|
||||||
|
raise RuntimeError("no llama-server at %s" % binary)
|
||||||
|
log = open(logpath, "wb")
|
||||||
|
p = subprocess.Popen([binary] + SERVER_ARGS, stdout=log,
|
||||||
|
stderr=subprocess.STDOUT, env=env,
|
||||||
|
start_new_session=True)
|
||||||
|
deadline = time.time() + 1200
|
||||||
|
while time.time() < deadline:
|
||||||
|
if p.poll() is not None:
|
||||||
|
raise RuntimeError("llama-server exited early rc=%s, see %s"
|
||||||
|
% (p.returncode, logpath))
|
||||||
|
if health_ok():
|
||||||
|
return p
|
||||||
|
time.sleep(2)
|
||||||
|
stop_server(p)
|
||||||
|
raise RuntimeError("llama-server not healthy within 1200s, see %s" % logpath)
|
||||||
|
|
||||||
|
|
||||||
|
def stop_server(p):
|
||||||
|
if p is None or p.poll() is not None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
|
||||||
|
except Exception:
|
||||||
|
p.terminate()
|
||||||
|
for _ in range(120):
|
||||||
|
if p.poll() is not None:
|
||||||
|
return
|
||||||
|
time.sleep(1)
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(p.pid), signal.SIGKILL)
|
||||||
|
except Exception:
|
||||||
|
p.kill()
|
||||||
|
p.wait()
|
||||||
|
|
||||||
|
|
||||||
|
def n_tokens(text):
|
||||||
|
"""Exact token count via /tokenize. Cheap: no prefill."""
|
||||||
|
return len(post("/tokenize", {"content": text}, timeout=300)["tokens"])
|
||||||
|
|
||||||
|
|
||||||
|
def calibrate(corpus, targets):
|
||||||
|
"""Recompute the SLICES table. Only needed if model or corpus changes."""
|
||||||
|
probe = corpus[:2000]
|
||||||
|
tk = n_tokens(probe)
|
||||||
|
r = post("/completion", {"prompt": probe, "n_predict": 1,
|
||||||
|
"cache_prompt": False, "temperature": 0})
|
||||||
|
offset = r["timings"]["prompt_n"] - tk
|
||||||
|
cpt = len(probe) / float(tk)
|
||||||
|
sys.stderr.write("calibrate: %.4f chars/token, prompt_n offset %d\n"
|
||||||
|
% (cpt, offset))
|
||||||
|
for target in targets:
|
||||||
|
want = target - offset
|
||||||
|
lo, hi = 1, len(corpus)
|
||||||
|
guess = min(len(corpus), max(1, int(want * cpt)))
|
||||||
|
best = None
|
||||||
|
for _ in range(60):
|
||||||
|
got = n_tokens(corpus[:guess])
|
||||||
|
if got == want:
|
||||||
|
best = guess
|
||||||
|
break
|
||||||
|
if got < want:
|
||||||
|
lo = guess + 1
|
||||||
|
else:
|
||||||
|
hi = guess - 1
|
||||||
|
if lo > hi:
|
||||||
|
break
|
||||||
|
guess = (lo + hi) // 2
|
||||||
|
if best is None:
|
||||||
|
best = guess
|
||||||
|
nbytes = len(corpus[:best].encode("utf-8"))
|
||||||
|
sys.stderr.write("calibrate: %d tokens -> chars=%d bytes=%d\n"
|
||||||
|
% (target, best, nbytes))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) < 3:
|
||||||
|
sys.stderr.write(__doc__)
|
||||||
|
return 2
|
||||||
|
build_dir = os.path.abspath(sys.argv[1])
|
||||||
|
label = sys.argv[2]
|
||||||
|
sizes = list(DEFAULT_SIZES)
|
||||||
|
reps = 3
|
||||||
|
results_path = DEFAULT_RESULTS
|
||||||
|
do_calibrate = "--calibrate" in sys.argv
|
||||||
|
args = sys.argv[3:]
|
||||||
|
for i, a in enumerate(args):
|
||||||
|
if a == "--sizes":
|
||||||
|
sizes = [int(x) for x in args[i + 1].split(",")]
|
||||||
|
elif a == "--reps":
|
||||||
|
reps = int(args[i + 1])
|
||||||
|
elif a == "--results":
|
||||||
|
results_path = args[i + 1]
|
||||||
|
|
||||||
|
corpus = open(CORPUS, encoding="utf-8").read()
|
||||||
|
logpath = os.path.join(BENCH_DIR, "server-%s-%d.log" % (label, int(time.time())))
|
||||||
|
|
||||||
|
p = None
|
||||||
|
try:
|
||||||
|
sys.stderr.write("[%s] starting llama-server from %s\n" % (label, build_dir))
|
||||||
|
t0 = time.time()
|
||||||
|
p = start_server(build_dir, logpath)
|
||||||
|
sys.stderr.write("[%s] healthy after %.1fs\n" % (label, time.time() - t0))
|
||||||
|
|
||||||
|
if do_calibrate:
|
||||||
|
calibrate(corpus, sizes)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
stamp = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||||
|
out = open(results_path, "a")
|
||||||
|
for target in sizes:
|
||||||
|
if target not in SLICES:
|
||||||
|
sys.stderr.write("no calibrated slice for %d tokens, skipping\n" % target)
|
||||||
|
continue
|
||||||
|
nchars = SLICES[target]["chars"]
|
||||||
|
prompt = corpus[:nchars]
|
||||||
|
rates, mss, ns = [], [], set()
|
||||||
|
for rep in range(1, reps + 1):
|
||||||
|
r = post("/completion", {"prompt": prompt, "n_predict": 1,
|
||||||
|
"cache_prompt": False, "temperature": 0})
|
||||||
|
t = r["timings"]
|
||||||
|
rates.append(t["prompt_per_second"])
|
||||||
|
mss.append(t["prompt_ms"])
|
||||||
|
ns.add(t["prompt_n"])
|
||||||
|
line = ("RESULT corpus label=%s build=%s target=%d chars=%d "
|
||||||
|
"prompt_n=%d rep=%d prompt_ms=%.2f tps=%.2f time=%s"
|
||||||
|
% (label, build_dir, target, nchars, t["prompt_n"], rep,
|
||||||
|
t["prompt_ms"], t["prompt_per_second"], stamp))
|
||||||
|
out.write(line + "\n")
|
||||||
|
out.flush()
|
||||||
|
sys.stderr.write(line + "\n")
|
||||||
|
if ns != {target}:
|
||||||
|
sys.stderr.write("WARNING: prompt_n %s != target %d; slice table "
|
||||||
|
"is stale, rerun with --calibrate\n" % (sorted(ns), target))
|
||||||
|
s = ("SUMMARY corpus label=%s target=%d prompt_n=%s median_tps=%.2f "
|
||||||
|
"min_tps=%.2f max_tps=%.2f median_ms=%.1f time=%s"
|
||||||
|
% (label, target, sorted(ns), median(rates), min(rates),
|
||||||
|
max(rates), median(mss), stamp))
|
||||||
|
out.write(s + "\n")
|
||||||
|
out.flush()
|
||||||
|
sys.stderr.write(s + "\n")
|
||||||
|
out.close()
|
||||||
|
finally:
|
||||||
|
stop_server(p)
|
||||||
|
sys.stderr.write("[%s] server stopped\n" % label)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Executable
+91
@@ -0,0 +1,91 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# One-command entry point for the RX 580 prompt-processing benchmark.
|
||||||
|
#
|
||||||
|
# Runs from your workstation. Copies the pod-side harness into the pod and
|
||||||
|
# execs it there over kubectl. All the real work happens on the pod; this is
|
||||||
|
# a thin wrapper so there is exactly one command to remember.
|
||||||
|
#
|
||||||
|
# Examples:
|
||||||
|
# ./scripts/rx580-bench/run.sh --build /root/llama.cpp/build --label fork-before
|
||||||
|
# ./scripts/rx580-bench/run.sh --build /root/arms/new-clean/build --label new-clean \
|
||||||
|
# --build-b /root/arms/new-fork/build --label-b new-fork
|
||||||
|
# ./scripts/rx580-bench/run.sh --summarize
|
||||||
|
# ./scripts/rx580-bench/run.sh --install-only
|
||||||
|
#
|
||||||
|
# Every argument other than the wrapper-only flags below is passed straight
|
||||||
|
# through to bench.sh on the pod. See bench.sh --help.
|
||||||
|
#
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
KUBECONFIG_PATH="${KUBECONFIG:-/home/user/Projects/klaster/talos/generated/kubeconfig}"
|
||||||
|
NS="${RX580_NS:-llama}"
|
||||||
|
DEPLOY="${RX580_DEPLOY:-deploy/supervisord}"
|
||||||
|
POD_BENCH_DIR=/root/bench
|
||||||
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
export KUBECONFIG="$KUBECONFIG_PATH"
|
||||||
|
|
||||||
|
kx() { kubectl -n "$NS" exec -i "$DEPLOY" -- sh -c "$1"; }
|
||||||
|
|
||||||
|
install_harness() {
|
||||||
|
echo "[run] installing harness into $POD_BENCH_DIR"
|
||||||
|
kx "mkdir -p $POD_BENCH_DIR"
|
||||||
|
for f in bench.sh ppbench.py lbsweep.sh summarize.py; do
|
||||||
|
kx "cat > $POD_BENCH_DIR/$f" < "$HERE/$f"
|
||||||
|
done
|
||||||
|
kx "chmod +x $POD_BENCH_DIR/bench.sh $POD_BENCH_DIR/lbsweep.sh $POD_BENCH_DIR/ppbench.py $POD_BENCH_DIR/summarize.py"
|
||||||
|
# The corpus is large and rarely changes; only push it if missing.
|
||||||
|
if ! kx "test -s $POD_BENCH_DIR/pan-tadeusz.txt" 2>/dev/null; then
|
||||||
|
if [ -f "$HERE/pan-tadeusz.txt" ]; then
|
||||||
|
echo "[run] uploading corpus"
|
||||||
|
kx "cat > $POD_BENCH_DIR/pan-tadeusz.txt" < "$HERE/pan-tadeusz.txt"
|
||||||
|
else
|
||||||
|
echo "[run] ERROR: $POD_BENCH_DIR/pan-tadeusz.txt missing on the pod and" >&2
|
||||||
|
echo " no local copy at $HERE/pan-tadeusz.txt to upload." >&2
|
||||||
|
exit 4
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
ARGS=()
|
||||||
|
INSTALL_ONLY=0
|
||||||
|
SUMMARIZE=0
|
||||||
|
for a in "$@"; do
|
||||||
|
case "$a" in
|
||||||
|
--install-only) INSTALL_ONLY=1 ;;
|
||||||
|
--summarize) SUMMARIZE=1 ;;
|
||||||
|
*) ARGS+=("$a") ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
install_harness
|
||||||
|
|
||||||
|
if [ "$INSTALL_ONLY" -eq 1 ]; then
|
||||||
|
echo "[run] harness installed; not running anything"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$SUMMARIZE" -eq 1 ]; then
|
||||||
|
kx "python3 $POD_BENCH_DIR/summarize.py ${ARGS[*]:-}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${#ARGS[@]} -eq 0 ]; then
|
||||||
|
kx "sh $POD_BENCH_DIR/bench.sh --help"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# A full 3-round A/B run takes hours; run detached on the pod and tail it, so a
|
||||||
|
# dropped kubectl connection cannot orphan a half-finished run with llama-swap
|
||||||
|
# still down. bench.sh restarts llama-swap from its own EXIT trap either way.
|
||||||
|
STAMP="$(date -u +%Y%m%d-%H%M%S)"
|
||||||
|
LOG="$POD_BENCH_DIR/run-$STAMP.log"
|
||||||
|
echo "[run] starting detached run on the pod, log: $LOG"
|
||||||
|
kx "cd $POD_BENCH_DIR && nohup sh $POD_BENCH_DIR/bench.sh ${ARGS[*]} > $LOG 2>&1 & echo started"
|
||||||
|
|
||||||
|
echo "[run] tailing until ALL_RUNS_DONE (safe to Ctrl-C: the pod keeps running)"
|
||||||
|
kx "i=0; while [ \$i -lt 100000 ]; do
|
||||||
|
if grep -qE 'ALL_RUNS_DONE|Traceback|error:' $LOG; then break; fi
|
||||||
|
sleep 15; i=\$((i+1));
|
||||||
|
done; cat $LOG"
|
||||||
Executable
+101
@@ -0,0 +1,101 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Summarize a results.txt produced by bench.sh into a per-label table.
|
||||||
|
|
||||||
|
Usage: summarize.py [results.txt]
|
||||||
|
|
||||||
|
Prints median and min-max of every metric for every label, and flags
|
||||||
|
cross-label deltas against the ~5 percent noise floor of this machine.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from statistics import median
|
||||||
|
|
||||||
|
NOISE_FLOOR_PCT = 5.0
|
||||||
|
|
||||||
|
RE_CORPUS = re.compile(
|
||||||
|
r"^RESULT corpus label=(\S+) build=\S+ target=(\d+) chars=\d+ "
|
||||||
|
r"prompt_n=(\d+) rep=\d+ prompt_ms=([\d.]+) tps=([\d.]+)")
|
||||||
|
RE_LB = re.compile(
|
||||||
|
r"^LBRESULT label=(\S+) build=\S+ test=(\S+) tps=([\d.]+)")
|
||||||
|
RE_TG_NODEPTH = re.compile(r"^tg\d+-llama-bench$")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
path = sys.argv[1] if len(sys.argv) > 1 else "/root/bench/results.txt"
|
||||||
|
# data[metric][label] = list of tps
|
||||||
|
data = {}
|
||||||
|
order = []
|
||||||
|
labels = []
|
||||||
|
for line in open(path):
|
||||||
|
m = RE_CORPUS.match(line)
|
||||||
|
if m:
|
||||||
|
label, target, _pn, _ms, tps = m.groups()
|
||||||
|
metric = "pp%s-corpus" % target
|
||||||
|
else:
|
||||||
|
m = RE_LB.match(line)
|
||||||
|
if not m:
|
||||||
|
continue
|
||||||
|
label, test, tps = m.groups()
|
||||||
|
metric = "%s-llama-bench" % test
|
||||||
|
if metric not in data:
|
||||||
|
data[metric] = {}
|
||||||
|
order.append(metric)
|
||||||
|
data[metric].setdefault(label, []).append(float(tps))
|
||||||
|
if label not in labels:
|
||||||
|
labels.append(label)
|
||||||
|
|
||||||
|
if not data:
|
||||||
|
print("no parseable results in %s" % path)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
def sort_key(m):
|
||||||
|
n = re.search(r"(\d+)", m)
|
||||||
|
return (0 if "corpus" in m else 1, int(n.group(1)) if n else 0)
|
||||||
|
order.sort(key=sort_key)
|
||||||
|
|
||||||
|
w = max(len(l) for l in labels) + 2
|
||||||
|
head = "metric".ljust(26) + "".join(l.ljust(max(w, 24)) for l in labels)
|
||||||
|
print("")
|
||||||
|
print("t/s, median (min-max), n samples")
|
||||||
|
print(head)
|
||||||
|
print("-" * len(head))
|
||||||
|
for metric in order:
|
||||||
|
row = metric.ljust(26)
|
||||||
|
for label in labels:
|
||||||
|
vals = data[metric].get(label)
|
||||||
|
if not vals:
|
||||||
|
row += "-".ljust(max(w, 24))
|
||||||
|
else:
|
||||||
|
cell = "%.1f (%.1f-%.1f) n=%d" % (
|
||||||
|
median(vals), min(vals), max(vals), len(vals))
|
||||||
|
row += cell.ljust(max(w, 24))
|
||||||
|
print(row)
|
||||||
|
if any(RE_TG_NODEPTH.match(m) for m in order):
|
||||||
|
print("note: tgN rows without @d<depth> ran at n_ctx ~N (old default);"
|
||||||
|
" INVALID decode metric on this box, history only")
|
||||||
|
|
||||||
|
if len(labels) >= 2:
|
||||||
|
base = labels[0]
|
||||||
|
print("")
|
||||||
|
print("deltas vs %s (noise floor %.0f%%, anything under it is UNPROVEN)"
|
||||||
|
% (base, NOISE_FLOOR_PCT))
|
||||||
|
for other in labels[1:]:
|
||||||
|
print(" %s vs %s:" % (other, base))
|
||||||
|
for metric in order:
|
||||||
|
a = data[metric].get(base)
|
||||||
|
b = data[metric].get(other)
|
||||||
|
if not a or not b:
|
||||||
|
continue
|
||||||
|
ma, mb = median(a), median(b)
|
||||||
|
pct = (mb - ma) / ma * 100.0
|
||||||
|
verdict = "SIGNIFICANT" if abs(pct) >= NOISE_FLOOR_PCT else "unproven"
|
||||||
|
if RE_TG_NODEPTH.match(metric):
|
||||||
|
verdict = "INVALID metric (tg at n_ctx ~N), ignore"
|
||||||
|
print(" %-26s %+6.1f%% %s" % (metric, pct, verdict))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
+156
-21
@@ -587,7 +587,8 @@ llama_context::~llama_context() {
|
|||||||
synchronize();
|
synchronize();
|
||||||
|
|
||||||
// when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
|
// when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
|
||||||
if (!model.hparams.no_alloc && !opt_ctx) {
|
// the scheduler is also gone if the context is destroyed while cold (on-demand VRAM eviction, see release_device)
|
||||||
|
if (sched && !model.hparams.no_alloc && !opt_ctx) {
|
||||||
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
||||||
ggml_backend_t backend = backend_ptrs[i];
|
ggml_backend_t backend = backend_ptrs[i];
|
||||||
ggml_backend_buffer_type_t buft = backend_buft[i];
|
ggml_backend_buffer_type_t buft = backend_buft[i];
|
||||||
@@ -918,6 +919,86 @@ void llama_context::synchronize() {
|
|||||||
t_compute_start_us = 0;
|
t_compute_start_us = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool llama_context::device_resident() const {
|
||||||
|
return model.weights_resident() && (!moe_cache || moe_cache->resident());
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_context::release_device(bool evict_kv) {
|
||||||
|
if (!device_resident() && !(evict_kv && !kv_device_evicted)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// ensure no compute is in flight before freeing the device buffers
|
||||||
|
synchronize();
|
||||||
|
model.release_device_weights();
|
||||||
|
// the banks are emptied, not shadowed: restored slot maps must not point at zeroed experts
|
||||||
|
if (moe_cache) {
|
||||||
|
moe_cache->release_device();
|
||||||
|
}
|
||||||
|
if (evict_kv && memory && !kv_device_evicted) {
|
||||||
|
memory->release_device_buffers();
|
||||||
|
kv_device_evicted = true;
|
||||||
|
// also free the scheduler and its worst-case compute buffer (hundreds of MiB) so a cold
|
||||||
|
// model holds essentially no VRAM. Rebuilt lazily by sched_reserve() on restore.
|
||||||
|
sched.reset();
|
||||||
|
sched_need_reserve = true;
|
||||||
|
// finally, free each backend's own compute-scratch (Vulkan prealloc/staging buffers, which
|
||||||
|
// are owned by the backend and survive sched.reset()). Reallocated lazily on next compute.
|
||||||
|
for (auto & backend : backends) {
|
||||||
|
ggml_backend_free_scratch(backend.get());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_context::discard_device() {
|
||||||
|
// same as release_device(true), minus the host copies of the weights and the KV
|
||||||
|
synchronize();
|
||||||
|
model.release_device_weights(/*keep_shadow =*/ false);
|
||||||
|
if (moe_cache) {
|
||||||
|
moe_cache->release_device();
|
||||||
|
}
|
||||||
|
if (memory) {
|
||||||
|
memory->release_device_buffers(/*keep_shadow =*/ false);
|
||||||
|
}
|
||||||
|
kv_device_evicted = true;
|
||||||
|
device_discarded = true;
|
||||||
|
sched.reset();
|
||||||
|
sched_need_reserve = true;
|
||||||
|
for (auto & backend : backends) {
|
||||||
|
ggml_backend_free_scratch(backend.get());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_context::restore_memory_host() {
|
||||||
|
device_discarded = true;
|
||||||
|
if (!kv_device_evicted || !memory) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return memory->restore_device_buffers(/*to_host =*/ true);
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_context::restore_device() {
|
||||||
|
GGML_ASSERT(!device_discarded && "device buffers were discarded");
|
||||||
|
if (device_resident() && !kv_device_evicted && sched) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
model.restore_device_weights();
|
||||||
|
if (moe_cache) {
|
||||||
|
moe_cache->restore_device();
|
||||||
|
}
|
||||||
|
if (kv_device_evicted && memory) {
|
||||||
|
memory->restore_device_buffers();
|
||||||
|
kv_device_evicted = false;
|
||||||
|
}
|
||||||
|
// rebuild the scheduler + compute buffer if they were freed on release (evict_kv mode)
|
||||||
|
if (!sched) {
|
||||||
|
sched_reserve();
|
||||||
|
}
|
||||||
|
// make sure all weight/KV uploads have completed before any compute reads them
|
||||||
|
if (sched) {
|
||||||
|
ggml_backend_sched_synchronize(sched.get());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const llama_model & llama_context::get_model() const {
|
const llama_model & llama_context::get_model() const {
|
||||||
return model;
|
return model;
|
||||||
}
|
}
|
||||||
@@ -1851,6 +1932,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// on-demand VRAM: if the weights were released while this model was idle, bring them back
|
||||||
|
// to the device before building the compute graph
|
||||||
|
if (!device_resident()) {
|
||||||
|
restore_device();
|
||||||
|
}
|
||||||
|
|
||||||
const auto & vocab = model.vocab;
|
const auto & vocab = model.vocab;
|
||||||
const auto & hparams = model.hparams;
|
const auto & hparams = model.hparams;
|
||||||
|
|
||||||
@@ -2803,16 +2890,40 @@ bool llama_context::sched_copy_experts(ggml_backend_t backend, const ggml_tensor
|
|||||||
const size_t expert_size = src->nb[2];
|
const size_t expert_size = src->nb[2];
|
||||||
|
|
||||||
if (ids != st.ids || (int64_t) st.used.size() != n_expert) {
|
if (ids != st.ids || (int64_t) st.used.size() != n_expert) {
|
||||||
st.ids_data.resize(ggml_nbytes(ids)/sizeof(int32_t));
|
// Reading the ids back forces a full pipeline flush, since the ids are
|
||||||
ggml_backend_tensor_get_async(backend, ids, st.ids_data.data(), 0, ggml_nbytes(ids));
|
// produced on the same device we are about to upload to. Once the batch
|
||||||
ggml_backend_synchronize(backend);
|
// draws enough experts, essentially all of them come back used and the
|
||||||
|
// readback has bought nothing, so skip it and copy the whole tensor as one
|
||||||
|
// range. Copying an expert that no id selects cannot change the result:
|
||||||
|
// mul_mat_id only reads the rows the ids point at, so this stays exact.
|
||||||
|
// With the MoE cache on, cached experts are still copied device-to-device.
|
||||||
|
// validation/A-B knob: LLAMA_MOE_IDS_SKIP=0 always reads the ids back (the upstream path),
|
||||||
|
// =log reports every skip on stderr
|
||||||
|
static const char * skip_env = getenv("LLAMA_MOE_IDS_SKIP");
|
||||||
|
static const bool skip_off = skip_env != nullptr && strcmp(skip_env, "0") == 0;
|
||||||
|
static const bool skip_log = skip_env != nullptr && strcmp(skip_env, "log") == 0;
|
||||||
|
|
||||||
st.used.assign(n_expert, false);
|
const bool all_experts_used = !skip_off && ggml_nelements(ids) >= 4*n_expert;
|
||||||
for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) {
|
|
||||||
for (int64_t i0 = 0; i0 < ids->ne[0]; i0++) {
|
if (all_experts_used) {
|
||||||
const int32_t id = st.ids_data[i1*ids->nb[1]/sizeof(int32_t) + i0*ids->nb[0]/sizeof(int32_t)];
|
if (skip_log) {
|
||||||
GGML_ASSERT(id >= 0 && id < n_expert);
|
fprintf(stderr, "sched_copy_experts: ids readback skipped, n_ids=%" PRId64 " n_expert=%" PRId64 "\n",
|
||||||
st.used[id] = true;
|
ggml_nelements(ids), n_expert);
|
||||||
|
}
|
||||||
|
st.ids_data.clear();
|
||||||
|
st.used.assign(n_expert, true);
|
||||||
|
} else {
|
||||||
|
st.ids_data.resize(ggml_nbytes(ids)/sizeof(int32_t));
|
||||||
|
ggml_backend_tensor_get_async(backend, ids, st.ids_data.data(), 0, ggml_nbytes(ids));
|
||||||
|
ggml_backend_synchronize(backend);
|
||||||
|
|
||||||
|
st.used.assign(n_expert, false);
|
||||||
|
for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) {
|
||||||
|
for (int64_t i0 = 0; i0 < ids->ne[0]; i0++) {
|
||||||
|
const int32_t id = st.ids_data[i1*ids->nb[1]/sizeof(int32_t) + i0*ids->nb[0]/sizeof(int32_t)];
|
||||||
|
GGML_ASSERT(id >= 0 && id < n_expert);
|
||||||
|
st.used[id] = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3728,17 +3839,21 @@ llama_memory_breakdown llama_context::memory_breakdown() const {
|
|||||||
ret[buft].context += size;
|
ret[buft].context += size;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (model.hparams.no_alloc) {
|
// the scheduler (and its compute buffers) may have been freed while the model is cold
|
||||||
for (size_t i = 0; i < backends.size(); ++i) {
|
// (on-demand VRAM eviction, see release_device); it contributes no compute VRAM then.
|
||||||
ggml_backend_t backend = backends[i].get();
|
if (sched) {
|
||||||
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
if (model.hparams.no_alloc) {
|
||||||
ret[buft].compute += backend_buf_exp_size[i];
|
for (size_t i = 0; i < backends.size(); ++i) {
|
||||||
}
|
ggml_backend_t backend = backends[i].get();
|
||||||
} else {
|
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
||||||
for (const auto & backend_ptr : backends) {
|
ret[buft].compute += backend_buf_exp_size[i];
|
||||||
ggml_backend_t backend = backend_ptr.get();
|
}
|
||||||
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
} else {
|
||||||
ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend);
|
for (const auto & backend_ptr : backends) {
|
||||||
|
ggml_backend_t backend = backend_ptr.get();
|
||||||
|
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
||||||
|
ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return ret;
|
return ret;
|
||||||
@@ -4231,6 +4346,26 @@ void llama_synchronize(llama_context * ctx) {
|
|||||||
ctx->synchronize();
|
ctx->synchronize();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void llama_context_release_device(llama_context * ctx, bool evict_kv) {
|
||||||
|
ctx->release_device(evict_kv);
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_context_restore_device(llama_context * ctx) {
|
||||||
|
ctx->restore_device();
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_context_discard_device(llama_context * ctx) {
|
||||||
|
ctx->discard_device();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_context_restore_memory_host(llama_context * ctx) {
|
||||||
|
return ctx->restore_memory_host();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_context_weights_resident(const llama_context * ctx) {
|
||||||
|
return ctx->device_resident();
|
||||||
|
}
|
||||||
|
|
||||||
float * llama_get_logits(llama_context * ctx) {
|
float * llama_get_logits(llama_context * ctx) {
|
||||||
ctx->synchronize();
|
ctx->synchronize();
|
||||||
|
|
||||||
|
|||||||
@@ -60,6 +60,23 @@ struct llama_context {
|
|||||||
|
|
||||||
void synchronize();
|
void synchronize();
|
||||||
|
|
||||||
|
// on-demand device (VRAM) residency: free / rebuild the model's GPU weight buffers so that
|
||||||
|
// VRAM can be time-shared with another model. release_device() synchronizes first; decode()
|
||||||
|
// auto-restores when it finds the weights have been released. If evict_kv is set, the KV cache
|
||||||
|
// device buffers are also evicted to a host shadow (for when the KV would not fit alongside the
|
||||||
|
// other model) - this adds a D2H/H2D copy of the live KV on each cycle.
|
||||||
|
void release_device(bool evict_kv = false);
|
||||||
|
void restore_device();
|
||||||
|
// free every device buffer for good, without host copies (before exit). The context must not
|
||||||
|
// compute again: restore_device() aborts.
|
||||||
|
void discard_device();
|
||||||
|
// before exit: bring an evicted KV/recurrent state back in host memory so the state API can read
|
||||||
|
// it without the device. The context must not compute again: restore_device() aborts.
|
||||||
|
bool restore_memory_host();
|
||||||
|
|
||||||
|
// the model weights and the MoE cache are on the device
|
||||||
|
bool device_resident() const;
|
||||||
|
|
||||||
const llama_model & get_model() const;
|
const llama_model & get_model() const;
|
||||||
const llama_cparams & get_cparams() const;
|
const llama_cparams & get_cparams() const;
|
||||||
|
|
||||||
@@ -392,6 +409,11 @@ private:
|
|||||||
int32_t n_reallocs_same = 0;
|
int32_t n_reallocs_same = 0;
|
||||||
int32_t n_reallocs_diff = 0;
|
int32_t n_reallocs_diff = 0;
|
||||||
|
|
||||||
|
// on-demand device residency: true while the KV cache device buffers have been evicted to host
|
||||||
|
bool kv_device_evicted = false;
|
||||||
|
// set by discard_device(): the device buffers are gone and cannot be restored
|
||||||
|
bool device_discarded = false;
|
||||||
|
|
||||||
ggml_backend_t backend_cpu = nullptr;
|
ggml_backend_t backend_cpu = nullptr;
|
||||||
std::vector<ggml_backend_ptr> backends;
|
std::vector<ggml_backend_ptr> backends;
|
||||||
|
|
||||||
|
|||||||
@@ -264,6 +264,17 @@ void llama_kv_cache_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id,
|
|||||||
kv_swa ->state_read(io, seq_id, flags);
|
kv_swa ->state_read(io, seq_id, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void llama_kv_cache_iswa::release_device_buffers(bool keep_shadow) {
|
||||||
|
kv_base->release_device_buffers(keep_shadow);
|
||||||
|
kv_swa->release_device_buffers(keep_shadow);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_kv_cache_iswa::restore_device_buffers(bool to_host) {
|
||||||
|
bool ok = kv_base->restore_device_buffers(to_host);
|
||||||
|
ok = kv_swa->restore_device_buffers(to_host) && ok;
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
llama_kv_cache * llama_kv_cache_iswa::get_base() const {
|
llama_kv_cache * llama_kv_cache_iswa::get_base() const {
|
||||||
return kv_base.get();
|
return kv_base.get();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,6 +83,9 @@ public:
|
|||||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||||
|
|
||||||
|
void release_device_buffers(bool keep_shadow = true) override;
|
||||||
|
bool restore_device_buffers(bool to_host = false) override;
|
||||||
|
|
||||||
//
|
//
|
||||||
// llama_kv_cache_iswa specific API
|
// llama_kv_cache_iswa specific API
|
||||||
//
|
//
|
||||||
|
|||||||
+91
-2
@@ -396,7 +396,9 @@ void llama_kv_cache::clear(bool data) {
|
|||||||
|
|
||||||
if (data) {
|
if (data) {
|
||||||
for (auto & [_, buf] : ctxs_bufs) {
|
for (auto & [_, buf] : ctxs_bufs) {
|
||||||
ggml_backend_buffer_clear(buf.get(), 0);
|
if (buf) { // may be null if evicted for on-demand VRAM sharing
|
||||||
|
ggml_backend_buffer_clear(buf.get(), 0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -707,6 +709,9 @@ llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const {
|
|||||||
std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache::memory_breakdown() const {
|
std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache::memory_breakdown() const {
|
||||||
std::map<ggml_backend_buffer_type_t, size_t> ret;
|
std::map<ggml_backend_buffer_type_t, size_t> ret;
|
||||||
for (const auto & [ctx, buf] : ctxs_bufs) {
|
for (const auto & [ctx, buf] : ctxs_bufs) {
|
||||||
|
if (!buf) { // may be null if evicted for on-demand VRAM sharing
|
||||||
|
continue;
|
||||||
|
}
|
||||||
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
|
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
|
||||||
|
|
||||||
if (hparams.no_alloc) {
|
if (hparams.no_alloc) {
|
||||||
@@ -1916,12 +1921,96 @@ size_t llama_kv_cache::total_size() const {
|
|||||||
size_t size = 0;
|
size_t size = 0;
|
||||||
|
|
||||||
for (const auto & [_, buf] : ctxs_bufs) {
|
for (const auto & [_, buf] : ctxs_bufs) {
|
||||||
size += ggml_backend_buffer_get_size(buf.get());
|
if (buf) { // may be null if evicted for on-demand VRAM sharing
|
||||||
|
size += ggml_backend_buffer_get_size(buf.get());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void llama_kv_cache::release_device_buffers(bool keep_shadow) {
|
||||||
|
// NOTE: the caller must have synchronized the backend so nothing references these buffers.
|
||||||
|
// The KV cache is read-write, so its host shadow is (re)captured fresh on every release.
|
||||||
|
if (dev_released) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
dev_shadows.assign(ctxs_bufs.size(), device_buffer_shadow{});
|
||||||
|
size_t freed = 0;
|
||||||
|
for (size_t i = 0; i < ctxs_bufs.size(); ++i) {
|
||||||
|
ggml_context * ctx = ctxs_bufs[i].first.get();
|
||||||
|
ggml_backend_buffer_t buf = ctxs_bufs[i].second.get();
|
||||||
|
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
|
||||||
|
continue; // only real device (VRAM) buffers are evictable
|
||||||
|
}
|
||||||
|
auto & sh = dev_shadows[i];
|
||||||
|
sh.releasable = true;
|
||||||
|
sh.buft = ggml_backend_buffer_get_type(buf);
|
||||||
|
|
||||||
|
// capture live contents compactly in stable iteration order (skip views, which alias a base)
|
||||||
|
if (keep_shadow) {
|
||||||
|
size_t total = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||||
|
}
|
||||||
|
sh.data.resize(total);
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src != nullptr) { continue; }
|
||||||
|
const size_t n = ggml_nbytes(t);
|
||||||
|
ggml_backend_tensor_get(t, sh.data.data() + off, 0, n);
|
||||||
|
off += n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
freed += ggml_backend_buffer_get_size(buf);
|
||||||
|
ctxs_bufs[i].second.reset(); // free the device buffer
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
t->buffer = nullptr;
|
||||||
|
t->data = nullptr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dev_released = true;
|
||||||
|
if (freed > 0) {
|
||||||
|
LLAMA_LOG_INFO("%s: released %.2f MiB of KV cache from device\n", __func__, freed / 1024.0 / 1024.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_kv_cache::restore_device_buffers(bool to_host) {
|
||||||
|
if (!dev_released) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
for (size_t i = 0; i < ctxs_bufs.size(); ++i) {
|
||||||
|
auto & sh = dev_shadows[i];
|
||||||
|
if (!sh.releasable) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ggml_context * ctx = ctxs_bufs[i].first.get();
|
||||||
|
if (to_host) {
|
||||||
|
ctxs_bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft);
|
||||||
|
if (buf == nullptr) {
|
||||||
|
LLAMA_LOG_ERROR("%s: failed to reallocate KV cache device buffer (out of VRAM?)\n", __func__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src != nullptr) { continue; }
|
||||||
|
const size_t n = ggml_nbytes(t);
|
||||||
|
ggml_backend_tensor_set(t, sh.data.data() + off, 0, n);
|
||||||
|
off += n;
|
||||||
|
}
|
||||||
|
ctxs_bufs[i].second.reset(buf);
|
||||||
|
}
|
||||||
|
dev_released = false;
|
||||||
|
if (!to_host) {
|
||||||
|
dev_shadows.clear(); // recaptured on next release; to_host buffers point into the shadows
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
size_t llama_kv_cache::size_k_bytes() const {
|
size_t llama_kv_cache::size_k_bytes() const {
|
||||||
size_t size_k_bytes = 0;
|
size_t size_k_bytes = 0;
|
||||||
|
|
||||||
|
|||||||
@@ -151,6 +151,10 @@ public:
|
|||||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||||
|
|
||||||
|
// on-demand device (VRAM) residency (see llama_memory_i)
|
||||||
|
void release_device_buffers(bool keep_shadow = true) override;
|
||||||
|
bool restore_device_buffers(bool to_host = false) override;
|
||||||
|
|
||||||
//
|
//
|
||||||
// llama_kv_cache specific API
|
// llama_kv_cache specific API
|
||||||
//
|
//
|
||||||
@@ -298,6 +302,16 @@ private:
|
|||||||
// ggml contexts for the KV cache along with the allocated backend buffers:
|
// ggml contexts for the KV cache along with the allocated backend buffers:
|
||||||
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
|
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
|
||||||
|
|
||||||
|
// on-demand device eviction: host shadow of each device buffer's live contents (re-captured on
|
||||||
|
// every release since the KV is read-write), plus the buffer type needed to reallocate it.
|
||||||
|
struct device_buffer_shadow {
|
||||||
|
ggml_backend_buffer_type_t buft = nullptr;
|
||||||
|
bool releasable = false; // device (VRAM) buffer that was freed
|
||||||
|
std::vector<uint8_t> data; // captured tensor bytes (compact, iteration order)
|
||||||
|
};
|
||||||
|
std::vector<device_buffer_shadow> dev_shadows; // parallel to ctxs_bufs
|
||||||
|
bool dev_released = false;
|
||||||
|
|
||||||
// the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
|
// the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
|
||||||
// note: this is not part of the KV state and it's only used to speed-up the find_slot() method
|
// note: this is not part of the KV state and it's only used to speed-up the find_slot() method
|
||||||
std::vector<uint32_t> v_heads;
|
std::vector<uint32_t> v_heads;
|
||||||
|
|||||||
@@ -205,6 +205,18 @@ void llama_memory_hybrid::state_read(llama_io_read_i & io, llama_seq_id seq_id,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void llama_memory_hybrid::release_device_buffers(bool keep_shadow) {
|
||||||
|
// evict both the attention KV (grows with context) and the recurrent/SSM state
|
||||||
|
mem_attn->release_device_buffers(keep_shadow);
|
||||||
|
mem_recr->release_device_buffers(keep_shadow);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_memory_hybrid::restore_device_buffers(bool to_host) {
|
||||||
|
bool ok = mem_attn->restore_device_buffers(to_host);
|
||||||
|
ok = mem_recr->restore_device_buffers(to_host) && ok;
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
llama_kv_cache * llama_memory_hybrid::get_mem_attn() const {
|
llama_kv_cache * llama_memory_hybrid::get_mem_attn() const {
|
||||||
return mem_attn.get();
|
return mem_attn.get();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,6 +76,9 @@ public:
|
|||||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||||
|
|
||||||
|
void release_device_buffers(bool keep_shadow = true) override;
|
||||||
|
bool restore_device_buffers(bool to_host = false) override;
|
||||||
|
|
||||||
//
|
//
|
||||||
// llama_memory_hybrid specific API
|
// llama_memory_hybrid specific API
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -158,7 +158,9 @@ void llama_memory_recurrent::clear(bool data) {
|
|||||||
|
|
||||||
if (data) {
|
if (data) {
|
||||||
for (auto & [_, buf] : ctxs_bufs) {
|
for (auto & [_, buf] : ctxs_bufs) {
|
||||||
ggml_backend_buffer_clear(buf.get(), 0);
|
if (buf) { // may be null if evicted for on-demand VRAM sharing
|
||||||
|
ggml_backend_buffer_clear(buf.get(), 0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,6 +438,7 @@ void llama_memory_recurrent::set_rs_idx(llama_seq_id seq_id, uint32_t idx) {
|
|||||||
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_recurrent::memory_breakdown() const {
|
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_recurrent::memory_breakdown() const {
|
||||||
std::map<ggml_backend_buffer_type_t, size_t> ret;
|
std::map<ggml_backend_buffer_type_t, size_t> ret;
|
||||||
for (const auto & [_, buf] : ctxs_bufs) {
|
for (const auto & [_, buf] : ctxs_bufs) {
|
||||||
|
if (!buf) { continue; } // may be null if evicted for on-demand VRAM sharing
|
||||||
ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
|
ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
|
||||||
}
|
}
|
||||||
return ret;
|
return ret;
|
||||||
@@ -743,12 +746,95 @@ bool llama_memory_recurrent::is_empty() const {
|
|||||||
size_t llama_memory_recurrent::total_size() const {
|
size_t llama_memory_recurrent::total_size() const {
|
||||||
size_t size = 0;
|
size_t size = 0;
|
||||||
for (const auto & [_, buf] : ctxs_bufs) {
|
for (const auto & [_, buf] : ctxs_bufs) {
|
||||||
size += ggml_backend_buffer_get_size(buf.get());
|
if (buf) { // may be null if evicted for on-demand VRAM sharing
|
||||||
|
size += ggml_backend_buffer_get_size(buf.get());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void llama_memory_recurrent::release_device_buffers(bool keep_shadow) {
|
||||||
|
// Same mechanism as llama_kv_cache: the recurrent (SSM/conv) state is read-write, so its host
|
||||||
|
// shadow is (re)captured on every release. The caller must have synchronized the backend.
|
||||||
|
if (dev_released) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
dev_shadows.assign(ctxs_bufs.size(), device_buffer_shadow{});
|
||||||
|
size_t freed = 0;
|
||||||
|
for (size_t i = 0; i < ctxs_bufs.size(); ++i) {
|
||||||
|
ggml_context * ctx = ctxs_bufs[i].first.get();
|
||||||
|
ggml_backend_buffer_t buf = ctxs_bufs[i].second.get();
|
||||||
|
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
auto & sh = dev_shadows[i];
|
||||||
|
sh.releasable = true;
|
||||||
|
sh.buft = ggml_backend_buffer_get_type(buf);
|
||||||
|
|
||||||
|
if (keep_shadow) {
|
||||||
|
size_t total = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||||
|
}
|
||||||
|
sh.data.resize(total);
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src != nullptr) { continue; }
|
||||||
|
const size_t n = ggml_nbytes(t);
|
||||||
|
ggml_backend_tensor_get(t, sh.data.data() + off, 0, n);
|
||||||
|
off += n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
freed += ggml_backend_buffer_get_size(buf);
|
||||||
|
ctxs_bufs[i].second.reset();
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
t->buffer = nullptr;
|
||||||
|
t->data = nullptr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dev_released = true;
|
||||||
|
if (freed > 0) {
|
||||||
|
LLAMA_LOG_INFO("%s: released %.2f MiB of recurrent state from device\n", __func__, freed / 1024.0 / 1024.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_memory_recurrent::restore_device_buffers(bool to_host) {
|
||||||
|
if (!dev_released) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
for (size_t i = 0; i < ctxs_bufs.size(); ++i) {
|
||||||
|
auto & sh = dev_shadows[i];
|
||||||
|
if (!sh.releasable) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ggml_context * ctx = ctxs_bufs[i].first.get();
|
||||||
|
if (to_host) {
|
||||||
|
ctxs_bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft);
|
||||||
|
if (buf == nullptr) {
|
||||||
|
LLAMA_LOG_ERROR("%s: failed to reallocate recurrent device buffer (out of VRAM?)\n", __func__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src != nullptr) { continue; }
|
||||||
|
const size_t n = ggml_nbytes(t);
|
||||||
|
ggml_backend_tensor_set(t, sh.data.data() + off, 0, n);
|
||||||
|
off += n;
|
||||||
|
}
|
||||||
|
ctxs_bufs[i].second.reset(buf);
|
||||||
|
}
|
||||||
|
dev_released = false;
|
||||||
|
if (!to_host) {
|
||||||
|
dev_shadows.clear(); // to_host buffers point into the shadows
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
size_t llama_memory_recurrent::size_r_bytes() const {
|
size_t llama_memory_recurrent::size_r_bytes() const {
|
||||||
size_t size_r_bytes = 0;
|
size_t size_r_bytes = 0;
|
||||||
|
|
||||||
|
|||||||
@@ -66,6 +66,10 @@ public:
|
|||||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||||
|
|
||||||
|
// on-demand device (VRAM) residency (see llama_memory_i)
|
||||||
|
void release_device_buffers(bool keep_shadow = true) override;
|
||||||
|
bool restore_device_buffers(bool to_host = false) override;
|
||||||
|
|
||||||
uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot())
|
uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot())
|
||||||
uint32_t size = 0; // total number of cells, shared across all sequences
|
uint32_t size = 0; // total number of cells, shared across all sequences
|
||||||
uint32_t used = 0; // used cells (i.e. at least one seq_id)
|
uint32_t used = 0; // used cells (i.e. at least one seq_id)
|
||||||
@@ -126,6 +130,16 @@ private:
|
|||||||
// true if no layers - can happen if the layer filter removes all layers
|
// true if no layers - can happen if the layer filter removes all layers
|
||||||
bool is_empty() const;
|
bool is_empty() const;
|
||||||
|
|
||||||
|
// on-demand device eviction (see release_device_buffers): host shadow of each device buffer's
|
||||||
|
// live contents (recaptured on every release since the recurrent state is read-write)
|
||||||
|
struct device_buffer_shadow {
|
||||||
|
ggml_backend_buffer_type_t buft = nullptr;
|
||||||
|
bool releasable = false;
|
||||||
|
std::vector<uint8_t> data;
|
||||||
|
};
|
||||||
|
std::vector<device_buffer_shadow> dev_shadows; // parallel to ctxs_bufs
|
||||||
|
bool dev_released = false;
|
||||||
|
|
||||||
size_t total_size() const;
|
size_t total_size() const;
|
||||||
|
|
||||||
size_t size_r_bytes() const;
|
size_t size_r_bytes() const;
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
#include "llama-ext.h"
|
#include "llama-ext.h"
|
||||||
#include "llama-io.h"
|
#include "llama-io.h"
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
#include <set>
|
#include <set>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
@@ -90,3 +91,22 @@ std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t
|
|||||||
|
|
||||||
return std::vector<ggml_backend_buffer_t>(writer.buffers.begin(), writer.buffers.end());
|
return std::vector<ggml_backend_buffer_t>(writer.buffers.begin(), writer.buffers.end());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data) {
|
||||||
|
// the buffer base must be aligned to 32 bytes; the bytes before data.data() are never accessed
|
||||||
|
uint8_t * base = (uint8_t *) ((uintptr_t) data.data() & ~(uintptr_t) 31);
|
||||||
|
ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr(base, data.size() + (data.data() - base));
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src == nullptr) {
|
||||||
|
ggml_backend_tensor_alloc(buf, t, data.data() + off);
|
||||||
|
off += ggml_nbytes(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src != nullptr) {
|
||||||
|
ggml_backend_view_init(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@
|
|||||||
#include <map>
|
#include <map>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <functional>
|
#include <functional>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
struct llama_ubatch;
|
struct llama_ubatch;
|
||||||
|
|
||||||
@@ -125,6 +126,22 @@ struct llama_memory_i {
|
|||||||
|
|
||||||
virtual void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const = 0;
|
virtual void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const = 0;
|
||||||
virtual void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) = 0;
|
virtual void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) = 0;
|
||||||
|
|
||||||
|
//
|
||||||
|
// on-demand device (VRAM) residency
|
||||||
|
//
|
||||||
|
|
||||||
|
// Free this memory's device (VRAM) buffers to a host shadow and rebuild them on demand, to
|
||||||
|
// hand VRAM to another model while keeping the cached data (no re-prefill). The caller must
|
||||||
|
// have synchronized the backend first. Default: no-op (the memory stays resident).
|
||||||
|
// keep_shadow = false frees the buffers for good (before exit): no host copy, no restore.
|
||||||
|
// to_host = true points them at the host shadow instead, without a copy (before exit, to read a released state).
|
||||||
|
virtual void release_device_buffers(bool /*keep_shadow*/ = true) {}
|
||||||
|
virtual bool restore_device_buffers(bool /*to_host*/ = false) { return true; }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// wrap a compact host shadow (tensor bytes in iteration order, views skipped) as the buffer of the tensors of ctx, without a copy
|
||||||
|
// the shadow must outlive the buffer
|
||||||
|
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data);
|
||||||
|
|
||||||
using llama_memory_ptr = std::unique_ptr<llama_memory_i>;
|
using llama_memory_ptr = std::unique_ptr<llama_memory_i>;
|
||||||
@@ -422,6 +422,19 @@ size_t llama_file::size() const { return pimpl->size; }
|
|||||||
size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
|
size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
|
||||||
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
|
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
|
||||||
|
|
||||||
|
void llama_file::drop_cache(size_t offset, size_t len) const {
|
||||||
|
#ifdef __linux__
|
||||||
|
// only a hint: the pages are clean, a failure just leaves them cached
|
||||||
|
const int err = posix_fadvise(file_id(), (off_t) offset, (off_t) len, POSIX_FADV_DONTNEED);
|
||||||
|
if (err != 0) {
|
||||||
|
LLAMA_LOG_DEBUG("%s: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", __func__, strerror(err));
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
GGML_UNUSED(offset);
|
||||||
|
GGML_UNUSED(len);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
int llama_file::file_id() const {
|
int llama_file::file_id() const {
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
return _fileno(pimpl->fp);
|
return _fileno(pimpl->fp);
|
||||||
|
|||||||
@@ -39,6 +39,9 @@ struct llama_file {
|
|||||||
|
|
||||||
size_t read_alignment() const;
|
size_t read_alignment() const;
|
||||||
bool has_direct_io() const;
|
bool has_direct_io() const;
|
||||||
|
|
||||||
|
// hint the kernel to drop the file's page cache in [offset, offset + len), len 0 = to the end (Linux only)
|
||||||
|
void drop_cache(size_t offset = 0, size_t len = 0) const;
|
||||||
private:
|
private:
|
||||||
struct impl;
|
struct impl;
|
||||||
std::unique_ptr<impl> pimpl;
|
std::unique_ptr<impl> pimpl;
|
||||||
|
|||||||
@@ -1817,6 +1817,11 @@ bool llama_model_loader::load_all_data(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// the tensor is copied: drop its page cache as we go, unless the file is mapped (lazy tensors)
|
||||||
|
if (mappings.empty()) {
|
||||||
|
file->drop_cache(weight->offs, n_size);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
size_done += n_size;
|
size_done += n_size;
|
||||||
@@ -1858,6 +1863,12 @@ bool llama_model_loader::load_all_data(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// copied files: also drop what the per-tensor drops leave (metadata, partial pages at tensor edges)
|
||||||
|
if (mappings.empty()) {
|
||||||
|
for (const auto & file : files) {
|
||||||
|
file->drop_cache();
|
||||||
|
}
|
||||||
|
}
|
||||||
if (progress_callback) {
|
if (progress_callback) {
|
||||||
// Even though the model is done loading, we still honor
|
// Even though the model is done loading, we still honor
|
||||||
// cancellation since we need to free allocations.
|
// cancellation since we need to free allocations.
|
||||||
|
|||||||
+129
-1
@@ -1277,6 +1277,16 @@ struct llama_model::impl {
|
|||||||
// contexts where the model tensors metadata is stored as well as the corresponding buffers:
|
// contexts where the model tensors metadata is stored as well as the corresponding buffers:
|
||||||
std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;
|
std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;
|
||||||
|
|
||||||
|
// on-demand device (VRAM) weight residency: per-ctxs_bufs slot describing whether the
|
||||||
|
// device weight buffer can be freed and rebuilt from a host shadow (see release/restore_device_weights)
|
||||||
|
struct weight_release_slot {
|
||||||
|
ggml_backend_buffer_type_t buft = nullptr; // buffer type to reallocate on restore
|
||||||
|
bool releasable = false; // single-buffer alloc-path device (VRAM) buffer
|
||||||
|
bool resident = true; // currently allocated on device
|
||||||
|
std::vector<uint8_t> shadow; // host copy, captured lazily on first release
|
||||||
|
};
|
||||||
|
std::vector<weight_release_slot> weight_release; // parallel to ctxs_bufs
|
||||||
|
|
||||||
buft_list_t cpu_buft_list;
|
buft_list_t cpu_buft_list;
|
||||||
std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;
|
std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;
|
||||||
|
|
||||||
@@ -1940,8 +1950,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
|||||||
|
|
||||||
// a lazy context is mapped whatever the load mode, but the memory-fit pass maps nothing
|
// a lazy context is mapped whatever the load mode, but the memory-fit pass maps nothing
|
||||||
const bool is_lazy_mapped = ctx_key.lazy && !ml.no_alloc;
|
const bool is_lazy_mapped = ctx_key.lazy && !ml.no_alloc;
|
||||||
|
const bool host_ptr_path = (ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft;
|
||||||
|
|
||||||
if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
|
if (host_ptr_path) {
|
||||||
GGML_ASSERT(!ml.no_alloc);
|
GGML_ASSERT(!ml.no_alloc);
|
||||||
const size_t max_size = ggml_get_max_tensor_size(ctx);
|
const size_t max_size = ggml_get_max_tensor_size(ctx);
|
||||||
|
|
||||||
@@ -2006,6 +2017,25 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
|||||||
|
|
||||||
pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
|
pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
|
||||||
|
|
||||||
|
// record on-demand release metadata (parallel to ctxs_bufs). Only the single-buffer
|
||||||
|
// alloc path on a device (non-host) buffer can be freed and rebuilt from a host shadow;
|
||||||
|
// the host_ptr path (mmap / lazy), the no_alloc dummy buffers and host (CPU) buffers are
|
||||||
|
// left resident.
|
||||||
|
{
|
||||||
|
llama_model::impl::weight_release_slot slot;
|
||||||
|
auto & last_bufs = pimpl->ctxs_bufs.back().second;
|
||||||
|
if (!host_ptr_path && !ml.no_alloc && last_bufs.size() == 1) {
|
||||||
|
ggml_backend_buffer_t b = last_bufs[0].get();
|
||||||
|
// CPU extra buffers (CPU_REPACK) are not host buffers, but they are in RAM and have no get_tensor
|
||||||
|
ggml_backend_dev_t buft_dev = ggml_backend_buft_get_device(buft);
|
||||||
|
if (b != nullptr && !ggml_backend_buffer_is_host(b) && (buft_dev == nullptr || ggml_backend_dev_type(buft_dev) != GGML_BACKEND_DEVICE_TYPE_CPU)) {
|
||||||
|
slot.releasable = true;
|
||||||
|
slot.buft = buft;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pimpl->weight_release.push_back(std::move(slot));
|
||||||
|
}
|
||||||
|
|
||||||
ctx_bufs.emplace_back(ctx, std::move(buf_ptrs));
|
ctx_bufs.emplace_back(ctx, std::move(buf_ptrs));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2330,6 +2360,7 @@ void llama_model_base::load_tensors_hadamard() {
|
|||||||
std::vector<ggml_backend_buffer_ptr> buffers;
|
std::vector<ggml_backend_buffer_ptr> buffers;
|
||||||
buffers.emplace_back(std::move(buffer));
|
buffers.emplace_back(std::move(buffer));
|
||||||
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
|
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
|
||||||
|
pimpl->weight_release.emplace_back(); // small, left resident
|
||||||
rotations.push_back({ block_size, buft, rotation });
|
rotations.push_back({ block_size, buft, rotation });
|
||||||
it = std::prev(rotations.end());
|
it = std::prev(rotations.end());
|
||||||
}
|
}
|
||||||
@@ -2375,6 +2406,7 @@ void llama_model_base::load_tensors_hadamard() {
|
|||||||
std::vector<ggml_backend_buffer_ptr> buffers;
|
std::vector<ggml_backend_buffer_ptr> buffers;
|
||||||
buffers.emplace_back(std::move(buffer));
|
buffers.emplace_back(std::move(buffer));
|
||||||
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
|
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
|
||||||
|
pimpl->weight_release.emplace_back(); // small, left resident
|
||||||
st = sign_tensors.emplace(key, signs).first;
|
st = sign_tensors.emplace(key, signs).first;
|
||||||
}
|
}
|
||||||
sign_tensor = st->second;
|
sign_tensor = st->second;
|
||||||
@@ -2405,6 +2437,97 @@ ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM
|
|||||||
tn, ne, flags);
|
tn, ne, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void llama_model::release_device_weights(bool keep_shadow) const {
|
||||||
|
// NOTE: the caller is responsible for synchronizing the backend scheduler first, so that no
|
||||||
|
// compute is in flight referencing these buffers when they are freed.
|
||||||
|
size_t freed = 0;
|
||||||
|
for (size_t i = 0; i < pimpl->ctxs_bufs.size(); ++i) {
|
||||||
|
auto & slot = pimpl->weight_release[i];
|
||||||
|
if (!slot.releasable || !slot.resident) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ggml_context * ctx = pimpl->ctxs_bufs[i].first.get();
|
||||||
|
ggml_backend_buffer_t buf = pimpl->ctxs_bufs[i].second[0].get();
|
||||||
|
const size_t sz = ggml_backend_buffer_get_size(buf);
|
||||||
|
|
||||||
|
// lazily capture a host shadow of the weights (read-only, so captured only once).
|
||||||
|
// Store tensor bytes compactly in stable iteration order (independent of buffer layout /
|
||||||
|
// alignment padding) so restore does not depend on the re-allocated offsets matching.
|
||||||
|
// View tensors alias their base, so they are skipped (restored implicitly via their base).
|
||||||
|
if (keep_shadow && slot.shadow.empty()) {
|
||||||
|
size_t total = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src != nullptr) continue;
|
||||||
|
total += ggml_nbytes(t);
|
||||||
|
}
|
||||||
|
slot.shadow.resize(total);
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src != nullptr) continue;
|
||||||
|
const size_t n = ggml_nbytes(t);
|
||||||
|
ggml_backend_tensor_get(t, slot.shadow.data() + off, 0, n);
|
||||||
|
off += n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// free the device (VRAM) buffer and clear the now-dangling tensor pointers so that
|
||||||
|
// ggml_backend_alloc_ctx_tensors_from_buft reallocates them cleanly on restore
|
||||||
|
pimpl->ctxs_bufs[i].second[0].reset();
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
t->buffer = nullptr;
|
||||||
|
t->data = nullptr;
|
||||||
|
}
|
||||||
|
slot.resident = false;
|
||||||
|
freed += sz;
|
||||||
|
}
|
||||||
|
if (freed > 0) {
|
||||||
|
LLAMA_LOG_INFO("%s: released %.2f MiB of device weights\n", __func__, freed / 1024.0 / 1024.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_model::restore_device_weights() const {
|
||||||
|
size_t restored = 0;
|
||||||
|
for (size_t i = 0; i < pimpl->ctxs_bufs.size(); ++i) {
|
||||||
|
auto & slot = pimpl->weight_release[i];
|
||||||
|
if (!slot.releasable || slot.resident) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ggml_context * ctx = pimpl->ctxs_bufs[i].first.get();
|
||||||
|
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, slot.buft);
|
||||||
|
if (buf == nullptr) {
|
||||||
|
LLAMA_LOG_ERROR("%s: failed to reallocate device weight buffer (out of VRAM?)\n", __func__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||||
|
|
||||||
|
// re-upload weights from the compact host shadow, using the same stable iteration order
|
||||||
|
// and view-skipping as the capture above
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||||
|
if (t->view_src != nullptr) continue;
|
||||||
|
const size_t n = ggml_nbytes(t);
|
||||||
|
ggml_backend_tensor_set(t, slot.shadow.data() + off, 0, n);
|
||||||
|
off += n;
|
||||||
|
}
|
||||||
|
pimpl->ctxs_bufs[i].second[0].reset(buf);
|
||||||
|
slot.resident = true;
|
||||||
|
restored += ggml_backend_buffer_get_size(buf);
|
||||||
|
}
|
||||||
|
if (restored > 0) {
|
||||||
|
LLAMA_LOG_INFO("%s: restored %.2f MiB of device weights\n", __func__, restored / 1024.0 / 1024.0);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_model::weights_resident() const {
|
||||||
|
for (const auto & slot : pimpl->weight_release) {
|
||||||
|
if (slot.releasable && !slot.resident) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
std::string llama_model::arch_name() const {
|
std::string llama_model::arch_name() const {
|
||||||
return llm_arch_name(arch);
|
return llm_arch_name(arch);
|
||||||
}
|
}
|
||||||
@@ -2457,6 +2580,11 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_model::memory_breakdown() con
|
|||||||
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
|
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
|
||||||
} else {
|
} else {
|
||||||
for (const auto & buf : bufs) {
|
for (const auto & buf : bufs) {
|
||||||
|
// buf may be null if the device weights were released for on-demand VRAM sharing
|
||||||
|
// (see release_device_weights); skip it in the breakdown
|
||||||
|
if (!buf) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
// GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base
|
// GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base
|
||||||
ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
|
ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -892,6 +892,14 @@ struct llama_model {
|
|||||||
|
|
||||||
const struct ggml_tensor * get_tensor(const char * name) const;
|
const struct ggml_tensor * get_tensor(const char * name) const;
|
||||||
|
|
||||||
|
// on-demand device (VRAM) weight residency: free the GPU weight buffers (keeping a host
|
||||||
|
// shadow) and rebuild them on demand. Used to time-share VRAM between models without
|
||||||
|
// reloading or losing the KV cache. release_device_weights() requires the caller to have
|
||||||
|
// synchronized the backend scheduler first.
|
||||||
|
void release_device_weights(bool keep_shadow = true) const;
|
||||||
|
bool restore_device_weights() const;
|
||||||
|
bool weights_resident() const;
|
||||||
|
|
||||||
float get_rope_freq_base (const llama_cparams & cparams, int il) const;
|
float get_rope_freq_base (const llama_cparams & cparams, int il) const;
|
||||||
float get_rope_freq_scale(const llama_cparams & cparams, int il) const;
|
float get_rope_freq_scale(const llama_cparams & cparams, int il) const;
|
||||||
|
|
||||||
|
|||||||
+93
-1
@@ -230,6 +230,7 @@ struct llama_moe_cache::impl {
|
|||||||
static constexpr int64_t max_batch = 32;
|
static constexpr int64_t max_batch = 32;
|
||||||
|
|
||||||
int32_t n_expert_used;
|
int32_t n_expert_used;
|
||||||
|
bool no_alloc;
|
||||||
|
|
||||||
stats stats_small; // up to 8 tokens per ubatch
|
stats stats_small; // up to 8 tokens per ubatch
|
||||||
stats stats_large;
|
stats stats_large;
|
||||||
@@ -253,7 +254,7 @@ struct llama_moe_cache::impl {
|
|||||||
ggml_context_ptr ctx_views;
|
ggml_context_ptr ctx_views;
|
||||||
|
|
||||||
impl(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
|
impl(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
|
||||||
n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()) {
|
n_expert_used(model.hparams.n_expert_used_max()), no_alloc(model.hparams.no_alloc), layers(model.layers.size()) {
|
||||||
for (size_t i = 0; i < backends.size(); ++i) {
|
for (size_t i = 0; i < backends.size(); ++i) {
|
||||||
const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i]));
|
const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i]));
|
||||||
if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
|
if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
|
||||||
@@ -616,6 +617,85 @@ struct llama_moe_cache::impl {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// on-demand VRAM: free the banks and forget the cached experts, the slot maps all read -1 again
|
||||||
|
void release_device() {
|
||||||
|
if (no_alloc) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
size_t freed = 0;
|
||||||
|
for (device & d : devices) {
|
||||||
|
if (!d.buf) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
freed += ggml_backend_buffer_get_size(d.buf.get());
|
||||||
|
d.buf.reset();
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) {
|
||||||
|
t->buffer = nullptr;
|
||||||
|
t->data = nullptr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
invalidate();
|
||||||
|
if (freed > 0) {
|
||||||
|
LLAMA_LOG_INFO("%s: released %.2f MiB of MoE cache\n", __func__, freed/1024.0/1024.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// the banks come back empty, so the slot maps stay invalidated
|
||||||
|
bool restore_device() {
|
||||||
|
if (no_alloc) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
size_t restored = 0;
|
||||||
|
for (device & d : devices) {
|
||||||
|
if (!d.ctx || d.buf) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// also re-inits the views of the banks
|
||||||
|
d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft));
|
||||||
|
if (!d.buf) {
|
||||||
|
LLAMA_LOG_ERROR("%s: failed to reallocate the MoE cache buffer (out of VRAM?)\n", __func__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ggml_backend_buffer_clear(d.buf.get(), 0);
|
||||||
|
ggml_backend_buffer_set_usage(d.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||||
|
d.buf_size = ggml_backend_buffer_get_size(d.buf.get());
|
||||||
|
restored += d.buf_size;
|
||||||
|
}
|
||||||
|
if (restored > 0) {
|
||||||
|
LLAMA_LOG_INFO("%s: restored %.2f MiB of MoE cache\n", __func__, restored/1024.0/1024.0);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool resident() const {
|
||||||
|
for (const device & d : devices) {
|
||||||
|
if (d.ctx && !d.buf) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// empty every SLRU and mark all experts as not cached
|
||||||
|
void invalidate() {
|
||||||
|
for (group & g : groups) {
|
||||||
|
if (g.n_slots == 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// start from a fresh SLRU: init() does not reset the protected segment
|
||||||
|
g.slru = moe_cache_slru();
|
||||||
|
g.slru.init((int32_t) layers.size(), (int32_t) g.ref[0]->ne[2], g.n_slots);
|
||||||
|
for (int32_t il : g.layers) {
|
||||||
|
if (layers[il].slot_map != nullptr) {
|
||||||
|
g.slru.slot_map[il] = (int32_t *) layers[il].slot_map->data;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (buf_host) {
|
||||||
|
ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void log_stats() const {
|
void log_stats() const {
|
||||||
auto log = [](const char * name, const stats & st) {
|
auto log = [](const char * name, const stats & st) {
|
||||||
const size_t n = st.hits + st.misses;
|
const size_t n = st.hits + st.misses;
|
||||||
@@ -659,6 +739,18 @@ int64_t llama_moe_cache::copy_experts(ggml_backend_t backend, const ggml_tensor
|
|||||||
return pimpl->copy_experts(backend, w, dst, e, last);
|
return pimpl->copy_experts(backend, w, dst, e, last);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void llama_moe_cache::release_device() {
|
||||||
|
pimpl->release_device();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_moe_cache::restore_device() {
|
||||||
|
return pimpl->restore_device();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool llama_moe_cache::resident() const {
|
||||||
|
return pimpl->resident();
|
||||||
|
}
|
||||||
|
|
||||||
std::map<ggml_backend_buffer_type_t, size_t> llama_moe_cache::memory_breakdown() const {
|
std::map<ggml_backend_buffer_type_t, size_t> llama_moe_cache::memory_breakdown() const {
|
||||||
std::map<ggml_backend_buffer_type_t, size_t> res;
|
std::map<ggml_backend_buffer_type_t, size_t> res;
|
||||||
for (const auto & d : pimpl->devices) {
|
for (const auto & d : pimpl->devices) {
|
||||||
|
|||||||
@@ -32,6 +32,12 @@ public:
|
|||||||
// returns the number of experts copied, 0 if expert e is not in the cache
|
// returns the number of experts copied, 0 if expert e is not in the cache
|
||||||
int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last);
|
int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last);
|
||||||
|
|
||||||
|
// on-demand VRAM: free the device banks and empty the cache, the caller must synchronize first
|
||||||
|
void release_device();
|
||||||
|
// reallocate the banks empty, returns false if out of memory
|
||||||
|
bool restore_device();
|
||||||
|
bool resident() const;
|
||||||
|
|
||||||
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
|
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|||||||
@@ -27,6 +27,11 @@
|
|||||||
#include <functional>
|
#include <functional>
|
||||||
#include <float.h>
|
#include <float.h>
|
||||||
|
|
||||||
|
#if defined(__linux__)
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL};
|
struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL};
|
||||||
|
|
||||||
//#define CLIP_DEBUG_FUNCTIONS
|
//#define CLIP_DEBUG_FUNCTIONS
|
||||||
@@ -159,6 +164,13 @@ struct clip_ctx {
|
|||||||
ggml_backend_t backend_cpu = nullptr;
|
ggml_backend_t backend_cpu = nullptr;
|
||||||
ggml_backend_buffer_ptr buf;
|
ggml_backend_buffer_ptr buf;
|
||||||
|
|
||||||
|
// on-demand device (VRAM) residency: the vision/audio encoder weights are read-only, so a host
|
||||||
|
// shadow is captured once and the device buffer can be freed while the model is cold, then
|
||||||
|
// rebuilt on wake (mirrors llama_model::release_device_weights). See clip_release_device().
|
||||||
|
ggml_backend_buffer_type_t dev_buft = nullptr;
|
||||||
|
std::vector<uint8_t> dev_shadow;
|
||||||
|
bool dev_released = false;
|
||||||
|
|
||||||
|
|
||||||
int max_nodes = 8192;
|
int max_nodes = 8192;
|
||||||
ggml_backend_sched_ptr sched;
|
ggml_backend_sched_ptr sched;
|
||||||
@@ -3987,6 +3999,14 @@ struct clip_model_loader {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
|
LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
|
||||||
|
#if defined(__linux__)
|
||||||
|
// the weights are copied into our buffer: drop the file's page cache (only a hint)
|
||||||
|
const int fd = ::open(fname.c_str(), O_RDONLY);
|
||||||
|
if (fd != -1) {
|
||||||
|
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
|
||||||
|
::close(fd);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
} else {
|
} else {
|
||||||
LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
|
LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
|
||||||
}
|
}
|
||||||
@@ -4394,6 +4414,74 @@ void clip_free(clip_ctx * ctx) {
|
|||||||
delete ctx;
|
delete ctx;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void clip_release_device(struct clip_ctx * ctx) {
|
||||||
|
if (ctx == nullptr || ctx->dev_released || ctx->no_alloc) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ggml_backend_buffer_t buf = ctx->buf.get();
|
||||||
|
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
|
||||||
|
return; // CPU-backed encoder: nothing in VRAM to free
|
||||||
|
}
|
||||||
|
// ensure no encode is in flight before freeing the weights
|
||||||
|
if (ctx->sched) {
|
||||||
|
ggml_backend_sched_synchronize(ctx->sched.get());
|
||||||
|
}
|
||||||
|
ctx->dev_buft = ggml_backend_buffer_get_type(buf);
|
||||||
|
|
||||||
|
ggml_context * cd = ctx->ctx_data.get();
|
||||||
|
// capture the host shadow once (weights are read-only): compact, stable iteration order,
|
||||||
|
// skipping view tensors (which alias a base and are restored implicitly)
|
||||||
|
if (ctx->dev_shadow.empty()) {
|
||||||
|
size_t total = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||||
|
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||||
|
}
|
||||||
|
ctx->dev_shadow.resize(total);
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||||
|
if (t->view_src != nullptr) { continue; }
|
||||||
|
const size_t n = ggml_nbytes(t);
|
||||||
|
ggml_backend_tensor_get(t, ctx->dev_shadow.data() + off, 0, n);
|
||||||
|
off += n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// free the device buffer and clear the now-dangling tensor pointers so restore reallocates cleanly
|
||||||
|
ctx->buf.reset();
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||||
|
t->buffer = nullptr;
|
||||||
|
t->data = nullptr;
|
||||||
|
}
|
||||||
|
ctx->dev_released = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool clip_restore_device(struct clip_ctx * ctx) {
|
||||||
|
if (ctx == nullptr || !ctx->dev_released) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
ggml_context * cd = ctx->ctx_data.get();
|
||||||
|
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(cd, ctx->dev_buft);
|
||||||
|
if (buf == nullptr) {
|
||||||
|
LOG_ERR("%s: failed to reallocate encoder device buffer (out of VRAM?)\n", __func__);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||||
|
size_t off = 0;
|
||||||
|
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||||
|
if (t->view_src != nullptr) { continue; }
|
||||||
|
const size_t n = ggml_nbytes(t);
|
||||||
|
ggml_backend_tensor_set(t, ctx->dev_shadow.data() + off, 0, n);
|
||||||
|
off += n;
|
||||||
|
}
|
||||||
|
ctx->buf.reset(buf);
|
||||||
|
ctx->dev_released = false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool clip_weights_resident(const struct clip_ctx * ctx) {
|
||||||
|
return ctx == nullptr || !ctx->dev_released;
|
||||||
|
}
|
||||||
|
|
||||||
const char * clip_patch_merge_type(const struct clip_ctx * ctx) {
|
const char * clip_patch_merge_type(const struct clip_ctx * ctx) {
|
||||||
return ctx->model.hparams.mm_patch_merge_type == PATCH_MERGE_SPATIAL_UNPAD ? "spatial_unpad" : "flat";
|
return ctx->model.hparams.mm_patch_merge_type == PATCH_MERGE_SPATIAL_UNPAD ? "spatial_unpad" : "flat";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -75,6 +75,12 @@ int clip_get_image_max_tokens(const struct clip_ctx * ctx);
|
|||||||
|
|
||||||
void clip_free(struct clip_ctx * ctx);
|
void clip_free(struct clip_ctx * ctx);
|
||||||
|
|
||||||
|
// on-demand device (VRAM) residency: free/rebuild the encoder weight buffer to shrink an idle
|
||||||
|
// (cold) multimodal model's VRAM footprint. No-op for a CPU-backed encoder. See clip.cpp.
|
||||||
|
void clip_release_device(struct clip_ctx * ctx);
|
||||||
|
bool clip_restore_device(struct clip_ctx * ctx);
|
||||||
|
bool clip_weights_resident(const struct clip_ctx * ctx);
|
||||||
|
|
||||||
// TODO: should be enum, not string
|
// TODO: should be enum, not string
|
||||||
const char * clip_patch_merge_type(const struct clip_ctx * ctx);
|
const char * clip_patch_merge_type(const struct clip_ctx * ctx);
|
||||||
|
|
||||||
|
|||||||
@@ -1221,6 +1221,24 @@ std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::
|
|||||||
return output;
|
return output;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void mtmd_release_device(mtmd_context * ctx) {
|
||||||
|
if (ctx == nullptr) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (ctx->ctx_v) { clip_release_device(ctx->ctx_v); }
|
||||||
|
if (ctx->ctx_a) { clip_release_device(ctx->ctx_a); }
|
||||||
|
}
|
||||||
|
|
||||||
|
bool mtmd_restore_device(mtmd_context * ctx) {
|
||||||
|
if (ctx == nullptr) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool ok = true;
|
||||||
|
if (ctx->ctx_v) { ok = clip_restore_device(ctx->ctx_v) && ok; }
|
||||||
|
if (ctx->ctx_a) { ok = clip_restore_device(ctx->ctx_a) && ok; }
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
struct mtmd_tokenizer {
|
struct mtmd_tokenizer {
|
||||||
const mtmd_context * ctx;
|
const mtmd_context * ctx;
|
||||||
|
|
||||||
|
|||||||
@@ -139,6 +139,12 @@ MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname,
|
|||||||
|
|
||||||
MTMD_API void mtmd_free(mtmd_context * ctx);
|
MTMD_API void mtmd_free(mtmd_context * ctx);
|
||||||
|
|
||||||
|
// on-demand device (VRAM) residency: free / rebuild the vision+audio encoder weight buffers so an
|
||||||
|
// idle (cold) multimodal model releases its encoder VRAM and reclaims it on wake. No-op for a
|
||||||
|
// CPU-backed encoder. Restore returns false if reallocation failed (out of VRAM).
|
||||||
|
MTMD_API void mtmd_release_device(mtmd_context * ctx);
|
||||||
|
MTMD_API bool mtmd_restore_device(mtmd_context * ctx);
|
||||||
|
|
||||||
// whether we need to set non-causal mask before llama_decode
|
// whether we need to set non-causal mask before llama_decode
|
||||||
// if chunk is nullptr, we assume the default case where chunk is an image chunk
|
// if chunk is nullptr, we assume the default case where chunk is an image chunk
|
||||||
MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);
|
MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);
|
||||||
|
|||||||
@@ -30,8 +30,11 @@
|
|||||||
#include <memory>
|
#include <memory>
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <random>
|
#include <random>
|
||||||
|
#include <sstream>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
|
#include <thread>
|
||||||
|
#include <atomic>
|
||||||
|
|
||||||
// fix problem with std::min and std::max
|
// fix problem with std::min and std::max
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
@@ -42,6 +45,16 @@
|
|||||||
#include <windows.h>
|
#include <windows.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
// POSIX file locking for the cross-process VRAM arbiter (see vram_share_* below)
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
#include <sys/file.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <poll.h>
|
||||||
|
#include <cerrno>
|
||||||
|
#endif
|
||||||
|
|
||||||
constexpr int HTTP_POLLING_SECONDS = 1;
|
constexpr int HTTP_POLLING_SECONDS = 1;
|
||||||
|
|
||||||
static common_speculative_output_limits server_output_limits(const common_params & params) {
|
static common_speculative_output_limits server_output_limits(const common_params & params) {
|
||||||
@@ -882,7 +895,36 @@ static int encode_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch,
|
|||||||
// TODO @ngxson : move this log line to debug when it become more stable
|
// TODO @ngxson : move this log line to debug when it become more stable
|
||||||
SLT_TRC(slot, "encoding mtmd batch from idx = %zu, n_chunks = %d\n", idx, n_added);
|
SLT_TRC(slot, "encoding mtmd batch from idx = %zu, n_chunks = %d\n", idx, n_added);
|
||||||
|
|
||||||
|
// Bring the vision/audio encoder into VRAM just for this encode. In on-demand mode the
|
||||||
|
// encoder is normally kept in RAM (its VRAM freed for KV / expert cache). If it does not
|
||||||
|
// fit, evict the LLM backbone weights first: they are NOT needed while the encoder runs (the
|
||||||
|
// decode that uses them happens afterwards) and their host shadow is read-only, so this is a
|
||||||
|
// cheap free (no D2H) and leaves the KV cache / prompt cache untouched.
|
||||||
|
static const bool mmproj_ondemand = getenv("LLAMA_MMPROJ_ONDEMAND") != nullptr;
|
||||||
|
bool weights_evicted = false;
|
||||||
|
if (mctx && !mtmd_restore_device(mctx)) {
|
||||||
|
if (mmproj_ondemand) {
|
||||||
|
llama_context_release_device(slot.ctx_tgt, /* evict_kv = */ false); // free backbone, keep KV
|
||||||
|
weights_evicted = true;
|
||||||
|
}
|
||||||
|
if (!mtmd_restore_device(mctx)) {
|
||||||
|
if (weights_evicted) { llama_context_restore_device(slot.ctx_tgt); }
|
||||||
|
SLT_ERR(slot, "%s", "failed to bring the multimodal encoder into VRAM for encoding\n");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
res = mtmd_batch_encode(mbatch.get());
|
res = mtmd_batch_encode(mbatch.get());
|
||||||
|
|
||||||
|
// release the encoder VRAM again (on-demand), then restore the backbone weights for decode.
|
||||||
|
// Order matters: free the encoder BEFORE re-uploading the weights so the peak stays within VRAM.
|
||||||
|
if (mctx && mmproj_ondemand) {
|
||||||
|
mtmd_release_device(mctx);
|
||||||
|
}
|
||||||
|
if (weights_evicted) {
|
||||||
|
llama_context_restore_device(slot.ctx_tgt);
|
||||||
|
}
|
||||||
|
|
||||||
if (res != 0) {
|
if (res != 0) {
|
||||||
SLT_ERR(slot, "failed to encode mtmd batch for chunk idx = %zu, res = %d\n", idx, res);
|
SLT_ERR(slot, "failed to encode mtmd batch for chunk idx = %zu, res = %d\n", idx, res);
|
||||||
return -1;
|
return -1;
|
||||||
@@ -998,6 +1040,15 @@ static bool add_mtmd_chunk(const server_slot & slot, const mtmd_input_chunk * ch
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// smaller prompts are not saved, and a smaller gain is not worth a file read
|
||||||
|
static constexpr size_t DISK_CACHE_MIN_TOKENS = 256;
|
||||||
|
|
||||||
|
// the token list of a text-only prompt, false if it holds media (get_tokens() asserts when an mmproj is loaded)
|
||||||
|
static bool disk_cache_text(const server_tokens & tokens, llama_tokens & out) {
|
||||||
|
out = tokens.get_text_tokens();
|
||||||
|
return out.size() == tokens.size();
|
||||||
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
// server_context_impl (private implementation)
|
// server_context_impl (private implementation)
|
||||||
//
|
//
|
||||||
@@ -1044,6 +1095,8 @@ public:
|
|||||||
// we don't call it again here to avoid double free
|
// we don't call it again here to avoid double free
|
||||||
destroy();
|
destroy();
|
||||||
}
|
}
|
||||||
|
// only once our VRAM is freed (normally vram_share_exit() has done both already)
|
||||||
|
vram_share_shutdown();
|
||||||
}
|
}
|
||||||
|
|
||||||
server_metrics get_metrics() const {
|
server_metrics get_metrics() const {
|
||||||
@@ -1070,6 +1123,423 @@ private:
|
|||||||
llama_model * model_dft = nullptr;
|
llama_model * model_dft = nullptr;
|
||||||
llama_context * ctx_dft = nullptr;
|
llama_context * ctx_dft = nullptr;
|
||||||
|
|
||||||
|
// Cross-process VRAM arbiter: when LLAMA_SLEEP_VRAM_ONLY is set, several always-loaded
|
||||||
|
// llama-server processes time-share one GPU. A single flock() on <arena>/token.lock is the
|
||||||
|
// baton: "resident (device buffers in VRAM) iff I hold the token". A model warms up (locks +
|
||||||
|
// restores) when it wakes for a request, and goes cold (releases + unlocks) when it is idle and
|
||||||
|
// another process waits for the token, so only one model holds VRAM at a time. The process, its
|
||||||
|
// context and its prompt cache stay alive, so nothing is re-prefilled. Going cold releases the
|
||||||
|
// weights; with LLAMA_SLEEP_EVICT_KV it also evicts the KV/recurrent state to a host shadow and
|
||||||
|
// frees the scheduler and backend scratch. All of it is restored bit-exact on wake.
|
||||||
|
bool vram_only = false; // release-mode sleep active (LLAMA_SLEEP_VRAM_ONLY)
|
||||||
|
bool vram_evict_kv = false; // also evict the KV cache to host on release (LLAMA_SLEEP_EVICT_KV)
|
||||||
|
bool vram_flock = false; // cross-process flock coordination available
|
||||||
|
std::atomic<bool> vram_cold{true}; // weights currently released
|
||||||
|
int vram_lock_fd = -1; // fd for <arena>/token.lock
|
||||||
|
|
||||||
|
// "want" lock: a process that waits for the token holds <arena>/want.lock shared, and drops it
|
||||||
|
// as soon as it owns the token. The holder probes that lock with a non-blocking exclusive
|
||||||
|
// flock, which fails only while somebody waits. This is level-triggered state owned by the
|
||||||
|
// kernel, so there is no event to miss and nothing goes stale if a waiter dies.
|
||||||
|
// The probe needs its own fd: flock() treats two open file descriptions of one file
|
||||||
|
// independently, so probing on the waiter fd would convert our own lock instead of conflicting.
|
||||||
|
int vram_want_fd = -1; // waiter side, LOCK_SH while waiting
|
||||||
|
int vram_want_probe_fd = -1; // probe side, LOCK_EX | LOCK_NB only
|
||||||
|
bool vram_want_held = false; // only touched by the waiting thread
|
||||||
|
|
||||||
|
// open the shared arena (token.lock + want.lock). Idempotent; sets vram_only/vram_flock.
|
||||||
|
void vram_arena_open() {
|
||||||
|
if (getenv("LLAMA_SLEEP_VRAM_ONLY") == nullptr) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
vram_only = true;
|
||||||
|
vram_evict_kv = getenv("LLAMA_SLEEP_EVICT_KV") != nullptr;
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
if (vram_lock_fd >= 0) {
|
||||||
|
return; // already open
|
||||||
|
}
|
||||||
|
const char * arena_env = getenv("LLAMA_VRAM_ARENA");
|
||||||
|
const std::string arena = arena_env ? arena_env : "/dev/shm/llama-vram";
|
||||||
|
mkdir(arena.c_str(), 0777);
|
||||||
|
const std::string lock_path = arena + "/token.lock";
|
||||||
|
vram_lock_fd = open(lock_path.c_str(), O_RDWR | O_CREAT, 0666);
|
||||||
|
if (vram_lock_fd >= 0) {
|
||||||
|
vram_flock = true;
|
||||||
|
const std::string want_path = arena + "/want.lock";
|
||||||
|
vram_want_fd = open(want_path.c_str(), O_RDWR | O_CREAT, 0666);
|
||||||
|
vram_want_probe_fd = open(want_path.c_str(), O_RDWR | O_CREAT, 0666);
|
||||||
|
if (vram_want_fd < 0 || vram_want_probe_fd < 0) {
|
||||||
|
SRV_WRN("VRAM arbiter: cannot open %s, the holder will only release on idle\n", want_path.c_str());
|
||||||
|
}
|
||||||
|
SRV_INF("VRAM arbiter: cross-process GPU sharing via %s (want %s)\n",
|
||||||
|
lock_path.c_str(), want_path.c_str());
|
||||||
|
} else {
|
||||||
|
SRV_WRN("VRAM arbiter: cannot open %s, running VRAM-only sleep without cross-process lock\n", lock_path.c_str());
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// Acquire the VRAM token BEFORE uploading this model's weights, so any model currently resident
|
||||||
|
// on the GPU releases first and the load uploads into free VRAM instead of racing it (which
|
||||||
|
// could OOM, e.g. the 4B task model holding VRAM while a large model loads). Blocks until free.
|
||||||
|
// Called from load_model() right before common_init_from_params().
|
||||||
|
void vram_acquire_for_load() {
|
||||||
|
vram_arena_open();
|
||||||
|
if (!vram_only) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!vram_take_token()) {
|
||||||
|
SRV_WRN("%s", "VRAM arbiter: loading weights without the GPU token\n");
|
||||||
|
}
|
||||||
|
vram_cold = false; // we hold the token; weights will be resident after the upload
|
||||||
|
}
|
||||||
|
|
||||||
|
// Called from init() after the (coordinated) load. The model stays warm (holding the token
|
||||||
|
// acquired in vram_acquire_for_load) and serves its first request without a re-warm; the queue
|
||||||
|
// releases it once another model registers on want.lock.
|
||||||
|
void vram_share_init() {
|
||||||
|
vram_arena_open();
|
||||||
|
if (!vram_only) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
vram_cold = false; // resident and holding the token after the coordinated load
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
if (vram_flock && vram_want_probe_fd >= 0) {
|
||||||
|
queue_tasks.on_should_yield([this]{ return vram_should_yield(); });
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// true while another process waits for the token and we still hold it. Called by should_sleep()
|
||||||
|
// on the start_loop() thread, under mutex_tasks, and only when the queue is idle - so a release
|
||||||
|
// can never happen underneath an in-flight decode, and there is no event to latch: the answer
|
||||||
|
// comes from live kernel state at the moment the loop asks.
|
||||||
|
bool vram_should_yield() {
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
if (!vram_flock || vram_cold || vram_want_probe_fd < 0) {
|
||||||
|
return false; // we hold no token to give away
|
||||||
|
}
|
||||||
|
// never blocks: LOCK_NB only, and we drop the probe lock at once if we win it
|
||||||
|
if (flock(vram_want_probe_fd, LOCK_EX | LOCK_NB) == 0) {
|
||||||
|
flock(vram_want_probe_fd, LOCK_UN);
|
||||||
|
return false; // nobody wants the GPU
|
||||||
|
}
|
||||||
|
return errno == EWOULDBLOCK || errno == EAGAIN;
|
||||||
|
#else
|
||||||
|
return false;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// register as a waiter, so the current token holder sees us and yields. Shared, so several
|
||||||
|
// waiters coexist. The kernel drops it if we die, so it can never strand the holder.
|
||||||
|
void vram_want_acquire() {
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
if (vram_want_fd < 0 || vram_want_held) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
while (flock(vram_want_fd, LOCK_SH) != 0) {
|
||||||
|
if (errno != EINTR) { // our signal handlers run without SA_RESTART
|
||||||
|
SRV_WRN("VRAM arbiter: cannot take want.lock (errno %d)\n", errno);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
vram_want_held = true;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// stop asking for the token. Must run on every path out of vram_take_token(), or the holder
|
||||||
|
// yields forever.
|
||||||
|
void vram_want_release() {
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
if (vram_want_fd < 0 || !vram_want_held) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
flock(vram_want_fd, LOCK_UN);
|
||||||
|
vram_want_held = false;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// take the shared VRAM token: register on want.lock, then wait for token.lock. want.lock is
|
||||||
|
// level-triggered state, so the holder still learns of us if it is loading or restoring when we
|
||||||
|
// start to wait. No timeout: the holder can be mid-generation, and decoding without the token
|
||||||
|
// risks an OOM that kills both processes. Returns false without the token if the server shuts
|
||||||
|
// down meanwhile (the caller checks queue_tasks.is_stopping()).
|
||||||
|
bool vram_take_token() {
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
if (!vram_flock) {
|
||||||
|
return true; // no cross-process lock: there is no token to take
|
||||||
|
}
|
||||||
|
vram_want_acquire();
|
||||||
|
bool logged = false;
|
||||||
|
while (flock(vram_lock_fd, LOCK_EX | LOCK_NB) != 0) {
|
||||||
|
if (errno == EWOULDBLOCK || errno == EAGAIN) {
|
||||||
|
if (!logged) {
|
||||||
|
SRV_INF("%s", "VRAM arbiter: waiting for the GPU token\n");
|
||||||
|
logged = true;
|
||||||
|
}
|
||||||
|
// poll, not a blocking flock: terminate() must be able to stop this wait, and it can
|
||||||
|
// only signal us through an atomic (we hold mutex_tasks here)
|
||||||
|
if (queue_tasks.is_stopping()) {
|
||||||
|
SRV_INF("%s", "VRAM arbiter: shutting down, no longer waiting for the GPU token\n");
|
||||||
|
vram_want_release();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (errno == EINTR) {
|
||||||
|
continue; // our signal handlers run without SA_RESTART
|
||||||
|
}
|
||||||
|
SRV_WRN("VRAM arbiter: flock failed (errno %d)\n", errno);
|
||||||
|
vram_want_release();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
vram_want_release();
|
||||||
|
return true;
|
||||||
|
#else
|
||||||
|
return true;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// acquire the VRAM token (blocking) and bring the device buffers back (weights; with
|
||||||
|
// LLAMA_SLEEP_EVICT_KV also the KV/recurrent state and the scheduler). Called on the loop thread:
|
||||||
|
// on wake from handle_sleeping_state(false), before update_slots() touches the KV, and again
|
||||||
|
// right before each decode (a no-op when already warm). It never races compute. Returns false,
|
||||||
|
// still cold, if the server shuts down first: the caller must not touch the device then.
|
||||||
|
bool vram_ensure_warm() {
|
||||||
|
if (!vram_only || !vram_cold) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (queue_tasks.is_stopping()) {
|
||||||
|
return false; // no restore just to exit
|
||||||
|
}
|
||||||
|
if (!vram_take_token()) {
|
||||||
|
if (queue_tasks.is_stopping()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
SRV_WRN("%s", "VRAM arbiter: restoring weights without the GPU token\n");
|
||||||
|
}
|
||||||
|
llama_context_restore_device(ctx_tgt);
|
||||||
|
if (ctx_dft != nullptr) {
|
||||||
|
llama_context_restore_device(ctx_dft);
|
||||||
|
}
|
||||||
|
// NOTE: the multimodal (vision/audio) encoder is intentionally NOT restored here. It is
|
||||||
|
// brought into VRAM just-in-time before an image/audio encode (encode_mtmd_chunk) and, in
|
||||||
|
// on-demand mode, released again right after - so a warm text-only model holds no encoder
|
||||||
|
// VRAM (that space is free for KV / expert cache). A cold->warm wake for a *text* request
|
||||||
|
// therefore leaves the encoder in RAM; an image request restores it at encode time.
|
||||||
|
// NOTE: keep this AFTER the restore: vram_cold == false means "holding the token AND
|
||||||
|
// resident". Today the only reader that could give the token away, vram_should_yield(), runs
|
||||||
|
// on this same loop thread and so cannot run before we return; the order is what kept the
|
||||||
|
// retired cross-thread doorbell design from yielding mid-restore (FORK.md F6, attempt 1).
|
||||||
|
// Do not move it without an A/B of ordinary handoff latency. We dropped want.lock when we
|
||||||
|
// took the token, so the next probe reports only the waiters that are still there.
|
||||||
|
vram_cold = false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// the loop has stopped: free our VRAM and hand the token over now, without waiting for the HTTP
|
||||||
|
// threads and the teardown. Discarded, not released: no host copy, and the context never decodes
|
||||||
|
// again. Runs on the loop thread after the loop returned, so nothing else uses the device.
|
||||||
|
void vram_share_exit() {
|
||||||
|
if (vram_only && !vram_cold && ctx_tgt != nullptr) {
|
||||||
|
llama_context_discard_device(ctx_tgt);
|
||||||
|
if (ctx_dft != nullptr) {
|
||||||
|
llama_context_discard_device(ctx_dft);
|
||||||
|
}
|
||||||
|
if (mctx != nullptr) {
|
||||||
|
mtmd_release_device(mctx);
|
||||||
|
}
|
||||||
|
vram_cold = true;
|
||||||
|
SRV_INF("%s", "VRAM arbiter: loop stopped, device memory freed, releasing the GPU token\n");
|
||||||
|
}
|
||||||
|
vram_share_shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
void vram_share_shutdown() {
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
vram_want_release();
|
||||||
|
if (vram_want_probe_fd >= 0) { close(vram_want_probe_fd); vram_want_probe_fd = -1; }
|
||||||
|
if (vram_want_fd >= 0) { close(vram_want_fd); vram_want_fd = -1; }
|
||||||
|
if (vram_lock_fd >= 0) { flock(vram_lock_fd, LOCK_UN); close(vram_lock_fd); vram_lock_fd = -1; }
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// release the device weights (their host shadow is kept; with LLAMA_SLEEP_EVICT_KV also the
|
||||||
|
// KV/recurrent state, scheduler and backend scratch) and drop the VRAM token so another model
|
||||||
|
// can warm up. Called on the loop thread from handle_sleeping_state(true): the queue is idle and
|
||||||
|
// another process waits for the token.
|
||||||
|
void vram_go_cold() {
|
||||||
|
if (!vram_only || vram_cold) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
llama_context_release_device(ctx_tgt, vram_evict_kv);
|
||||||
|
if (ctx_dft != nullptr) {
|
||||||
|
llama_context_release_device(ctx_dft, vram_evict_kv);
|
||||||
|
}
|
||||||
|
// also release the multimodal (vision/audio) encoder weights (dead weight while cold; a
|
||||||
|
// no-op when LLAMA_MMPROJ_ONDEMAND already keeps them out). Their read-only host shadow is
|
||||||
|
// captured once. vram_ensure_warm() does not bring them back: encode_mtmd_chunk() restores
|
||||||
|
// the encoder only for the duration of an encode.
|
||||||
|
if (mctx != nullptr) {
|
||||||
|
mtmd_release_device(mctx);
|
||||||
|
}
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
if (vram_flock) {
|
||||||
|
flock(vram_lock_fd, LOCK_UN);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
vram_cold = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// the model file and every parameter that changes the state layout or the KV values
|
||||||
|
std::string disk_cache_key() const {
|
||||||
|
std::ostringstream key;
|
||||||
|
const auto add_file = [&](const std::string & path) {
|
||||||
|
std::error_code ec;
|
||||||
|
const auto size = std::filesystem::file_size(path, ec);
|
||||||
|
const auto mtime = std::filesystem::last_write_time(path, ec).time_since_epoch().count();
|
||||||
|
key << path << " " << size << " " << (long long) mtime << "\n";
|
||||||
|
};
|
||||||
|
key << "commit " << llama_commit() << "\n";
|
||||||
|
add_file(params_base.model.path);
|
||||||
|
for (const auto & la : params_base.lora_adapters) {
|
||||||
|
add_file(la.path);
|
||||||
|
key << la.scale << "\n";
|
||||||
|
}
|
||||||
|
for (const auto & cv : params_base.control_vectors) {
|
||||||
|
add_file(cv.fname);
|
||||||
|
key << cv.strength << "\n";
|
||||||
|
}
|
||||||
|
key << llama_n_ctx(ctx_tgt) << " " << llama_n_ctx_seq(ctx_tgt) << " " << llama_n_seq_max(ctx_tgt) << " " << llama_n_ubatch(ctx_tgt) << "\n";
|
||||||
|
key << params_base.kv_unified << " " << params_base.swa_full << " " << ggml_type_name(params_base.cache_type_k) << " "
|
||||||
|
<< ggml_type_name(params_base.cache_type_v) << " " << llama_flash_attn_type_name(params_base.flash_attn_type) << "\n";
|
||||||
|
key << std::hexfloat << (int) params_base.rope_scaling_type << " " << params_base.rope_freq_base << " " << params_base.rope_freq_scale << " "
|
||||||
|
<< params_base.yarn_ext_factor << " " << params_base.yarn_attn_factor << " " << params_base.yarn_beta_fast << " "
|
||||||
|
<< params_base.yarn_beta_slow << " " << params_base.yarn_orig_ctx << "\n";
|
||||||
|
return key.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
// on shutdown, the loop has stopped: copy the state of each slot with a prompt to host memory
|
||||||
|
// while it is still readable. disk_cache_flush() writes them once the GPU token is released.
|
||||||
|
void disk_cache_collect() {
|
||||||
|
if (!disk_cache || ctx_tgt == nullptr) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
std::vector<server_slot *> todo;
|
||||||
|
for (auto & slot : slots) {
|
||||||
|
llama_tokens tokens;
|
||||||
|
if (!disk_cache_text(slot.prompt.tokens, tokens) || tokens.size() < DISK_CACHE_MIN_TOKENS) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// the last batch is not decoded if the shutdown came during a token wait
|
||||||
|
if (llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id) + 1 != (llama_pos) tokens.size()) {
|
||||||
|
SLT_WRN(slot, "disk cache: the state does not match the %zu prompt tokens, not saved\n", tokens.size());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (disk_cache->has(tokens)) {
|
||||||
|
SLT_INF(slot, "disk cache: the prompt of %zu tokens is already saved\n", tokens.size());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
todo.push_back(&slot);
|
||||||
|
}
|
||||||
|
if (todo.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// cold: the state is in a host shadow (LLAMA_SLEEP_EVICT_KV) or still in VRAM, read it without the device
|
||||||
|
if (vram_only && vram_cold && !llama_context_restore_memory_host(ctx_tgt)) {
|
||||||
|
SRV_WRN("%s", "disk cache: cannot read the released state, not saved\n");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (auto * pslot : todo) {
|
||||||
|
auto & slot = *pslot;
|
||||||
|
const int64_t t_start = ggml_time_us();
|
||||||
|
server_prompt_cache_state st;
|
||||||
|
try {
|
||||||
|
st.data.main.resize(llama_state_seq_get_size_ext(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE));
|
||||||
|
} catch (const std::bad_alloc &) {
|
||||||
|
SLT_WRN(slot, "%s", "disk cache: out of memory, not saved\n");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (st.data.main.empty() || llama_state_seq_get_data_ext(ctx_tgt, st.data.main.data(), st.data.main.size(), slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) != st.data.main.size()) {
|
||||||
|
SLT_WRN(slot, "%s", "disk cache: failed to copy the state, not saved\n");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
st.prompt = std::move(slot.prompt);
|
||||||
|
SLT_INF(slot, "disk cache: copied the state of %d tokens (%.1f MiB, %zu checkpoints) in %.0f ms\n",
|
||||||
|
st.prompt.n_tokens(), st.size() / (1024.0 * 1024.0), st.prompt.checkpoints.size(), (ggml_time_us() - t_start) / 1000.0);
|
||||||
|
disk_pending.push_back(std::move(st));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void disk_cache_flush() {
|
||||||
|
for (const auto & st : disk_pending) {
|
||||||
|
const int64_t t_start = ggml_time_us();
|
||||||
|
if (disk_cache->save(st.prompt, st.data.main)) {
|
||||||
|
SRV_INF("disk cache: saved the prompt of %d tokens (%.1f MiB) in %.0f ms\n",
|
||||||
|
st.prompt.n_tokens(), st.size() / (1024.0 * 1024.0), (ggml_time_us() - t_start) / 1000.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
disk_pending.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
// replace the slot prompt with a disk cache entry that reuses at least DISK_CACHE_MIN_TOKENS more of the new prompt
|
||||||
|
void disk_cache_load(server_slot & slot, const server_tokens & tokens_new) {
|
||||||
|
if (vram_only && vram_cold) {
|
||||||
|
return; // restore only while holding the GPU
|
||||||
|
}
|
||||||
|
llama_tokens tokens;
|
||||||
|
if (!disk_cache_text(tokens_new, tokens)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// a recurrent or SWA state cannot be cut back to an arbitrary prefix (see update_slots)
|
||||||
|
const bool can_trim = ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_PART && n_swa == 0;
|
||||||
|
std::vector<int64_t> ckpt_n_tokens;
|
||||||
|
for (const auto & c : slot.prompt.checkpoints) {
|
||||||
|
ckpt_n_tokens.push_back(c.n_tokens);
|
||||||
|
}
|
||||||
|
llama_tokens tokens_slot;
|
||||||
|
const size_t n_have = disk_cache_text(slot.prompt.tokens, tokens_slot) ?
|
||||||
|
server_prompt_disk_cache::n_reuse(tokens_slot, ckpt_n_tokens, tokens, can_trim) : slot.prompt.tokens.get_common_prefix(tokens_new);
|
||||||
|
const int i = disk_cache->find(tokens, n_have, DISK_CACHE_MIN_TOKENS, can_trim);
|
||||||
|
if (i < 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const std::string path = disk_cache->entries[i].path;
|
||||||
|
const size_t n_reuse = server_prompt_disk_cache::n_reuse(disk_cache->entries[i].tokens, disk_cache->entries[i].ckpt_n_tokens, tokens, can_trim);
|
||||||
|
|
||||||
|
const int64_t t_start = ggml_time_us();
|
||||||
|
server_prompt prompt;
|
||||||
|
std::vector<uint8_t> data;
|
||||||
|
if (!disk_cache->load(i, prompt, data, mctx != nullptr)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const int64_t t_read = ggml_time_us();
|
||||||
|
|
||||||
|
// keep the current slot prompt in the RAM cache, as get_available_slot() does
|
||||||
|
if (prompt_cache && slot.prompt_save(*prompt_cache)) {
|
||||||
|
prompt_cache->update();
|
||||||
|
}
|
||||||
|
slot.prompt_clear();
|
||||||
|
if (llama_state_seq_set_data_ext(ctx_tgt, data.data(), data.size(), slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) != data.size()) {
|
||||||
|
SLT_WRN(slot, "disk cache: failed to restore the state from '%s', removing it\n", path.c_str());
|
||||||
|
slot.prompt_clear();
|
||||||
|
disk_cache->remove(i);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
while (prompt.checkpoints.size() > (size_t) std::max(0, params_base.n_ctx_checkpoints)) {
|
||||||
|
prompt.checkpoints.pop_front();
|
||||||
|
}
|
||||||
|
size_t size = data.size();
|
||||||
|
for (const auto & c : prompt.checkpoints) {
|
||||||
|
size += c.size();
|
||||||
|
}
|
||||||
|
slot.prompt = std::move(prompt);
|
||||||
|
SLT_INF(slot, "disk cache: restored %d tokens (%.1f MiB, %zu checkpoints) from '%s', read %.0f ms, restore %.0f ms, %zu of %zu prompt tokens reusable (had %zu)\n",
|
||||||
|
slot.prompt.n_tokens(), size / (1024.0 * 1024.0), slot.prompt.checkpoints.size(), path.c_str(),
|
||||||
|
(t_read - t_start) / 1000.0, (ggml_time_us() - t_read) / 1000.0, n_reuse, tokens.size(), n_have);
|
||||||
|
}
|
||||||
|
|
||||||
common_speculative_init_result_ptr spec_init;
|
common_speculative_init_result_ptr spec_init;
|
||||||
|
|
||||||
common_context_seq_rm_type ctx_tgt_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
|
common_context_seq_rm_type ctx_tgt_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
|
||||||
@@ -1096,6 +1566,10 @@ private:
|
|||||||
|
|
||||||
std::unique_ptr<server_prompt_cache> prompt_cache;
|
std::unique_ptr<server_prompt_cache> prompt_cache;
|
||||||
|
|
||||||
|
// prompt states saved to disk on shutdown and restored after a restart (--cache-disk-path)
|
||||||
|
std::unique_ptr<server_prompt_disk_cache> disk_cache;
|
||||||
|
std::vector<server_prompt_cache_state> disk_pending; // copied before the device buffers are freed, written after
|
||||||
|
|
||||||
server_metrics metrics;
|
server_metrics metrics;
|
||||||
|
|
||||||
// queued prompt stats - llama_decode() is async, so the timing is only valid after a sync
|
// queued prompt stats - llama_decode() is async, so the timing is only valid after a sync
|
||||||
@@ -1135,6 +1609,32 @@ private:
|
|||||||
|
|
||||||
void handle_sleeping_state(bool new_state) {
|
void handle_sleeping_state(bool new_state) {
|
||||||
GGML_ASSERT(sleeping != new_state);
|
GGML_ASSERT(sleeping != new_state);
|
||||||
|
|
||||||
|
// Lightweight VRAM-only sleep: instead of a full unload/reload (which frees RAM and the
|
||||||
|
// KV cache and pays a full reload on wake), just release the model's device (VRAM) buffers
|
||||||
|
// to a host shadow (keeping the context, the prompt cache and the host weights; the KV stays
|
||||||
|
// on the device unless LLAMA_SLEEP_EVICT_KV is set) and drop the shared VRAM token. Waking
|
||||||
|
// restores eagerly, on the wake transition below: vram_ensure_warm() re-acquires the token
|
||||||
|
// and restores before update_slots() runs. A single GPU is so time-shared between models
|
||||||
|
// with no re-prefill.
|
||||||
|
if (vram_only && ctx_tgt != nullptr) {
|
||||||
|
if (new_state) {
|
||||||
|
SRV_INF("server entering sleeping state (VRAM-only: releasing device weights, %s)\n",
|
||||||
|
vram_evict_kv ? "evicting KV cache to host" : "keeping KV cache in VRAM");
|
||||||
|
vram_go_cold();
|
||||||
|
} else {
|
||||||
|
SRV_INF("%s", "server exiting sleeping state (VRAM-only: restoring device weights/KV)\n");
|
||||||
|
// Restore NOW, on wake, before update_slots runs. update_slots touches the KV cache
|
||||||
|
// (e.g. SWA checkpoint creation reads it via ggml_backend_tensor_get) before the
|
||||||
|
// decode-time vram_ensure_warm(), so with KV eviction the KV must already be resident
|
||||||
|
// here or those reads hit a freed (null) buffer.
|
||||||
|
// false (still cold) only on shutdown: the loop then exits before update_slots
|
||||||
|
vram_ensure_warm();
|
||||||
|
}
|
||||||
|
sleeping = new_state;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (new_state) {
|
if (new_state) {
|
||||||
if (callback_state) {
|
if (callback_state) {
|
||||||
callback_state(SERVER_STATE_SLEEPING, {});
|
callback_state(SERVER_STATE_SLEEPING, {});
|
||||||
@@ -1287,6 +1787,11 @@ private:
|
|||||||
params_base.load_progress_callback_user_data = &load_progress_text;
|
params_base.load_progress_callback_user_data = &load_progress_text;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// VRAM arbiter: acquire the GPU token before uploading weights, so any resident model (e.g.
|
||||||
|
// the warm 4B task model) releases first and this load uploads into free VRAM instead of
|
||||||
|
// racing it and OOM-ing. No-op unless LLAMA_SLEEP_VRAM_ONLY is set.
|
||||||
|
vram_acquire_for_load();
|
||||||
|
|
||||||
llama_init = common_init_from_params(params_base);
|
llama_init = common_init_from_params(params_base);
|
||||||
|
|
||||||
model_tgt = llama_init->model();
|
model_tgt = llama_init->model();
|
||||||
@@ -1386,6 +1891,13 @@ private:
|
|||||||
init_opt.video_params.ffmpeg_bin_dir = params_base.video_ffmpeg_bin_dir.empty()
|
init_opt.video_params.ffmpeg_bin_dir = params_base.video_ffmpeg_bin_dir.empty()
|
||||||
? nullptr : params_base.video_ffmpeg_bin_dir.c_str();
|
? nullptr : params_base.video_ffmpeg_bin_dir.c_str();
|
||||||
|
|
||||||
|
// on-demand encoder: keep the vision/audio encoder weights in RAM (out of VRAM) until an
|
||||||
|
// image/audio actually needs encoding, freeing that VRAM for KV / expert cache on the
|
||||||
|
// common text-only path. encode_mtmd_chunk() brings the encoder in just-in-time.
|
||||||
|
if (getenv("LLAMA_MMPROJ_ONDEMAND") != nullptr) {
|
||||||
|
mtmd_release_device(mctx);
|
||||||
|
}
|
||||||
|
|
||||||
if (params_base.ctx_shift) {
|
if (params_base.ctx_shift) {
|
||||||
params_base.ctx_shift = false;
|
params_base.ctx_shift = false;
|
||||||
SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");
|
SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");
|
||||||
@@ -1586,8 +2098,18 @@ private:
|
|||||||
} else {
|
} else {
|
||||||
SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
|
SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
SRV_TRC("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n");
|
SRV_TRC("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n");
|
||||||
|
|
||||||
|
disk_cache.reset();
|
||||||
|
if (!params_base.cache_disk_path.empty()) {
|
||||||
|
if (spec) {
|
||||||
|
SRV_WRN("%s", "disk cache: not supported with speculative decoding, disabled\n");
|
||||||
|
} else {
|
||||||
|
disk_cache = std::make_unique<server_prompt_disk_cache>(params_base.cache_disk_path, params_base.cache_disk_mib, disk_cache_key());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (params_base.n_ctx_checkpoints > 0) {
|
if (params_base.n_ctx_checkpoints > 0) {
|
||||||
SRV_TRC("context checkpoints enabled, max = %d, min spacing = %d\n",
|
SRV_TRC("context checkpoints enabled, max = %d, min spacing = %d\n",
|
||||||
params_base.n_ctx_checkpoints, params_base.checkpoint_min_step);
|
params_base.n_ctx_checkpoints, params_base.checkpoint_min_step);
|
||||||
@@ -1641,6 +2163,11 @@ private:
|
|||||||
handle_sleeping_state(sleeping);
|
handle_sleeping_state(sleeping);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// VRAM arbiter: arm the yield probe. The load was coordinated (vram_acquire_for_load
|
||||||
|
// grabbed the token before uploading), so we stay warm holding the token and serve the
|
||||||
|
// first request without a re-warm; the loop releases it when another model waits for it.
|
||||||
|
vram_share_init();
|
||||||
|
|
||||||
metrics.init();
|
metrics.init();
|
||||||
|
|
||||||
if (params_base.cache_idle_slots) {
|
if (params_base.cache_idle_slots) {
|
||||||
@@ -1876,6 +2403,10 @@ private:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (ret && disk_cache && !ret->is_processing() && task.params.cache_prompt && !task.is_stateless()) {
|
||||||
|
disk_cache_load(*ret, task.tokens);
|
||||||
|
}
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4576,6 +5107,13 @@ private:
|
|||||||
has_output |= batch.tokens[i].output;
|
has_output |= batch.tokens[i].output;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// VRAM arbiter: make sure our weights are resident (acquiring the shared GPU token first)
|
||||||
|
// before we decode. Runs on the loop thread before entering yield_to_queue(), so it never
|
||||||
|
// races an in-flight decode.
|
||||||
|
if (!vram_ensure_warm()) {
|
||||||
|
throw std::runtime_error("server is shutting down");
|
||||||
|
}
|
||||||
|
|
||||||
// yield to the queue, so we can still handle metrics tasks while decoding
|
// yield to the queue, so we can still handle metrics tasks while decoding
|
||||||
// note: the sync is done here too, so that the wait is also covered by the yield
|
// note: the sync is done here too, so that the wait is also covered by the yield
|
||||||
int ret = 0;
|
int ret = 0;
|
||||||
@@ -5092,9 +5630,15 @@ bool server_context::load_model(common_params & params) {
|
|||||||
void server_context::start_loop() {
|
void server_context::start_loop() {
|
||||||
auto & params = impl->params_base;
|
auto & params = impl->params_base;
|
||||||
impl->queue_tasks.start_loop(params.sleep_idle_seconds * 1000);
|
impl->queue_tasks.start_loop(params.sleep_idle_seconds * 1000);
|
||||||
|
// copy the slot states while the device buffers exist, write them after the GPU token is released
|
||||||
|
impl->disk_cache_collect();
|
||||||
|
impl->vram_share_exit();
|
||||||
|
impl->disk_cache_flush();
|
||||||
}
|
}
|
||||||
|
|
||||||
void server_context::terminate() {
|
void server_context::terminate() {
|
||||||
|
// wake the HTTP threads waiting for results first: the loop will not produce them any more
|
||||||
|
impl->queue_results.terminate();
|
||||||
impl->queue_tasks.terminate();
|
impl->queue_tasks.terminate();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5162,8 +5706,12 @@ struct server_res_generator : server_res_spipe {
|
|||||||
server_response_reader rd;
|
server_response_reader rd;
|
||||||
server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false)
|
server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false)
|
||||||
: rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) {
|
: rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) {
|
||||||
// fast path in case sleeping is disabled
|
// fast path in case sleeping is disabled. Note: the VRAM arbiter (LLAMA_SLEEP_VRAM_ONLY)
|
||||||
bypass_sleep |= sleep_idle_seconds < 0;
|
// can put the server to sleep to hand over the GPU token even when idle-sleep is disabled
|
||||||
|
// (sleep_idle_seconds < 0), so in that case requests must still wake it - otherwise an
|
||||||
|
// arbiter-slept server would hang, never returning from a request.
|
||||||
|
static const bool vram_arbiter = getenv("LLAMA_SLEEP_VRAM_ONLY") != nullptr;
|
||||||
|
bypass_sleep |= (sleep_idle_seconds < 0 && !vram_arbiter);
|
||||||
if (!bypass_sleep) {
|
if (!bypass_sleep) {
|
||||||
queue_tasks.wait_until_no_sleep();
|
queue_tasks.wait_until_no_sleep();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,22 +114,22 @@ void server_queue::pop_deferred_task(int id_slot) {
|
|||||||
|
|
||||||
void server_queue::wait_until_no_sleep() {
|
void server_queue::wait_until_no_sleep() {
|
||||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||||
if (!sleeping) {
|
// re-ask on every wake: the loop clears req_stop_sleeping on the way into sleep, so a loop that
|
||||||
return;
|
// goes back to sleep before we run would strand us here forever
|
||||||
} else {
|
// a loop that terminates while asleep never wakes: give up then, the reader reports the shutdown
|
||||||
|
while (sleeping && !stopping) {
|
||||||
if (!req_stop_sleeping) {
|
if (!req_stop_sleeping) {
|
||||||
QUE_DBG("%s", "requesting to stop sleeping\n");
|
QUE_DBG("%s", "requesting to stop sleeping\n");
|
||||||
req_stop_sleeping = true;
|
req_stop_sleeping = true;
|
||||||
condition_tasks.notify_one(); // only main thread is waiting on this
|
condition_tasks.notify_all(); // other threads may wait on this too
|
||||||
}
|
}
|
||||||
QUE_DBG("%s", "waiting until no sleep\n");
|
QUE_DBG("%s", "waiting until no sleep\n");
|
||||||
condition_tasks.wait(lock, [&]{
|
condition_tasks.wait(lock);
|
||||||
return !sleeping;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void server_queue::terminate() {
|
void server_queue::terminate() {
|
||||||
|
stopping = true; // before the lock: the loop may hold it while it waits for the VRAM token
|
||||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||||
running = false;
|
running = false;
|
||||||
condition_tasks.notify_all();
|
condition_tasks.notify_all();
|
||||||
@@ -138,7 +138,7 @@ void server_queue::terminate() {
|
|||||||
bool server_queue::process_new_tasks(bool is_yielding) {
|
bool server_queue::process_new_tasks(bool is_yielding) {
|
||||||
while (true) {
|
while (true) {
|
||||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||||
if (!running) {
|
if (!running || stopping) {
|
||||||
QUE_DBG("%s", "terminate\n");
|
QUE_DBG("%s", "terminate\n");
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -286,9 +286,18 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
|
|||||||
worker.yielding = false;
|
worker.yielding = false;
|
||||||
worker.thread = std::thread([this]() { worker_loop(); });
|
worker.thread = std::thread([this]() { worker_loop(); });
|
||||||
|
|
||||||
constexpr auto max_wait_time = std::chrono::seconds(1);
|
// the arbiter predicate is read from live state, so this timeout bounds how fast we notice a
|
||||||
|
// process waiting for the VRAM token
|
||||||
|
const auto max_wait_time = should_yield_cb ? std::chrono::milliseconds(10) : std::chrono::milliseconds(1000);
|
||||||
|
// after a wake, keep the VRAM for at least this long: the request that woke us is not in the
|
||||||
|
// queue yet, so yielding at once would only send it through another wake
|
||||||
|
constexpr int64_t yield_grace_ms = 100;
|
||||||
|
int64_t time_last_wake = 0;
|
||||||
auto should_sleep = [&]() -> bool {
|
auto should_sleep = [&]() -> bool {
|
||||||
// caller must hold mutex_tasks
|
// caller must hold mutex_tasks
|
||||||
|
if (should_yield_cb && ggml_time_ms() - time_last_wake >= yield_grace_ms && should_yield_cb()) {
|
||||||
|
return true; // another process waits for the VRAM token - release now
|
||||||
|
}
|
||||||
if (idle_sleep_ms < 0) {
|
if (idle_sleep_ms < 0) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -332,9 +341,11 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
|
|||||||
cb(true);
|
cb(true);
|
||||||
}
|
}
|
||||||
req_stop_sleeping = false;
|
req_stop_sleeping = false;
|
||||||
// wait until we are requested to exit sleeping state
|
// wait until we are requested to exit sleeping state, or a task arrives: post() only
|
||||||
|
// notifies, so a task queued right after wait_until_no_sleep() saw us awake must be
|
||||||
|
// able to wake us by itself, else it waits here for an unrelated request
|
||||||
condition_tasks.wait(lock, [&]{
|
condition_tasks.wait(lock, [&]{
|
||||||
return (!running || req_stop_sleeping);
|
return (!running || req_stop_sleeping || !queue_tasks.empty());
|
||||||
});
|
});
|
||||||
if (!running) { // may changed during sleep
|
if (!running) { // may changed during sleep
|
||||||
break; // terminate
|
break; // terminate
|
||||||
@@ -347,17 +358,21 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
|
|||||||
}
|
}
|
||||||
sleeping = false;
|
sleeping = false;
|
||||||
time_last_task = ggml_time_ms();
|
time_last_task = ggml_time_ms();
|
||||||
|
time_last_wake = time_last_task;
|
||||||
condition_tasks.notify_all(); // notify wait_until_no_sleep()
|
condition_tasks.notify_all(); // notify wait_until_no_sleep()
|
||||||
break; // process new tasks
|
break; // process new tasks
|
||||||
} else {
|
} else {
|
||||||
// wait for new tasks or timeout for checking sleeping condition
|
// wait for new tasks, or timeout for checking sleeping condition
|
||||||
bool res = condition_tasks.wait_for(lock, max_wait_time, [&]{
|
bool res = condition_tasks.wait_for(lock, max_wait_time, [&]{
|
||||||
return (!queue_tasks.empty() || !running);
|
return (!queue_tasks.empty() || !running);
|
||||||
});
|
});
|
||||||
if (res) {
|
if (res && !queue_tasks.empty()) {
|
||||||
break; // new task arrived or terminate
|
break; // new task arrived or terminate
|
||||||
}
|
}
|
||||||
// otherwise, loop again to check sleeping condition
|
if (!running) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
// otherwise (timeout), loop again to re-check should_sleep
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -460,11 +475,12 @@ server_task_result_ptr server_response::recv_with_timeout(const std::unordered_s
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::cv_status cr_res = condition_results.wait_until(lock, deadline);
|
|
||||||
if (!running) {
|
if (!running) {
|
||||||
RES_DBG("%s : queue result stop\n", __func__);
|
RES_DBG("%s : queue result stop\n", __func__);
|
||||||
std::terminate(); // we cannot return here since the caller is HTTP code
|
return nullptr; // the caller checks is_running()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::cv_status cr_res = condition_results.wait_until(lock, deadline);
|
||||||
if (cr_res == std::cv_status::timeout) {
|
if (cr_res == std::cv_status::timeout) {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
@@ -505,6 +521,7 @@ void server_response::broadcast(server_task_result_ptr && result) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void server_response::terminate() {
|
void server_response::terminate() {
|
||||||
|
std::unique_lock<std::mutex> lock(mutex_results);
|
||||||
running = false;
|
running = false;
|
||||||
condition_results.notify_all();
|
condition_results.notify_all();
|
||||||
}
|
}
|
||||||
@@ -552,6 +569,14 @@ server_task_result_ptr server_response_reader::next(const std::function<bool()>
|
|||||||
while (true) {
|
while (true) {
|
||||||
server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
|
server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
|
||||||
if (result == nullptr) {
|
if (result == nullptr) {
|
||||||
|
if (!queue_results.is_running()) {
|
||||||
|
// the server is shutting down: no result will ever come
|
||||||
|
stop();
|
||||||
|
auto res = std::make_unique<server_task_result_error>();
|
||||||
|
res->err_type = ERROR_TYPE_UNAVAILABLE;
|
||||||
|
res->err_msg = "server is shutting down";
|
||||||
|
return res;
|
||||||
|
}
|
||||||
// timeout, check stop condition
|
// timeout, check stop condition
|
||||||
if (should_stop()) {
|
if (should_stop()) {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
#include "server-task.h"
|
#include "server-task.h"
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
#include <condition_variable>
|
#include <condition_variable>
|
||||||
#include <deque>
|
#include <deque>
|
||||||
#include <exception>
|
#include <exception>
|
||||||
@@ -16,6 +17,9 @@ struct server_queue {
|
|||||||
private:
|
private:
|
||||||
int id = 0;
|
int id = 0;
|
||||||
bool running = false;
|
bool running = false;
|
||||||
|
// set by terminate() before it takes mutex_tasks, so a loop blocked under that mutex (e.g. in a
|
||||||
|
// VRAM arbiter token wait) can see the shutdown and bail out
|
||||||
|
std::atomic<bool> stopping{false};
|
||||||
bool sleeping = false;
|
bool sleeping = false;
|
||||||
bool req_stop_sleeping = false;
|
bool req_stop_sleeping = false;
|
||||||
int64_t time_last_task = 0;
|
int64_t time_last_task = 0;
|
||||||
@@ -45,6 +49,7 @@ private:
|
|||||||
std::function<bool(server_task &&, bool)> callback_new_task;
|
std::function<bool(server_task &&, bool)> callback_new_task;
|
||||||
std::function<void(void)> callback_update_slots;
|
std::function<void(void)> callback_update_slots;
|
||||||
std::vector<std::function<void(bool)>> callback_sleeping_state;
|
std::vector<std::function<void(bool)>> callback_sleeping_state;
|
||||||
|
std::function<bool()> should_yield_cb;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
~server_queue() { worker_stop(); }
|
~server_queue() { worker_stop(); }
|
||||||
@@ -77,6 +82,11 @@ public:
|
|||||||
// end the start_loop routine
|
// end the start_loop routine
|
||||||
void terminate();
|
void terminate();
|
||||||
|
|
||||||
|
// true once terminate() was called; safe from any thread, including under mutex_tasks
|
||||||
|
bool is_stopping() const {
|
||||||
|
return stopping.load();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Main loop consists of these steps:
|
* Main loop consists of these steps:
|
||||||
* - Wait until a new task arrives
|
* - Wait until a new task arrives
|
||||||
@@ -127,6 +137,14 @@ public:
|
|||||||
callback_update_slots = std::move(callback);
|
callback_update_slots = std::move(callback);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Register a predicate asking the loop to sleep (release VRAM) as soon as it is idle - used by
|
||||||
|
// the VRAM arbiter to check whether another process waits for the GPU token.
|
||||||
|
// Called on the start_loop() thread while holding mutex_tasks, so it must not block or post
|
||||||
|
// tasks. While it is set, the idle wait polls faster, so the loop reacts without a notify.
|
||||||
|
void on_should_yield(std::function<bool()> callback) {
|
||||||
|
should_yield_cb = std::move(callback);
|
||||||
|
}
|
||||||
|
|
||||||
// Register callback for sleeping state change; multiple callbacks are allowed
|
// Register callback for sleeping state change; multiple callbacks are allowed
|
||||||
// for example: register order cb0, cb1, cb2
|
// for example: register order cb0, cb1, cb2
|
||||||
// entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true)
|
// entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true)
|
||||||
@@ -193,8 +211,13 @@ public:
|
|||||||
// (used by router mode)
|
// (used by router mode)
|
||||||
void broadcast(server_task_result_ptr && result);
|
void broadcast(server_task_result_ptr && result);
|
||||||
|
|
||||||
// terminate the waiting loop
|
// terminate the waiting loop: pending and later readers get an error instead of a result
|
||||||
void terminate();
|
void terminate();
|
||||||
|
|
||||||
|
bool is_running() {
|
||||||
|
std::unique_lock<std::mutex> lock(mutex_results);
|
||||||
|
return running;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// RAII wrapper to make working with server_queue and server_response easier
|
// RAII wrapper to make working with server_queue and server_response easier
|
||||||
|
|||||||
@@ -10,8 +10,19 @@
|
|||||||
#include "speculative.h"
|
#include "speculative.h"
|
||||||
#include "server-common.h"
|
#include "server-common.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cerrno>
|
||||||
|
#include <chrono>
|
||||||
|
#include <cstdio>
|
||||||
|
#include <cstring>
|
||||||
|
#include <filesystem>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
|
|
||||||
|
#if defined(__linux__)
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
//
|
//
|
||||||
// task_params
|
// task_params
|
||||||
//
|
//
|
||||||
@@ -1934,3 +1945,441 @@ void server_prompt_cache::update() {
|
|||||||
(const void *)&state, state.prompt.n_tokens(), state.prompt.checkpoints.size(), state.size() / (1024.0 * 1024.0));
|
(const void *)&state, state.prompt.n_tokens(), state.prompt.checkpoints.size(), state.size() / (1024.0 * 1024.0));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// server_prompt_disk_cache
|
||||||
|
//
|
||||||
|
|
||||||
|
// file = header + payload; the payload is a list of sections, each padded to 8 bytes:
|
||||||
|
// tokens, checkpoint table, state size, state, the state of each checkpoint
|
||||||
|
// hash_payload is FNV-1a over the 64-bit words of the payload, so a truncated or corrupt file is never used
|
||||||
|
static constexpr uint32_t DISK_CACHE_MAGIC = 0x4443504c; // "LPCD"
|
||||||
|
static constexpr uint32_t DISK_CACHE_VERSION = 1;
|
||||||
|
static constexpr const char * DISK_CACHE_EXT = ".lpcd";
|
||||||
|
|
||||||
|
// keep this much space free on the file system
|
||||||
|
static constexpr uint64_t DISK_CACHE_MIN_FREE = 1024ull*1024ull*1024ull;
|
||||||
|
|
||||||
|
struct disk_cache_header {
|
||||||
|
uint32_t magic;
|
||||||
|
uint32_t version;
|
||||||
|
uint64_t key_hash;
|
||||||
|
uint64_t n_tokens;
|
||||||
|
uint64_t n_ckpt;
|
||||||
|
uint64_t size_payload;
|
||||||
|
uint64_t hash_payload;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct disk_cache_ckpt {
|
||||||
|
int64_t n_tokens;
|
||||||
|
llama_pos pos_min;
|
||||||
|
llama_pos pos_max;
|
||||||
|
uint64_t size;
|
||||||
|
};
|
||||||
|
|
||||||
|
static_assert(sizeof(disk_cache_header) == 48 && sizeof(disk_cache_ckpt) == 24, "disk cache layout");
|
||||||
|
|
||||||
|
static uint64_t disk_cache_pad(uint64_t n) {
|
||||||
|
return (8 - n % 8) % 8;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint64_t disk_cache_hash_bytes(const void * data, size_t n) {
|
||||||
|
uint64_t h = 0xcbf29ce484222325ull;
|
||||||
|
for (size_t i = 0; i < n; i++) {
|
||||||
|
h = (h ^ ((const uint8_t *) data)[i]) * 0x100000001b3ull;
|
||||||
|
}
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint64_t disk_cache_payload_size(uint64_t n_tokens, const std::vector<disk_cache_ckpt> & ckpts, uint64_t n_state) {
|
||||||
|
const auto padded = [](uint64_t n) { return n + disk_cache_pad(n); };
|
||||||
|
uint64_t res = padded(n_tokens*sizeof(llama_token)) + padded(ckpts.size()*sizeof(disk_cache_ckpt)) + sizeof(uint64_t) + padded(n_state);
|
||||||
|
for (const auto & c : ckpts) {
|
||||||
|
res += padded(c.size);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
// sequential section IO with the running payload hash
|
||||||
|
struct disk_cache_io {
|
||||||
|
FILE * f = nullptr;
|
||||||
|
uint64_t hash = 0xcbf29ce484222325ull;
|
||||||
|
bool ok = true;
|
||||||
|
|
||||||
|
~disk_cache_io() {
|
||||||
|
if (f) {
|
||||||
|
fclose(f);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void add_hash(const uint8_t * p, size_t size, const uint8_t * pad) {
|
||||||
|
const size_t n_full = size - size % 8;
|
||||||
|
for (size_t i = 0; i < n_full; i += 8) {
|
||||||
|
uint64_t w;
|
||||||
|
memcpy(&w, p + i, 8);
|
||||||
|
hash = (hash ^ w) * 0x100000001b3ull;
|
||||||
|
}
|
||||||
|
if (size % 8) {
|
||||||
|
uint64_t w;
|
||||||
|
memcpy(&w, p + n_full, size % 8);
|
||||||
|
memcpy((uint8_t *) &w + size % 8, pad, disk_cache_pad(size));
|
||||||
|
hash = (hash ^ w) * 0x100000001b3ull;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void write(const void * src, size_t size) {
|
||||||
|
static const uint8_t zero[8] = {};
|
||||||
|
const size_t n_pad = disk_cache_pad(size);
|
||||||
|
ok = ok && fwrite(src, 1, size, f) == size && fwrite(zero, 1, n_pad, f) == n_pad;
|
||||||
|
add_hash((const uint8_t *) src, size, zero);
|
||||||
|
}
|
||||||
|
|
||||||
|
void read(void * dst, size_t size) {
|
||||||
|
uint8_t pad[8];
|
||||||
|
const size_t n_pad = disk_cache_pad(size);
|
||||||
|
ok = ok && fread(dst, 1, size, f) == size && fread(pad, 1, n_pad, f) == n_pad;
|
||||||
|
if (ok) {
|
||||||
|
add_hash((const uint8_t *) dst, size, pad);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// the data is in our buffers, the cached pages are waste (see llama_file::drop_cache)
|
||||||
|
void drop_cache() {
|
||||||
|
#if defined(__linux__)
|
||||||
|
posix_fadvise(fileno(f), 0, 0, POSIX_FADV_DONTNEED);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// open a file and read everything up to the state data: 1 = entry of this key, 0 = other key or version, -1 = bad file
|
||||||
|
static int disk_cache_open(const std::string & path, uint64_t key_hash, disk_cache_io & io, disk_cache_header & hdr,
|
||||||
|
llama_tokens & tokens, std::vector<disk_cache_ckpt> & ckpts, uint64_t & n_state) {
|
||||||
|
std::error_code ec;
|
||||||
|
const uint64_t size = std::filesystem::file_size(path, ec);
|
||||||
|
io.f = ec ? nullptr : fopen(path.c_str(), "rb");
|
||||||
|
if (io.f == nullptr || fread(&hdr, 1, sizeof(hdr), io.f) != sizeof(hdr) || hdr.magic != DISK_CACHE_MAGIC) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (hdr.version != DISK_CACHE_VERSION) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (size != sizeof(hdr) + hdr.size_payload || hdr.n_tokens > size/sizeof(llama_token) || hdr.n_ckpt > size/sizeof(disk_cache_ckpt)) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (hdr.key_hash != key_hash) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
tokens.resize(hdr.n_tokens);
|
||||||
|
ckpts.resize(hdr.n_ckpt);
|
||||||
|
io.read(tokens.data(), tokens.size()*sizeof(llama_token));
|
||||||
|
io.read(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt));
|
||||||
|
io.read(&n_state, sizeof(n_state));
|
||||||
|
if (!io.ok || n_state > size) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
for (const auto & c : ckpts) {
|
||||||
|
if (c.size > size) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return disk_cache_payload_size(hdr.n_tokens, ckpts, n_state) == hdr.size_payload ? 1 : -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void disk_cache_touch(const std::string & path) {
|
||||||
|
std::error_code ec;
|
||||||
|
std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now(), ec);
|
||||||
|
}
|
||||||
|
|
||||||
|
server_prompt_disk_cache::server_prompt_disk_cache(const std::string & dir, int32_t limit_size_mib, const std::string & key) : dir(dir) {
|
||||||
|
key_hash = disk_cache_hash_bytes(key.data(), key.size());
|
||||||
|
limit_size = 1024ull*1024ull*std::max(0, limit_size_mib);
|
||||||
|
|
||||||
|
size_t n_other = 0;
|
||||||
|
size_t n_bad = 0;
|
||||||
|
try {
|
||||||
|
std::filesystem::create_directories(dir);
|
||||||
|
const auto now = std::filesystem::file_time_type::clock::now();
|
||||||
|
for (const auto & de : std::filesystem::directory_iterator(dir)) {
|
||||||
|
const std::string path = de.path().string();
|
||||||
|
if (!de.is_regular_file()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// left by a writer that was killed
|
||||||
|
if (path.find(std::string(DISK_CACHE_EXT) + ".tmp") != std::string::npos) {
|
||||||
|
if (now - de.last_write_time() > std::chrono::hours(1)) {
|
||||||
|
std::filesystem::remove(de.path());
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (de.path().extension() != DISK_CACHE_EXT) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
disk_cache_io io;
|
||||||
|
disk_cache_header hdr;
|
||||||
|
server_prompt_disk_entry e;
|
||||||
|
std::vector<disk_cache_ckpt> ckpts;
|
||||||
|
uint64_t n_state = 0;
|
||||||
|
const int res = disk_cache_open(path, key_hash, io, hdr, e.tokens, ckpts, n_state);
|
||||||
|
if (res < 0) {
|
||||||
|
SRV_WRN("disk cache: '%s' is truncated or corrupt, removing it\n", path.c_str());
|
||||||
|
std::filesystem::remove(de.path());
|
||||||
|
n_bad++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (res == 0) {
|
||||||
|
n_other++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
e.path = path;
|
||||||
|
for (const auto & c : ckpts) {
|
||||||
|
e.ckpt_n_tokens.push_back(c.n_tokens);
|
||||||
|
}
|
||||||
|
entries.push_back(std::move(e));
|
||||||
|
}
|
||||||
|
} catch (const std::exception & err) {
|
||||||
|
SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what());
|
||||||
|
}
|
||||||
|
|
||||||
|
SRV_INF("disk cache: '%s', %zu prompts of this model, %zu other files, %zu bad files removed, limit %.0f MiB\n",
|
||||||
|
dir.c_str(), entries.size(), n_other, n_bad, limit_size / (1024.0 * 1024.0));
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t server_prompt_disk_cache::n_reuse(const llama_tokens & tokens, const std::vector<int64_t> & ckpt_n_tokens, const llama_tokens & tokens_new, bool can_trim) {
|
||||||
|
size_t lcp = 0;
|
||||||
|
while (lcp < tokens.size() && lcp < tokens_new.size() && tokens[lcp] == tokens_new[lcp]) {
|
||||||
|
lcp++;
|
||||||
|
}
|
||||||
|
if (lcp == tokens.size() || can_trim) {
|
||||||
|
return lcp;
|
||||||
|
}
|
||||||
|
size_t res = 0;
|
||||||
|
for (const int64_t n : ckpt_n_tokens) {
|
||||||
|
if (n <= (int64_t) lcp) {
|
||||||
|
res = std::max(res, (size_t) n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
int server_prompt_disk_cache::find(const llama_tokens & tokens_new, size_t n_have, size_t n_min, bool can_trim) const {
|
||||||
|
int best = -1;
|
||||||
|
size_t n_best = 0;
|
||||||
|
for (size_t i = 0; i < entries.size(); i++) {
|
||||||
|
const size_t n = n_reuse(entries[i].tokens, entries[i].ckpt_n_tokens, tokens_new, can_trim);
|
||||||
|
if (n >= n_have + n_min && n > n_best) {
|
||||||
|
best = (int) i;
|
||||||
|
n_best = n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool server_prompt_disk_cache::has(const llama_tokens & tokens) {
|
||||||
|
for (const auto & e : entries) {
|
||||||
|
std::error_code ec;
|
||||||
|
if (e.tokens == tokens && std::filesystem::exists(e.path, ec)) {
|
||||||
|
disk_cache_touch(e.path);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void server_prompt_disk_cache::remove(int i) {
|
||||||
|
std::error_code ec;
|
||||||
|
std::filesystem::remove(entries[i].path, ec);
|
||||||
|
entries.erase(entries.begin() + i);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool server_prompt_disk_cache::load(int i, server_prompt & prompt, std::vector<uint8_t> & data, bool has_mtmd) {
|
||||||
|
const std::string path = entries[i].path;
|
||||||
|
|
||||||
|
disk_cache_io io;
|
||||||
|
disk_cache_header hdr;
|
||||||
|
llama_tokens tokens;
|
||||||
|
std::vector<disk_cache_ckpt> ckpts;
|
||||||
|
uint64_t n_state = 0;
|
||||||
|
std::list<common_prompt_checkpoint> checkpoints;
|
||||||
|
|
||||||
|
bool ok = disk_cache_open(path, key_hash, io, hdr, tokens, ckpts, n_state) == 1 && tokens == entries[i].tokens;
|
||||||
|
if (ok) {
|
||||||
|
try {
|
||||||
|
data.resize(n_state);
|
||||||
|
io.read(data.data(), data.size());
|
||||||
|
for (const auto & c : ckpts) {
|
||||||
|
auto & cur = checkpoints.emplace_back();
|
||||||
|
cur.id_task = -1; // not created by a task: may fail to load, then the prompt is re-processed
|
||||||
|
cur.n_tokens = c.n_tokens;
|
||||||
|
cur.pos_min = c.pos_min;
|
||||||
|
cur.pos_max = c.pos_max;
|
||||||
|
cur.data_tgt.resize(c.size);
|
||||||
|
io.read(cur.data_tgt.data(), cur.data_tgt.size());
|
||||||
|
}
|
||||||
|
} catch (const std::bad_alloc &) {
|
||||||
|
SRV_WRN("disk cache: out of memory reading '%s'\n", path.c_str());
|
||||||
|
data.clear();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ok = io.ok && io.hash == hdr.hash_payload;
|
||||||
|
io.drop_cache();
|
||||||
|
}
|
||||||
|
if (!ok) {
|
||||||
|
SRV_WRN("disk cache: '%s' is missing, truncated or corrupt, removing it\n", path.c_str());
|
||||||
|
data.clear();
|
||||||
|
remove(i);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
disk_cache_touch(path);
|
||||||
|
|
||||||
|
prompt.tokens = server_tokens(tokens, has_mtmd);
|
||||||
|
prompt.checkpoints = std::move(checkpoints);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool server_prompt_disk_cache::save(const server_prompt & prompt, const std::vector<uint8_t> & data) {
|
||||||
|
// text only, see disk_cache_collect
|
||||||
|
const llama_tokens tokens = prompt.tokens.get_text_tokens();
|
||||||
|
|
||||||
|
std::vector<disk_cache_ckpt> ckpts;
|
||||||
|
for (const auto & c : prompt.checkpoints) {
|
||||||
|
ckpts.push_back({ c.n_tokens, c.pos_min, c.pos_max, c.data_tgt.size() });
|
||||||
|
}
|
||||||
|
|
||||||
|
disk_cache_header hdr = {
|
||||||
|
/*.magic =*/ DISK_CACHE_MAGIC,
|
||||||
|
/*.version =*/ DISK_CACHE_VERSION,
|
||||||
|
/*.key_hash =*/ key_hash,
|
||||||
|
/*.n_tokens =*/ tokens.size(),
|
||||||
|
/*.n_ckpt =*/ ckpts.size(),
|
||||||
|
/*.size_payload =*/ disk_cache_payload_size(tokens.size(), ckpts, data.size()),
|
||||||
|
/*.hash_payload =*/ 0,
|
||||||
|
};
|
||||||
|
const uint64_t size_file = sizeof(hdr) + hdr.size_payload;
|
||||||
|
|
||||||
|
char name[64];
|
||||||
|
snprintf(name, sizeof(name), "%016llx-%016llx", (unsigned long long) key_hash,
|
||||||
|
(unsigned long long) disk_cache_hash_bytes(tokens.data(), tokens.size()*sizeof(llama_token)));
|
||||||
|
const std::string path = (std::filesystem::path(dir) / name).string() + DISK_CACHE_EXT;
|
||||||
|
const std::string path_tmp = path + ".tmp";
|
||||||
|
|
||||||
|
// entries contained in the new one are obsolete: removed once it is written, not counted for the limit
|
||||||
|
const auto is_obsolete = [&](const server_prompt_disk_entry & e) {
|
||||||
|
return e.path != path && e.tokens.size() <= tokens.size() && std::equal(e.tokens.begin(), e.tokens.end(), tokens.begin());
|
||||||
|
};
|
||||||
|
std::vector<std::string> obsolete;
|
||||||
|
for (const auto & e : entries) {
|
||||||
|
if (is_obsolete(e)) {
|
||||||
|
obsolete.push_back(e.path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (limit_size > 0 && size_file > limit_size) {
|
||||||
|
SRV_WRN("disk cache: prompt state of %.1f MiB exceeds the limit of %.1f MiB, not saved\n", size_file / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// evict the least recently used files of every model in dir
|
||||||
|
struct file_info {
|
||||||
|
std::filesystem::path path;
|
||||||
|
uint64_t size;
|
||||||
|
std::filesystem::file_time_type time;
|
||||||
|
};
|
||||||
|
std::vector<file_info> files;
|
||||||
|
uint64_t size_total = 0;
|
||||||
|
for (const auto & de : std::filesystem::directory_iterator(dir)) {
|
||||||
|
if (de.is_regular_file() && de.path().extension() == DISK_CACHE_EXT && de.path().string() != path &&
|
||||||
|
std::find(obsolete.begin(), obsolete.end(), de.path().string()) == obsolete.end()) {
|
||||||
|
files.push_back({ de.path(), de.file_size(), de.last_write_time() });
|
||||||
|
size_total += files.back().size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::sort(files.begin(), files.end(), [](const file_info & a, const file_info & b) { return a.time < b.time; });
|
||||||
|
for (size_t i = 0; limit_size > 0 && size_total + size_file > limit_size && i < files.size(); i++) {
|
||||||
|
SRV_INF("disk cache: size limit, removing '%s' (%.1f MiB)\n", files[i].path.string().c_str(), files[i].size / (1024.0 * 1024.0));
|
||||||
|
std::filesystem::remove(files[i].path);
|
||||||
|
size_total -= files[i].size;
|
||||||
|
for (size_t j = 0; j < entries.size(); j++) {
|
||||||
|
if (entries[j].path == files[i].path.string()) {
|
||||||
|
entries.erase(entries.begin() + j);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const uint64_t avail = std::filesystem::space(dir).available;
|
||||||
|
if (avail < size_file + DISK_CACHE_MIN_FREE) {
|
||||||
|
SRV_WRN("disk cache: only %.1f MiB free in '%s', prompt state of %.1f MiB not saved\n", avail / (1024.0 * 1024.0), dir.c_str(), size_file / (1024.0 * 1024.0));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} catch (const std::exception & err) {
|
||||||
|
SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ok;
|
||||||
|
{
|
||||||
|
disk_cache_io io;
|
||||||
|
io.f = fopen(path_tmp.c_str(), "wb");
|
||||||
|
ok = io.f != nullptr && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr);
|
||||||
|
if (ok) {
|
||||||
|
const uint64_t n_state = data.size();
|
||||||
|
io.write(tokens.data(), tokens.size()*sizeof(llama_token));
|
||||||
|
io.write(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt));
|
||||||
|
io.write(&n_state, sizeof(n_state));
|
||||||
|
io.write(data.data(), data.size());
|
||||||
|
for (const auto & c : prompt.checkpoints) {
|
||||||
|
io.write(c.data_tgt.data(), c.data_tgt.size());
|
||||||
|
}
|
||||||
|
hdr.hash_payload = io.hash;
|
||||||
|
ok = io.ok && fseek(io.f, 0, SEEK_SET) == 0 && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr) && fflush(io.f) == 0;
|
||||||
|
#if defined(__linux__)
|
||||||
|
// write back now, so the pages can be dropped and the rename never exposes missing data
|
||||||
|
ok = ok && fdatasync(fileno(io.f)) == 0;
|
||||||
|
io.drop_cache();
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
if (io.f != nullptr && fclose(io.f) != 0) {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
io.f = nullptr;
|
||||||
|
if (!ok) {
|
||||||
|
SRV_WRN("disk cache: failed to write '%s': %s, not saved\n", path_tmp.c_str(), strerror(errno));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::error_code ec;
|
||||||
|
if (ok) {
|
||||||
|
std::filesystem::rename(path_tmp, path, ec);
|
||||||
|
ok = !ec;
|
||||||
|
if (!ok) {
|
||||||
|
SRV_WRN("disk cache: failed to rename '%s': %s, not saved\n", path_tmp.c_str(), ec.message().c_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!ok) {
|
||||||
|
std::filesystem::remove(path_tmp, ec);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (size_t i = 0; i < entries.size(); ) {
|
||||||
|
const auto & e = entries[i];
|
||||||
|
if (e.path == path) {
|
||||||
|
entries.erase(entries.begin() + i);
|
||||||
|
} else if (is_obsolete(e)) {
|
||||||
|
SRV_INF("disk cache: removing '%s', a prefix of the new prompt\n", e.path.c_str());
|
||||||
|
remove((int) i);
|
||||||
|
} else {
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server_prompt_disk_entry e;
|
||||||
|
e.path = path;
|
||||||
|
e.tokens = tokens;
|
||||||
|
for (const auto & c : ckpts) {
|
||||||
|
e.ckpt_n_tokens.push_back(c.n_tokens);
|
||||||
|
}
|
||||||
|
entries.push_back(std::move(e));
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -719,6 +719,42 @@ struct server_prompt_cache {
|
|||||||
void update();
|
void update();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// prompt states saved as files, to reuse a prompt after a server restart
|
||||||
|
struct server_prompt_disk_entry {
|
||||||
|
std::string path;
|
||||||
|
llama_tokens tokens;
|
||||||
|
std::vector<int64_t> ckpt_n_tokens;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct server_prompt_disk_cache {
|
||||||
|
// key: model file and every parameter that changes the state layout, see the server context
|
||||||
|
server_prompt_disk_cache(const std::string & dir, int32_t limit_size_mib, const std::string & key);
|
||||||
|
|
||||||
|
std::string dir;
|
||||||
|
uint64_t key_hash;
|
||||||
|
size_t limit_size;
|
||||||
|
|
||||||
|
// entries of this key only, read from the file headers at startup
|
||||||
|
std::vector<server_prompt_disk_entry> entries;
|
||||||
|
|
||||||
|
// tokens of tokens_new that a state of `tokens` with checkpoints at `ckpt_n_tokens` can reuse
|
||||||
|
// a recurrent or SWA state is reusable only up to an exact prefix or a checkpoint
|
||||||
|
static size_t n_reuse(const llama_tokens & tokens, const std::vector<int64_t> & ckpt_n_tokens, const llama_tokens & tokens_new, bool can_trim);
|
||||||
|
|
||||||
|
// index of the entry that reuses the most of tokens_new, and at least n_min more than n_have; -1 if none
|
||||||
|
int find(const llama_tokens & tokens_new, size_t n_have, size_t n_min, bool can_trim) const;
|
||||||
|
|
||||||
|
bool has(const llama_tokens & tokens);
|
||||||
|
|
||||||
|
// read and verify entry i; a bad file is removed
|
||||||
|
bool load(int i, server_prompt & prompt, std::vector<uint8_t> & data, bool has_mtmd);
|
||||||
|
|
||||||
|
// write atomically, after evicting the oldest files in dir over the size limit
|
||||||
|
bool save(const server_prompt & prompt, const std::vector<uint8_t> & data);
|
||||||
|
|
||||||
|
void remove(int i);
|
||||||
|
};
|
||||||
|
|
||||||
// used exclusively by router mode
|
// used exclusively by router mode
|
||||||
struct server_task_result_router : server_task_result {
|
struct server_task_result_router : server_task_result {
|
||||||
json data;
|
json data;
|
||||||
|
|||||||
@@ -14,6 +14,7 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
|
#include <cerrno>
|
||||||
#include <clocale>
|
#include <clocale>
|
||||||
#include <exception>
|
#include <exception>
|
||||||
#include <signal.h>
|
#include <signal.h>
|
||||||
@@ -21,11 +22,22 @@
|
|||||||
|
|
||||||
#if defined(_WIN32)
|
#if defined(_WIN32)
|
||||||
#include <windows.h>
|
#include <windows.h>
|
||||||
|
#else
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <unistd.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
static std::function<void(int)> shutdown_handler;
|
static std::function<void(int)> shutdown_handler;
|
||||||
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
|
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
|
||||||
|
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
// The handler can interrupt the start_loop() thread while it holds server_queue::mutex_tasks (e.g.
|
||||||
|
// in the sleeping callbacks, during a VRAM arbiter release or token wait), so it must not run
|
||||||
|
// shutdown_handler itself: terminate() takes that mutex and the thread would deadlock on itself.
|
||||||
|
// It only writes the signal number here; shutdown_handler runs on a dedicated thread.
|
||||||
|
static int shutdown_pipe[2] = { -1, -1 };
|
||||||
|
#endif
|
||||||
|
|
||||||
static inline void signal_handler(int signal) {
|
static inline void signal_handler(int signal) {
|
||||||
if (is_terminating.test_and_set()) {
|
if (is_terminating.test_and_set()) {
|
||||||
// in case it hangs, we can force terminate the server by hitting Ctrl+C twice
|
// in case it hangs, we can force terminate the server by hitting Ctrl+C twice
|
||||||
@@ -34,9 +46,48 @@ static inline void signal_handler(int signal) {
|
|||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
if (shutdown_pipe[1] >= 0) {
|
||||||
|
const char sig = (char) signal;
|
||||||
|
const ssize_t res = write(shutdown_pipe[1], &sig, 1); // async-signal-safe
|
||||||
|
(void) res;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
shutdown_handler(signal);
|
shutdown_handler(signal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
// runs shutdown_handler for signal_handler; joined (after a 0 byte) when llama_server() returns
|
||||||
|
struct shutdown_thread {
|
||||||
|
std::thread thread;
|
||||||
|
|
||||||
|
void start() {
|
||||||
|
if (pipe(shutdown_pipe) != 0) {
|
||||||
|
return; // fall back to calling shutdown_handler from the signal handler
|
||||||
|
}
|
||||||
|
fcntl(shutdown_pipe[0], F_SETFD, FD_CLOEXEC); // not inherited by spawned tools / MCP servers
|
||||||
|
fcntl(shutdown_pipe[1], F_SETFD, FD_CLOEXEC);
|
||||||
|
thread = std::thread([]() {
|
||||||
|
char sig = 0;
|
||||||
|
while (read(shutdown_pipe[0], &sig, 1) < 0 && errno == EINTR) {}
|
||||||
|
if (sig != 0) {
|
||||||
|
shutdown_handler(sig);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
~shutdown_thread() {
|
||||||
|
if (thread.joinable()) {
|
||||||
|
const char sig = 0;
|
||||||
|
const ssize_t res = write(shutdown_pipe[1], &sig, 1);
|
||||||
|
(void) res;
|
||||||
|
thread.join();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
#endif
|
||||||
|
|
||||||
// satisfies -Wmissing-declarations (used by llama command)
|
// satisfies -Wmissing-declarations (used by llama command)
|
||||||
int llama_server(int argc, char ** argv);
|
int llama_server(int argc, char ** argv);
|
||||||
|
|
||||||
@@ -528,8 +579,12 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
|
|||||||
}
|
}
|
||||||
|
|
||||||
// register signal handler if not running by CLI
|
// register signal handler if not running by CLI
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
shutdown_thread sig_thread; // declared after ctx_server & co, so joined before they are destroyed
|
||||||
|
#endif
|
||||||
if (!is_run_by_cli) {
|
if (!is_run_by_cli) {
|
||||||
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
|
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
|
||||||
|
sig_thread.start();
|
||||||
struct sigaction sigint_action;
|
struct sigaction sigint_action;
|
||||||
sigint_action.sa_handler = signal_handler;
|
sigint_action.sa_handler = signal_handler;
|
||||||
sigemptyset (&sigint_action.sa_mask);
|
sigemptyset (&sigint_action.sa_mask);
|
||||||
|
|||||||
Reference in new issue
Block a user