Compare commits
65
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2caae7e113 | ||
|
|
f3397cded0 | ||
|
|
510aa6e090 | ||
|
|
166ec22e2b | ||
|
|
e8386267ec | ||
|
|
b13c413555 | ||
|
|
312c29c2b1 | ||
|
|
24b605331c | ||
|
|
3621025e8f | ||
|
|
72da62cea0 | ||
|
|
d2396e4e36 | ||
|
|
a27217e278 | ||
|
|
3bf6b55b39 | ||
|
|
3ab97b345f | ||
|
|
1df5ced65e | ||
|
|
cf217b0318 | ||
|
|
383dbf4e6c | ||
|
|
060e46516e | ||
|
|
35d58d62cd | ||
|
|
da244dc458 | ||
|
|
062a025714 | ||
|
|
7a233df171 | ||
|
|
7dbe7fdafa | ||
|
|
2766d7977b | ||
|
|
4b29f38b92 | ||
|
|
dbae58b893 | ||
|
|
f58520aa4f | ||
|
|
263d75cba3 | ||
|
|
5f753f7a0c | ||
|
|
4e455b79df | ||
|
|
9537f885e0 | ||
|
|
12a2add2c5 | ||
|
|
deb09fb305 | ||
|
|
3b2c7188fb | ||
|
|
3141ad2a9e | ||
|
|
0ecd1a43cc | ||
|
|
bf3bda9f65 | ||
|
|
bdf2b81b81 | ||
|
|
a213514855 | ||
|
|
8742446dba | ||
|
|
79e6ade8b3 | ||
|
|
3b068bdb3a | ||
|
|
3d4e7bc065 | ||
|
|
a620add3b7 | ||
|
|
9c042626b3 | ||
|
|
2c6aa3b6e7 | ||
|
|
c61db474d8 | ||
|
|
7194d47311 | ||
|
|
04552507fe | ||
|
|
d773ab0ffc | ||
|
|
58e693b794 | ||
|
|
b7cb284e54 | ||
|
|
fa24747925 | ||
|
|
98c7e109b7 | ||
|
|
bc55fb9990 | ||
|
|
25c12e56ca | ||
|
|
8e575c9e54 | ||
|
|
1ee9806b3d | ||
|
|
558f7f65a2 | ||
|
|
cb471f9b2f | ||
|
|
56a1010bbc | ||
|
|
cfda297cbb | ||
|
|
e1bc1f7398 | ||
|
|
b5085151f2 | ||
|
|
ab7e03ec6c |
No files matched your search
@@ -23,6 +23,32 @@ The main goal of `llama.cpp` is to enable LLM (and MLLM) inference with minimal
|
||||
|
||||
The `llama.cpp` project is built on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## This fork - RX 580 (Polaris / GCN) serving of large MoE models
|
||||
|
||||
A private fork maintained for one machine: an AMD RX 580 8 GB (RADV Vulkan) serving Qwen3.6-35B-A3B,
|
||||
gemma-4-26B-A4B and smaller models under llama-swap, with the MoE experts in host RAM
|
||||
(`--n-cpu-moe`). **[FORK.md](FORK.md) is the source of truth**: every fork change with its
|
||||
invariants, rebase hazards and validation, the measured dead ends, the production configuration and
|
||||
the measurement traps. The rebase and deploy procedure is
|
||||
[docs/fork-rebase-runbook.md](docs/fork-rebase-runbook.md); `scripts/fork-validate.sh` validates a
|
||||
build against a reference; `scripts/rx580-bench/` is the benchmark harness.
|
||||
|
||||
In short, the fork adds:
|
||||
|
||||
- three Polaris prefill wins, on without a flag and token-identical to mainline: the MoE routing-ids
|
||||
readback skip in `sched_copy_experts`, flash-attention `mask_opt` for GCN at head size >= 256, and
|
||||
a tiled-transpose fast path for the delta-net conv-state concat;
|
||||
- opt-in VRAM time-sharing between several llama-server processes (`LLAMA_SLEEP_VRAM_ONLY`,
|
||||
`LLAMA_SLEEP_EVICT_KV`, `LLAMA_MMPROJ_ONDEMAND`, `LLAMA_VRAM_ARENA`): a model releases everything it
|
||||
holds in VRAM and gets it back without a restart, so its RAM prompt cache survives;
|
||||
- fixes to the server's sleep, wake and shutdown paths and to cold teardown that the sharing exposed;
|
||||
- validation-only env knobs (`GGML_VK_CONCAT_FASTPATH`, `LLAMA_MOE_IDS_SKIP`), unset in production.
|
||||
|
||||
Production settings for the 35B are in FORK.md section 5 (`-lm none -t 6 -ngl 99 --n-cpu-moe 40
|
||||
-b 2048 -ub 2048 -fa 1 --parallel 1`, ctx 131072). Keep all experts on the host (`--n-cpu-moe 40`)
|
||||
for long contexts: lower values are faster only at short context and run out of VRAM as the KV grows
|
||||
(FORK.md section 4).
|
||||
|
||||
## Quick start
|
||||
|
||||
A few options to get `llama.cpp` installed on your machine:
|
||||
|
||||
@@ -1704,6 +1704,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.cache_ram_explicit = true;
|
||||
}
|
||||
).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
|
||||
add_opt(common_arg(
|
||||
{"--cache-disk-path"}, "PATH",
|
||||
"save the state of each slot with a prompt to this directory on shutdown, and restore it for a prompt that starts with it after a restart (default: disabled)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.cache_disk_path = value;
|
||||
}
|
||||
).set_env("LLAMA_ARG_CACHE_DISK_PATH").set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--cache-disk-size"}, "N",
|
||||
string_format("size limit of the --cache-disk-path directory in MiB, the least recently used files are removed (default: %d, 0 - no limit)", params.cache_disk_mib),
|
||||
[](common_params & params, int value) {
|
||||
if (value < 0) {
|
||||
throw std::invalid_argument("cache-disk-size must be non-negative");
|
||||
}
|
||||
params.cache_disk_mib = value;
|
||||
}
|
||||
).set_env("LLAMA_ARG_CACHE_DISK_SIZE").set_examples({LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"-kvu", "--kv-unified"},
|
||||
{"-no-kvu", "--no-kv-unified"},
|
||||
|
||||
@@ -663,6 +663,8 @@ struct common_params {
|
||||
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
|
||||
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
|
||||
bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise
|
||||
std::string cache_disk_path; // directory of the disk prompt cache, empty = disabled
|
||||
int32_t cache_disk_mib = 4096; // size limit of cache_disk_path in MiB, 0 = no limit
|
||||
|
||||
// llama-connect params
|
||||
bool server_connect = false;
|
||||
|
||||
@@ -0,0 +1,716 @@
|
||||
# Fork rebase and deploy runbook (RX 580 fork)
|
||||
|
||||
Step-by-step procedure for moving the fork onto a new upstream and putting it into production, as it
|
||||
has evolved over the 2026-09-09, 2026-10-09, 2026-10-24 and 2026-11-21 rebases. Read [FORK.md](../FORK.md) first:
|
||||
it lists every fork feature with its invariants and rebase hazards, and this runbook refers to it.
|
||||
|
||||
Ground rules:
|
||||
|
||||
- **Never build on the workstation.** All builds run on the RX 580 pod. Locally, rely on careful reading.
|
||||
- **Never resolve a conflict with `-X theirs` or `-X ours`.** Resolve every hunk by hand.
|
||||
- **"Applied cleanly" is not "correct".** Re-read every clean pick against the new upstream.
|
||||
- **Never `pkill -f` on the pod.** It matches the `kubectl exec` shell and kills it. Kill by explicit PID.
|
||||
- **Never leave the pod not serving.** Anything that stops llama-swap must restart it from a trap.
|
||||
- Keep the previous production branch untouched as the safety net. Do not push for the owner; the owner
|
||||
reviews and pushes.
|
||||
|
||||
Contents: 1. Survey; 2. Trial rebase; 3. The real rebase; 4. Checks without a compiler; 5. Build on
|
||||
the pod; 6. Validate; 7. Deploy; 8. Post-deploy checks; 9. Rollback; 10. Recovering a hung GPU;
|
||||
11. Operations reference.
|
||||
|
||||
**Redeploying a new tip on the same upstream base (no rebase)**, e.g. new fork commits on the
|
||||
production branch (2026-10-25: `6c480f7e7` -> `2fc9f9c35`, same branch, same base):
|
||||
- Skip sections 1-3. Of section 4 only the ASCII check (on `git diff <prod>..<tip>`) and a careful
|
||||
read of that diff apply; `<prod>` is `git -C /root/llama.cpp rev-parse HEAD` on the pod. There is
|
||||
nothing to range-diff.
|
||||
- Do sections 5-9 as written: build the tip in a scratch worktree, validate it with `--ref` = the
|
||||
production build, then deploy.
|
||||
- The branch name stays the same, so deploy step 2's `git checkout -B <same branch> <tip>` moves the
|
||||
production branch on the pod to the tip. After that the rollback pointer file is the only record
|
||||
of the old production commit on the pod, so write it first (deploy step 1).
|
||||
|
||||
---
|
||||
|
||||
## 1. Survey upstream
|
||||
|
||||
Local remotes: `origin` is upstream (github.com/ggerganov/llama.cpp), `gitea` is the fork
|
||||
(git@gitea.lumpiasty.xyz:Lumpiasty/llama.cpp.git, SSH push from the workstation). On the pod,
|
||||
`origin` is the same gitea repository over HTTPS (fetch only).
|
||||
|
||||
```sh
|
||||
git fetch origin
|
||||
OLD_BASE=$(git merge-base origin/master HEAD) # the old upstream base (still an ancestor of the new master)
|
||||
OLD_TIP=$(git rev-parse HEAD)
|
||||
git log --oneline "$OLD_BASE"..HEAD # the fork commits, oldest last
|
||||
git log --oneline "$OLD_BASE"..origin/master | wc -l # size of the upstream delta
|
||||
```
|
||||
|
||||
Then look at upstream changes in the areas FORK.md lists as hazards:
|
||||
|
||||
```sh
|
||||
git log --oneline "$OLD_BASE"..origin/master -- \
|
||||
src/llama-context.cpp src/llama-context.h src/llama-model.cpp src/llama-model.h \
|
||||
src/llama-kv-cache.cpp src/llama-kv-cache.h src/llama-kv-cache-iswa.cpp \
|
||||
src/llama-memory.h src/llama-memory-recurrent.cpp src/llama-memory-hybrid.cpp \
|
||||
src/llama-moe-cache.cpp src/llama-moe-cache.h src/models/delta-net-base.cpp \
|
||||
ggml/src/ggml-vulkan/ ggml/src/ggml-backend.cpp ggml/src/ggml-backend-impl.h ggml/include/ggml-backend.h \
|
||||
tools/server/server-queue.cpp tools/server/server-queue.h tools/server/server-context.cpp tools/server/server.cpp \
|
||||
tools/mtmd/clip.cpp tools/mtmd/mtmd.cpp common/arg.cpp tools/llama-bench/
|
||||
```
|
||||
|
||||
**Temporary upstream carries (FORK.md F12)** come first: `git log --oneline --grep '^upstream-carry'
|
||||
"$OLD_BASE"..HEAD`. For each: merged into the new upstream -> do not cherry-pick it (check that the
|
||||
merged version is the carried head or newer); still open -> re-fetch the PR head and re-carry it if
|
||||
it moved; closed -> ask the owner. To carry a PR (2026-11-22: #32735, #32740):
|
||||
```sh
|
||||
git fetch https://github.com/ggml-org/llama.cpp pull/<N>/head:pr-<N>
|
||||
git cherry-pick $(git merge-base pr-<N> HEAD)..pr-<N> # the PR's commits, oldest first; resolve by hand
|
||||
# squash them into one commit with the PR author, subject "upstream-carry #<N>: <PR title>", and a
|
||||
# body with the PR URL, the carried head and any conflict resolution (git commit-tree on the final tree)
|
||||
diff <(git diff $(git merge-base pr-<N> HEAD) pr-<N> | grep '^[+-]') <(git diff HEAD~1 HEAD | grep '^[+-]')
|
||||
```
|
||||
The last diff shows only the lines you changed while resolving conflicts (for #32740: none, the F2
|
||||
line differs only in context). Validate a carry like any compiled change (sections 5-6), plus the
|
||||
PR's own tests.
|
||||
|
||||
For each fork commit decide: still needed, needs a semantic merge, or obsolete. Examples of past
|
||||
decisions:
|
||||
- 2026-09-09: dropped the weight-pinning chain, the expert prefetch and fractional `-ncmoe` (all measured
|
||||
losses or no win on this card, FORK.md section 4).
|
||||
- 2026-10-09: the routing-ids readback skip had nothing to apply to (upstream #29943 deleted its block
|
||||
in `ggml-backend.cpp`); it was re-authored in `llama_context::sched_copy_experts`. Upstream's new MoE
|
||||
cache (#29887) needed a new fork commit so the VRAM arbiter releases and invalidates it.
|
||||
- Also check CLI changes that break production config or the scripts (upstream #28334 removed
|
||||
`--no-mmap`/`--mmap`/`--mlock` and llama-bench `-mmp`; the default `--port` changed to 9931).
|
||||
|
||||
## 2. Trial rebase in a scratch worktree
|
||||
|
||||
Measure the conflicts before doing the real thing:
|
||||
|
||||
```sh
|
||||
git worktree add /tmp/trial-rebase origin/master
|
||||
cd /tmp/trial-rebase
|
||||
git cherry-pick "$OLD_BASE".."$OLD_TIP" # stop at each conflict, note file + hunk, then:
|
||||
git cherry-pick --abort # or resolve roughly to see the next one
|
||||
cd - && git worktree remove --force /tmp/trial-rebase
|
||||
```
|
||||
|
||||
Write down, per commit: clean / conflict (which file, which hunk, what both sides did). That list
|
||||
becomes the brief for the real rebase.
|
||||
|
||||
## 3. The real rebase
|
||||
|
||||
```sh
|
||||
NEW=fable5/vram-ondemand-rebase-$(date +%Y%m%d)
|
||||
git switch -c "$NEW" origin/master
|
||||
git cherry-pick <first-fork-commit> # one at a time, in the original order
|
||||
```
|
||||
|
||||
Rules:
|
||||
- Cherry-pick in the original order, keep authors and messages. New commits you author get the
|
||||
fork's trailer (`Co-Authored-By: Claude ...` for agent-written commits, FORK.md status block),
|
||||
unless the owner asks for something else.
|
||||
- Resolve every conflict by hand. Union both sides where both added code; keep upstream's new terms.
|
||||
- Fold each fix into the commit it belongs to so every commit builds on its own:
|
||||
`git commit --fixup=<sha>` then `GIT_SEQUENCE_EDITOR=: git rebase -i --autosquash origin/master`.
|
||||
- Leave the previous production branch untouched.
|
||||
|
||||
**What `-X theirs` silently broke in a trial (2026-10-09).** It "applied" every commit and produced
|
||||
broken code:
|
||||
- resurrected the scheduler readback block upstream had deleted from `ggml-backend.cpp`;
|
||||
- dropped `!use_sparse` from the GCN `use_mask_opt` condition;
|
||||
- dropped upstream's `copy_experts_info` (compile error);
|
||||
- dropped upstream's `is_empty()` in the recurrent memory;
|
||||
- dropped upstream's MoE-cache memory breakdown block;
|
||||
- left a static/extern mismatch with the new `ggml-vulkan-common.h` declarations (compile error).
|
||||
|
||||
The correct resolutions in that rebase: keep upstream's `n_embd` width check plus our decode
|
||||
auto-restore; `copy_experts_info` plus our `kv_device_evicted`; `is_empty()` plus our `dev_shadows`;
|
||||
upstream's `if (moe_cache)` breakdown block outside our `if (sched)` guard; make `ggml_vk_concat` and
|
||||
`ggml_backend_vk_free` non-static to match the new header while our helpers stay static.
|
||||
|
||||
**Clean picks that were wrong (2026-10-24).**
|
||||
- Concat fast path (3df719e1e): applied cleanly, did not compile. Upstream `733f57a99` renamed
|
||||
`pipeline_cpy_f32_f32`/`pipeline_cpy_f16_f16` to `pipeline_cpy_32`/`pipeline_cpy_16` (bitwise copies,
|
||||
same push constants).
|
||||
- MoE cache release/restore (7ac2014a8): applied cleanly, did not compile (upstream `33d6135cb` changed
|
||||
LRU to SLRU, `g.lru` -> `g.slru`) and had a runtime bug: `moe_cache_slru::init()` resets only the
|
||||
probation segment, so re-initialising a used cache corrupts its lists. `invalidate()` now assigns a
|
||||
fresh `moe_cache_slru()` first.
|
||||
- VRAM sharing core (481be8b28): besides its one conflict, upstream's new `load_tensors_hadamard()`
|
||||
pushes two more entries into `pimpl->ctxs_bufs`. Without matching `weight_release.emplace_back()`
|
||||
calls, `release_device_weights()` reads past the end of `weight_release`. Nothing conflicted.
|
||||
- Conflicts that round: `ctx_buf_maps.emplace_back` replaced upstream by `ctx_bufs.emplace_back(ctx,
|
||||
std::move(buf_ptrs))` (keep upstream's line, our release-slot block before it); `llama-context.h`
|
||||
(keep upstream's `n_reallocs_same/diff` beside our `kv_device_evicted`); `recv_with_timeout`
|
||||
(upstream `4e201388c` uses a fixed deadline: keep our order "check `!running`, return null", then
|
||||
`wait_until(lock, deadline)`).
|
||||
|
||||
**What the 2026-11-21 rebase (upstream `e6d624b9a`, 757 commits past `3f03b1772`) changed** (nothing
|
||||
built or run locally; every item below is by reading only and needs the pod build):
|
||||
- Upstream refactors that moved fork code (hand ports, not conflicts of one hunk):
|
||||
- `clip.cpp`/`clip.h` are gone (#31865). The encoder release/restore (F5) is now
|
||||
`mtmd_model_release_device`/`mtmd_model_restore_device` on `mtmd_model` (`mtmd-model.cpp`), called
|
||||
from `mtmd_release_device`/`mtmd_restore_device` under `compute_mutex` after syncing the shared
|
||||
scheduler; the page-cache drop of F10 moved to `mtmd_model_loader::load_tensors`.
|
||||
- `server_prompt_cache` moved to `server-slot-sched.{h,cpp}` and `get_available_slot()` was replaced
|
||||
by `server_slot_sched::select()`. The disk prompt cache (F11) lives there now
|
||||
(`server_prompt_disk_cache`), is called from the task dispatch after `update_prompt_cache()`, and
|
||||
uses `common_state_data` and `shared_ptr<const common_prompt_checkpoint>`. Only prompts made with
|
||||
the server lora config are saved, and a restored prompt gets that config in `prompt.lora`.
|
||||
- Upstream's `llama_memory_bufs` replaced the raw `ctxs_bufs` vectors of the KV cache and the recurrent
|
||||
memory: the null-buffer guard moved into `llama_memory_bufs::memory_breakdown()`, and the release
|
||||
code indexes `ctxs_bufs.bufs[i]`. Those two did not compile after a clean pick.
|
||||
- The MoE cache is now a ClassLRU (`moe_cache_lru`, `g.lru`) with `no_alloc` as a constructor
|
||||
argument and `cpu_share`. `invalidate()` rebuilds `g.lru`, refills every slot map with
|
||||
`g.lru.empty` (not -1), clears the CPU maps and `cpu_experts`.
|
||||
- `server.cpp` was split into `server_run_instance` and friends: the self-pipe shutdown thread
|
||||
(F7) is started in `server_run_instance` only, just before `server_install_signal_handlers()`.
|
||||
- `server_queue` got `n_inflight` (#32133). `should_sleep` checks it first, so a response that is still
|
||||
open also blocks the want-lock yield. This is a decision, not an accident: check it with `--arbiter`.
|
||||
- Upstream added its own tiled-transpose concat (#30591) with no size gate. The fork path still runs
|
||||
first (`ne0 >= 32`); the gate and the `GGML_VK_CONCAT_FASTPATH` knob were added to upstream's selector
|
||||
(`ggml_vk_concat_is_transposed`), otherwise decode would take upstream's tiled path again (F3).
|
||||
- Memory types with no release override (stay in VRAM when cold, not used in production):
|
||||
`llama_kv_cache_dsa`, `_dsa_iswa`, `_dsv4`, `_msa`, `llama_memory_hybrid_iswa`. The KV rotation
|
||||
tensors (`attn_rot_bufs`, #31073) are small and are not released either.
|
||||
|
||||
Open questions from that rebase:
|
||||
- Is the fork's own concat fast path (F3) now redundant with upstream's tiled concat (#30591)?
|
||||
Benchmark `GGML_VK_CONCAT_FASTPATH=0` against the default on pp and decode before deciding.
|
||||
- Does `n_inflight` blocking the want-lock yield delay handoffs? Validation says no: handoff median
|
||||
candidate 692 ms vs reference 1068 ms (`a-handoff-latency`, 2026-11-21).
|
||||
|
||||
**Semantic re-checks to do for every rebase, clean or not** (details in FORK.md):
|
||||
- F1: `sched_copy_experts` and its `st` struct unchanged in meaning.
|
||||
- F2: `use_mask_opt` still contains upstream's terms plus our `>= 256` (and `!bias &&` from #32740, carried as F12 or merged upstream).
|
||||
- F3: `build_conv_state` still emits `ggml_concat(conv_states, ggml_transpose(qkv_mixed), 0)` with
|
||||
`qkv_mixed` contiguous `[ch, n_seq_tokens, n_seqs]`; pipeline names; `types_ok` still excludes
|
||||
quantized types.
|
||||
- F4: every `ctxs_bufs` push in `src/llama-model.cpp` has a `weight_release` push
|
||||
(`git grep -n "ctxs_bufs.emplace_back\|weight_release" src/llama-model.cpp`); list memory types
|
||||
(`git grep -n "public llama_memory_i" src/`) and note any without a release override; MoE-cache
|
||||
internals; `sched_reserve()` still goes through `sched_new()` and re-registers callbacks;
|
||||
`free_scratch` still the last `ggml_backend_i` member; Vulkan prealloc fields match `ggml_vk_cleanup`.
|
||||
- F6/F7: `server_queue::start_loop` still evaluates `should_sleep()` only from the idle branch, never
|
||||
inside `yield_to_queue()`; `handle_sleeping_state`, `server_res_generator`, `load_model` ordering;
|
||||
signal setup in `server.cpp`.
|
||||
- F8: new iterators over `ctxs_bufs` or users of `sched` on teardown paths.
|
||||
- Scripts: flag spellings used by `scripts/rx580-bench/*` and `scripts/fork-validate.sh`, and the log
|
||||
lines `fork-validate.sh` greps for (FORK.md F9).
|
||||
- Production config: every flag in `/root/config.yaml` still exists in `common/arg.cpp`.
|
||||
|
||||
## 4. Checks without a compiler
|
||||
|
||||
Before handing the branch over for a build:
|
||||
|
||||
```sh
|
||||
git diff origin/master..HEAD | grep -n '^[+-]\(<<<<<<<\|=======\|>>>>>>>\)' # no conflict markers
|
||||
git diff origin/master..HEAD | grep -nP '^\+.*[^\x00-\x7F]' # ASCII only in added lines
|
||||
git range-diff "$OLD_BASE".."$OLD_TIP" origin/master..HEAD # patch vs patch
|
||||
```
|
||||
|
||||
- The range-diff must show only the edits you made on purpose. In 2026-10-24 the whole patch was
|
||||
identical to the old one except for 4 deliberate edits.
|
||||
- Brace balance per changed file: the change in `{`/`}` balance from new base to new tip should equal
|
||||
the change from old base to old tip.
|
||||
- Every identifier in added lines must exist on the new master or be defined by the fork. This catches
|
||||
a renamed function but not a renamed member (like `lru` -> `slru`); re-read member accesses by hand.
|
||||
- Write down every spot where you would normally lean on the compiler and hand that list to whoever
|
||||
builds.
|
||||
|
||||
Then the owner pushes the branch to gitea, and the pod fetches it from there (section 5).
|
||||
|
||||
**If you were told not to push**, do not stop at an unbuilt branch. Unless the task says otherwise, a
|
||||
rebase task continues through the build and validation (sections 5 and 6) and reports the results;
|
||||
the owner pushes afterwards. Get the branch onto the pod with a git bundle:
|
||||
```sh
|
||||
# workstation; <base> is a commit the pod already has, e.g. production's HEAD
|
||||
git bundle create /tmp/x.bundle <base>..<branch>
|
||||
export KUBECONFIG=/home/user/Projects/klaster/talos/generated/kubeconfig
|
||||
kubectl -n llama exec -i deploy/supervisord -- sh -c 'cat > /root/x.bundle' < /tmp/x.bundle
|
||||
kubectl -n llama exec deploy/supervisord -- sh -c 'cd /root/llama.cpp && git fetch /root/x.bundle <branch>:<branch>'
|
||||
```
|
||||
The branch is then local in `/root/llama.cpp`: in section 5 skip the `git fetch origin` and use
|
||||
`git worktree add "$W" <branch>`. Delete `/root/x.bundle` afterwards (disk is tight). Deploy step 2
|
||||
likewise uses the local branch instead of fetching from origin.
|
||||
|
||||
## 5. Build on the pod, in a scratch worktree
|
||||
|
||||
Production keeps serving during this step.
|
||||
|
||||
```sh
|
||||
export KUBECONFIG=/home/user/Projects/klaster/talos/generated/kubeconfig
|
||||
kubectl -n llama exec -i deploy/supervisord -- sh -c '
|
||||
set -e
|
||||
B=fable5/vram-ondemand-rebase-YYYYMMDD
|
||||
W=/root/arms/rebYYYYMMDD
|
||||
git -C /root/llama.cpp fetch origin "$B"
|
||||
git -C /root/llama.cpp worktree add "$W" FETCH_HEAD
|
||||
cd "$W"
|
||||
export HF_UI_VERSION=b11906 # production's: cut -d'|' -f2 /root/llama.cpp/build/tools/ui/.ui-stamp
|
||||
cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_VULKAN=1 -DGGML_NATIVE=ON -DLLAMA_BUILD_UI=OFF
|
||||
nice -n 19 cmake --build build --target llama-server llama-bench test-backend-ops -j 8
|
||||
'
|
||||
```
|
||||
|
||||
- Build only the targets you need; the full `all` target fails on the UI assets (needs npm/network).
|
||||
If a tree has a stale `tools/ui/dist` with a stub `index.html`, remove it (it breaks llama-ui-embed).
|
||||
- `nice -n 19 -j 8` leaves the serving models CPU. A full build takes about 3.5 minutes.
|
||||
- Report compile errors verbatim against the list from step 4.
|
||||
- Do not overlap a build with a large benchmark (host OOM). Disk is tight: `/root` had about 13 GB
|
||||
free (97% full) on 2026-10-25, so check `df -h /root` first. Each build tree is about 700 MB;
|
||||
remove scratch worktrees you no longer need with `git -C /root/llama.cpp worktree remove <dir>`.
|
||||
- `-DLLAMA_BUILD_UI=OFF` does NOT make the UI asset step harmless. The `llama-ui-assets` step still
|
||||
runs, and when the build number has no HF tag it falls back to `latest`. On 2026-11-21 `latest`
|
||||
broke the build (missing `bundle[hash].js`, `.css` and `workbox[hash].js`). Set
|
||||
`HF_UI_VERSION=<stamp>` for both `cmake -B build` and `cmake --build`, where the stamp is
|
||||
production's current one in `build/tools/ui/.ui-stamp` (b11906 on 2026-11-21). The file holds
|
||||
`ggml-org/llama-ui|b11906` and CMake requires `^[A-Za-z0-9._-]+$`, so take only the tag with
|
||||
`cut -d'|' -f2 build/tools/ui/.ui-stamp`; never pass `$(cat ...)`. The validated
|
||||
worktree's UI assets may then differ from the deployed ones (the code is identical).
|
||||
- Long runs: launch detached (`setsid nohup sh X > X.log 2>&1 </dev/null &`) and poll, rather than
|
||||
holding a `kubectl exec` open for hours.
|
||||
- The validated build must contain all three targets: `llama-server`, `llama-bench` and
|
||||
`test-backend-ops` (`--perf` and `--ops` run the last two from `--build`). A build script that
|
||||
makes only `llama-server` (as the 2026-10-25 `diskcache-build.sh` first did) is not a validated
|
||||
build.
|
||||
- Detach only one script. Upload it first (`kubectl exec -i ... sh -c 'cat > /root/arms/X.sh' <
|
||||
X.sh`), then start it alone: `setsid nohup sh /root/arms/X.sh > X.log 2>&1 </dev/null &`. Do not
|
||||
background a chain such as `cat > X && sh -n X && setsid ... &` inside the `kubectl exec` shell. The
|
||||
chain is a child of that session, and it can die with the session before `setsid` runs.
|
||||
|
||||
## 6. Validate with `scripts/fork-validate.sh`
|
||||
|
||||
Copy the script from the candidate tree (it is in the repo) and run it in two phases. The header of
|
||||
the script describes each check.
|
||||
|
||||
Rule: any test of a production model must use production's exact command line: the `cmd` of its
|
||||
`/root/config.yaml` entry with the macros expanded, including the mmproj that `-hf` pulls (production
|
||||
never passes `--no-mmproj` to the 35B or gemma), and supervisord's `environment=`. Change only
|
||||
`--port`, the arena and scratch paths. The 2026-10-25 deploy of the disk prompt cache aborted on the
|
||||
first request: every test had used `--no-mmproj`, which hid a `GGML_ASSERT(!has_mtmd)`, and it was
|
||||
rolled back (FORK.md F11).
|
||||
|
||||
```sh
|
||||
# phase 1, beside production
|
||||
bash /root/arms/rebYYYYMMDD/scripts/fork-validate.sh \
|
||||
--build /root/arms/rebYYYYMMDD/build --ref /root/llama.cpp/build --ops --concat
|
||||
```
|
||||
|
||||
Add `--arbiter` to phase 1 only if the card has >= 1536 MiB free (8192 MiB minus the bytes in
|
||||
`cat /sys/class/drm/card*/device/mem_info_vram_used`); below that it FAILs at once. With gemma-26B
|
||||
warm in production only about 1.2 GiB is free and nothing frees it without downtime; then run
|
||||
`--arbiter` in the phase-2 swapdown window instead (about 2 minutes more downtime), as below.
|
||||
|
||||
The same applies to `--concat`'s `candidate-timing` gate: with the card nearly full (2026-11-22: 956
|
||||
MiB free beside a warm Inkling) every concat arm ran about 2x slower and the fast path showed no
|
||||
gain (off/cand 0.9x, FAIL); in the swapdown window the same build passed with 5.1x. A phase-1
|
||||
`candidate-timing` FAIL with little free VRAM is not a verdict: rerun `--concat` in phase 2.
|
||||
|
||||
**Page cache before and after phase 2.** The `--ref` build may lack F10 (FORK.md), and `--perf`
|
||||
loads the 35B many times: during a reference arm on 2026-10-25 MemAvailable fell to 27 GB. Run the
|
||||
eviction snippet of deploy step 3 before phase 2 and again after it (swapdown does not), and check
|
||||
`/proc/meminfo`.
|
||||
|
||||
```sh
|
||||
# phase 2, with llama-swap stopped: run it under the swapdown wrapper, whose trap restarts llama-swap
|
||||
cat > /root/arms/fv-perf.sh <<'EOF'
|
||||
HUB=/root/.cache/huggingface/hub
|
||||
Q35=$HUB/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
||||
G26=$HUB/models--SC117--gemma-4-26B-A4B-it-qat-heretic-GGUF/snapshots/e6759b29958180a998f62f991e4c741fe9b4d452/gemma-4-26B-A4B-it-qat-heretic-UD-Q4_K_XL.gguf
|
||||
bash /root/arms/rebYYYYMMDD/scripts/fork-validate.sh \
|
||||
--build /root/arms/rebYYYYMMDD/build --ref /root/llama.cpp/build \
|
||||
--arbiter --perf --moe-exact "$Q35" --moecache "$G26" --keep
|
||||
EOF
|
||||
setsid nohup sh /root/bench/moe/swapdown.sh /root/arms/fv-perf.sh > /root/arms/fv-perf.log 2>&1 </dev/null &
|
||||
```
|
||||
|
||||
- `G26` is the production SC117 gemma-26B, the `refs/main` snapshot (`e6759b29...` on 2026-10-25;
|
||||
an older snapshot `82c751632...` is also on disk). If it moves, read the snapshot name from
|
||||
`$HUB/models--SC117--gemma-4-26B-A4B-it-qat-heretic-GGUF/refs/main`.
|
||||
- `/root/bench/moe/swapdown.sh <script>` (pod-only helper, not in the repo) stops llama-swap, waits for
|
||||
an idle GPU, runs the script, and restarts llama-swap from an EXIT/INT/TERM/HUP trap. It prints
|
||||
`SWAPDOWN_DONE` at the end. Afterwards confirm llama-swap is RUNNING and `/v1/models` returns 200.
|
||||
- The `--concat` check needs a build with the `GGML_VK_CONCAT_FASTPATH` knob (in the fork since
|
||||
e085c347d, so a rebased candidate has it; older candidates need `--concat-build DIR`).
|
||||
- `--moecache` needs a cache that actually hits (default 2048 MiB; at 256 MiB gemma-26B had 0 hits,
|
||||
which proves nothing).
|
||||
- `--arbiter` also checks that a cold server really frees its VRAM (`v-vram-release`,
|
||||
`v-vram-restore`; SKIPs if production's VRAM moved during the measurement, then rerun) and A/Bs
|
||||
ordinary handoff latency against `--ref` (`a-handoff-latency`). `--ref` must be the build currently
|
||||
in production, so both comparisons are against what serves today.
|
||||
- Phase 2 downtime, measured 2026-10-25: about 32 minutes in all (`--arbiter` about 2 min, `--perf`
|
||||
about 28 min, `--moecache` under 30 s, `--moe-exact` 78 s). `--moe-exact` proves the routing-ids
|
||||
readback skip exact (F1): the candidate's greedy output equals the reference's and, with the
|
||||
`LLAMA_MOE_IDS_SKIP` knob (any build of this branch), the candidate's with the skip forced off.
|
||||
- `--perf` gates on medians of all repetition samples (since 2512290f9); the samples are in
|
||||
`<scratch>/perf.tsv`, and `fork-validate.sh --perf-report <file>` re-aggregates them offline. A pod
|
||||
restart empties `/tmp`, so copy a scratch dir you want to keep to `/root/arms/` (the 2026-10-24
|
||||
scratch dirs were lost that way).
|
||||
- F14 run (2026-11-22, candidate `f3397cded` = `166ec22e2` (docs on `b13c41355`) + F14, server-only change; one
|
||||
swapdown window, `/root/arms/usw-window.sh`, log `/root/arms/usw-window.log`, about 15 minutes):
|
||||
the 35B TTFT A/B with and without `LLAMA_USERSTART_MERGE=0` (FORK.md F14, M U U M, production 35B
|
||||
think command line with the mmproj), `--diskcache` 15 PASS, 0 FAIL, 0 SKIP, and the
|
||||
`diskcache-window3.sh` 35B legs (all passed). The full `--perf`/`--arbiter` set was not rerun: the
|
||||
change touches only the server prompt-batch loop. Deploy downtime 33 s (incremental in-place
|
||||
build).
|
||||
- Last full run (2026-11-22, candidate `b13c41355` = `3621025e8` + F13 vs reference `3621025e8`, one
|
||||
swapdown window, `--ops --concat --diskcache --arbiter --perf --moe-exact --moecache`, log
|
||||
`/root/arms/overnight/fv.log`): 52 PASS, 0 FAIL, 0 SKIP in 41 minutes. Perf medians ref -> cand:
|
||||
pp2048 286.61 -> 284.94 (-0.6%), d16384 207.93 -> 207.96, d32768 161.93 -> 162.12, tg256@d4096
|
||||
14.11 -> 14.47; handoff median ref 715 / cand 706.5 ms; peak VRAM ref 5478 / cand 7771 MiB (the
|
||||
first-arm spike of FORK.md F9, this time in a candidate arm). With the card free, phase 1 and 2
|
||||
can run as one invocation like this.
|
||||
- Previous full run (2026-11-22, candidate `3621025e8` = `3bf6b55b3` + the F12 carries, vs reference
|
||||
`3bf6b55b3`; logs `/root/arms/carry1122-p1.log`, `-p2.log`): phase 1 `--ops --concat --diskcache`
|
||||
20 PASS, 1 FAIL (`concat/candidate-timing` beside a full card, see above), 1 SKIP (diskcache GPU
|
||||
leg); phase 2 `--concat --arbiter --diskcache --perf --moe-exact --moecache` 47 PASS, 0 FAIL, 0
|
||||
SKIP. Perf medians ref -> cand: pp2048 286.97 -> 286.32, d16384 208.31 -> 208.21, d32768 162.25 ->
|
||||
162.37; handoff median ref 696 / cand 700 ms; peak VRAM ref 7771 / cand 5502 MiB. Plus SCALE
|
||||
15/15 and `test-flash-attn-indexed-bias` (FORK.md F12). Window about 30 minutes.
|
||||
- Earlier full run (2026-11-21, candidate `3bf6b55b3` vs reference `adc2aa3ec`): phase 1 39 PASS, phase 2
|
||||
9 PASS plus the 35B disk-cache legs (`q35.py`, mmproj loaded). Perf medians ref -> cand: pp2048
|
||||
259.39 -> 286.61 (+10.5%), d16384 199.26 -> 207.63 (+4.2%), d32768 156.61 -> 157.51 (+0.6%),
|
||||
tg256@d4096 9.17 -> 8.85 t/s (noisy, not gated). Handoff median: candidate 692 ms, reference
|
||||
1068 ms. Downtime: about 42 minutes of validation and about 7 minutes of deploy.
|
||||
- Earlier full run (2026-10-25, candidate `2fc9f9c35` vs reference `6c480f7e7`, logs
|
||||
`/root/arms/dep-2fc9f9c35-fv1.log` and `-fv2.log`): phase 1 `--ops --concat` 8 PASS, phase 2
|
||||
`--arbiter --perf --moe-exact --moecache` 29 PASS; 37 PASS, 0 FAIL. Perf medians ref/cand: pp2048
|
||||
274.53/273.82, d16384 200.52/200.37, d32768 157.60/157.57, tg256@d4096 16.82/16.66 t/s. Handoff
|
||||
median ref 693.5 / cand 678 ms. `--moe-exact`: 39 skips logged, identical to the reference and to
|
||||
the skip forced off. Peak VRAM per arm ref 5476, cand 7062, cand 5767, ref 5476 MiB: the ~1.6 GiB
|
||||
spike in the first candidate arm has now been seen twice (open question, FORK.md F9).
|
||||
- Any arbiter change: `a-handoff-latency` is the ordinary-handoff A/B that two lost deploys showed
|
||||
is needed. For a change aimed at latency itself, also run more blocks by hand; 25%/200 ms is a
|
||||
regression gate, not a measurement.
|
||||
- Disk prompt cache (FORK.md F11, from `fable5/disk-prompt-cache`): add `--diskcache` to phase 1. It
|
||||
is CPU only and runs beside production (about 2 minutes). Its GPU leg SKIPs below 1536 MiB of free
|
||||
VRAM; then run `--diskcache` again in the phase-2 window. A 35B measurement driver is on the pod:
|
||||
`/root/arms/diskcache-test/q35.py [CHARS]` (scratch port 8095, scratch arena and cache dir, the
|
||||
production 35B flags, a pan-tadeusz conversation; about 6 minutes). The 2026-10-25 window script
|
||||
`/root/arms/diskcache-window.sh` (page-cache evictions, `--diskcache --arbiter`, `q35.py`) runs
|
||||
under swapdown (run 2026-10-25: 29 PASS). `/root/arms/diskcache-window3.sh` runs `q35.py`
|
||||
with production's exact 35B command line (mmproj loaded): save/restore, cold save, the
|
||||
same-reuse-point identity and an image request (FORK.md F11). Run 2026-11-21 on adc2aa3ec: all
|
||||
legs passed (18.8k-token chat, prefill 84.6 s -> turn 2 4.7 s after the restart, identical
|
||||
output; save 557 MB in 1.0 s, token free 0.34 s, peak RSS +430 MB; cold save identity; image
|
||||
answered and not saved; log `/root/arms/diskcache-test/q35-mm.log`). It supersedes
|
||||
`diskcache-window2.sh` (`cmp.py` runs with `--no-mmproj`).
|
||||
`q35.py` and the window scripts live only on the pod (not in the repo) and hard-code a build path.
|
||||
Before each use, edit them to point at the candidate build (`/root/arms/rebYYYYMMDD/build`), not
|
||||
production's `/root/llama.cpp/build`, or you validate the wrong binary.
|
||||
|
||||
On any FAIL: stop and report; the scratch dir (`/tmp/fork-validate.*`) is kept with all logs.
|
||||
|
||||
## 7. Deploy
|
||||
|
||||
Deploy exactly the validated commit. Downtime: 6 minutes for the 2026-10-24 redeploy. The deploy
|
||||
rebuilds the production tree in place (step 6) from the same commit and with the same cmake flags as
|
||||
the validated worktree build, so nothing is copied over; compare
|
||||
`grep -E 'GGML_NATIVE|LLAMA_BUILD_UI|BUILD_SHARED_LIBS|CMAKE_BUILD_TYPE' build/CMakeCache.txt` in both
|
||||
trees if in doubt.
|
||||
|
||||
Run the stop, build and start (steps 4-7) as one script on the pod, started with `setsid nohup sh
|
||||
<script> > <log> 2>&1 </dev/null &`, never as a backgrounded `cat && sh -n && setsid` chain from the
|
||||
`kubectl exec` shell (see section 9).
|
||||
|
||||
1. **Rollback pointer.** Record the current production commit in the next free pointer file, as the
|
||||
full SHA alone on line 1 (older files mix full SHAs, short SHAs and extra text; see section 9):
|
||||
```sh
|
||||
ls /root/DEPLOY-ROLLBACK-commit-*.txt # last used: 16 (holds b13c41355)
|
||||
git -C /root/llama.cpp rev-parse HEAD > /root/DEPLOY-ROLLBACK-commit-17.txt
|
||||
git -C /root/llama.cpp status --short # must be clean
|
||||
```
|
||||
2. **Check out the candidate** in the production tree (fetch first; the branch must be on gitea, or already local after a bundle fetch, section 3):
|
||||
```sh
|
||||
cd /root/llama.cpp
|
||||
git fetch origin fable5/vram-ondemand-rebase-YYYYMMDD
|
||||
git checkout -B fable5/vram-ondemand-rebase-YYYYMMDD <validated-commit>
|
||||
```
|
||||
On a redeploy with the same branch name this moves the production branch to the new commit.
|
||||
3. **Evict the page cache.** The pod is BestEffort QoS on a shared 62 GB node, so it is the first OOM
|
||||
victim, and benchmarks, mmap loads and builds without F10 leave tens of GB of model files cached.
|
||||
During the first 2026-10-24 deploy the Talos OOM controller killed the whole pod 17 s into
|
||||
verification (cause likely, not proven, memory pressure). Evict while production still runs
|
||||
(unprivileged and safe: `-lm none` servers hold their weights in anonymous memory, and pages
|
||||
mapped by an mmap model are not dropped). The same snippet is used before and after validation
|
||||
phase 2:
|
||||
```sh
|
||||
python3 - <<'EOF'
|
||||
import glob, os
|
||||
n = 0
|
||||
for root in ('/root/.cache/huggingface/hub', '/root/models'):
|
||||
for p in glob.glob(root + '/**/*.gguf', recursive=True):
|
||||
fd = os.open(p, os.O_RDONLY)
|
||||
try:
|
||||
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
|
||||
n += 1
|
||||
finally:
|
||||
os.close(fd)
|
||||
print('evicted', n, 'files')
|
||||
EOF
|
||||
```
|
||||
F10 (FORK.md) drops the cache of every `-lm none` load by itself (verified in production on a 35B
|
||||
GPU load: node page cache stayed at 4.2-5.3 GB instead of growing by about 20 GB), but files cached
|
||||
by mmap loads, benchmarks or older builds still need this step. `/sys/fs/cgroup/memory.stat`
|
||||
inside the container is the HOST root cgroup; the pod's own cgroup is under
|
||||
`/sys/fs/cgroup/kubepods/besteffort/pod<uid>/`.
|
||||
4. **Back up the current build, then stop llama-swap and reap orphans.** The deploy rebuilds
|
||||
`/root/llama.cpp/build` in place, so copy it first (production keeps serving during the copy);
|
||||
rollback (section 9) is then a copy, not a rebuild. Check `df -h /root` first (about 850 MB; the
|
||||
2026-11-21 deploy made `/root/arms/build-adc2aa3ec`, 851 MB):
|
||||
```sh
|
||||
cp -a /root/llama.cpp/build /root/arms/build-<oldsha> # <oldsha>: the HEAD recorded in step 1
|
||||
```
|
||||
Delete the copy once the deploy is confirmed stable. Then:
|
||||
```sh
|
||||
supervisorctl -c /root/supervisord.conf stop llama-swap
|
||||
ps -eo pid,ppid,etimes,args | grep '[l]lama-server'
|
||||
kill -TERM <pid> <pid> # explicit PIDs only, never pkill -f
|
||||
ps -eo pid,args | grep '[l]lama-server' # must be empty
|
||||
```
|
||||
`supervisorctl stop llama-swap` has left its llama-server children running as orphans (ppid 1),
|
||||
one still generating for a client for a minute. They exit within about 1-2 s of a SIGTERM. Keep
|
||||
this check even when it finds nothing (2026-10-25: no orphans).
|
||||
5. **Check memory**, now that the running models are gone: `grep -E 'MemAvailable|MemFree'
|
||||
/proc/meminfo`. Proceed only with MemAvailable >= 25 GB. In 2026-10-24 the eviction took node-wide
|
||||
file cache from 15.6 to 4.9 GB and MemAvailable to 45.8 GB.
|
||||
6. **Build in place**, now that nothing runs from `/root/llama.cpp/build/bin` (llama-swap's
|
||||
`LD_LIBRARY_PATH` points there, so do not rebuild it under running servers):
|
||||
```sh
|
||||
cd /root/llama.cpp && HF_UI_VERSION=$(cut -d'|' -f2 build/tools/ui/.ui-stamp) \
|
||||
cmake --build build --target llama-server llama-bench test-backend-ops -j 12
|
||||
ls -lL build/bin/llama-server build/bin/libllama-server-impl.so build/bin/libllama-common.so \
|
||||
build/bin/libllama.so build/bin/libmtmd.so build/bin/libggml.so build/bin/libggml-base.so \
|
||||
build/bin/libggml-cpu.so build/bin/libggml-vulkan.so
|
||||
```
|
||||
All must have fresh mtimes. Use `-L` (follow the symlinks): `build/bin` keeps every old versioned
|
||||
`libggml-*.so.0.x` and `libmtmd.so.0.x`, so a plain `ls` lists stale files beside the new ones.
|
||||
`llama-server` is a 17 KB stub. The server code is in `libllama-server-impl.so`, the common code
|
||||
(arguments, sampling) in `libllama-common.so`; a server change shows only in their mtimes.
|
||||
7. **Start and probe.**
|
||||
```sh
|
||||
supervisorctl -c /root/supervisord.conf start llama-swap
|
||||
curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/v1/models # 200
|
||||
```
|
||||
Sample `/proc/meminfo` every few seconds during the first model loads. Confirm the running commit
|
||||
from the first chat response (section 8): `system_fingerprint` is `b<build number>-<short sha>`
|
||||
(the build number changes with each rebase; the sha is the deployed commit). The first 35B request after that deploy took 34 s including
|
||||
the model load and the answer.
|
||||
8. **Disk prompt cache** (FORK.md F11), only once `--cache-disk-path` is in `/root/config.yaml`:
|
||||
- Every deploy empties the cache: the key holds the build commit, so the previous build's files are
|
||||
never restored. They stay in the directory as "other files" until the size limit evicts them.
|
||||
To free the space at once, `rm /root/prompt-cache/*.lpcd` while llama-swap is stopped (step 4).
|
||||
- Check `/root` free space (`df -h /root`; 11-13 GB on 2026-10-25). The cap (`--cache-disk-size`)
|
||||
plus a 1 GiB floor bounds the cache, but builds and model downloads share the PVC.
|
||||
- Stopping llama-swap takes longer: each large model writes its slot state on SIGTERM, about 1 s
|
||||
per GB of state, before it exits (FORK.md F7, F11).
|
||||
|
||||
## 8. Post-deploy checks
|
||||
|
||||
All through llama-swap on :8080. A request helper:
|
||||
|
||||
```sh
|
||||
ask() { curl -s -m 300 http://127.0.0.1:8080/v1/chat/completions -H 'Content-Type: application/json' \
|
||||
-d "{\"model\":\"$1\",\"messages\":[{\"role\":\"user\",\"content\":\"$2\"}],\"max_tokens\":32,\"temperature\":0}"; echo; }
|
||||
holder() { ino=$(stat -c %i /dev/shm/llama-vram/token.lock); grep "FLOCK.* WRITE .*:$ino " /proc/locks; }
|
||||
# image request body: a 64x64 solid red binary PPM, no image tools needed (like fork-validate.sh)
|
||||
img() { python3 - "$1" <<'EOF'
|
||||
import base64, json, sys
|
||||
ppm = b'P6\n64 64\n255\n' + b'\xff\x00\x00' * (64 * 64)
|
||||
url = 'data:image/x-portable-pixmap;base64,' + base64.b64encode(ppm).decode()
|
||||
print(json.dumps({'model': sys.argv[1], 'max_tokens': 32, 'temperature': 0, 'messages': [{'role': 'user',
|
||||
'content': [{'type': 'image_url', 'image_url': {'url': url}},
|
||||
{'type': 'text', 'text': 'What is the main colour of this image? One word.'}]}]}))
|
||||
EOF
|
||||
}
|
||||
# img MODEL | curl -s -m 300 http://127.0.0.1:8080/v1/chat/completions -H 'Content-Type: application/json' -d @-
|
||||
```
|
||||
|
||||
Use the `-nothink` aliases with these `max_tokens: 32` helpers. A thinking alias spends the 32 tokens
|
||||
on `reasoning_content` and returns an empty `content`; for one, raise `max_tokens` to >= 512.
|
||||
|
||||
1. `ask 'unsloth/Qwen3.6-35B-A3B-GGUF-nothink:UD-Q4_K_XL' 'Capital of France? One word.'` answers;
|
||||
`holder` shows the 35B's PID (check with `ps -o args -p <pid>`).
|
||||
2. Handoffs: alternate `Qwen3.5-4B-task:Q4_K_M` and the 35B two or three times. Each answer is
|
||||
correct and `holder` shows the PID of the model that just answered. Expect a few seconds per switch
|
||||
(2026-10-24: 3.1-5.6 s).
|
||||
3. Multimodal: one image request to `SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-nothink:UD-Q4_K_XL`
|
||||
with the `img` helper; the answer names red. This also forces an eviction in the llama-swap matrix: the
|
||||
evicted process must exit by itself, logging `VRAM arbiter: loop stopped, device memory freed,
|
||||
releasing the GPU token`, with no kill escalation.
|
||||
4. Text to gemma and to the 4B after the eviction; `holder` correct each time.
|
||||
With the disk prompt cache on (F11): send the 35B a chat of at least 256 tokens before step 3.
|
||||
Its eviction must log `disk cache: saved the prompt of N tokens`. Then resend the same chat plus one
|
||||
new user message to the 35B (earlier messages must stay byte-identical: editing an earlier message
|
||||
changes the prefix and legitimately gives no restore): the log shows `disk cache: restored`, and `timings.prompt_n` is
|
||||
only the new suffix.
|
||||
5. Ports: the children bind the `--port ${PORT}` values llama-swap passes (not upstream's default 9931).
|
||||
6. Log scan of the new llama-swap log lines (`curl -s http://127.0.0.1:8080/logs`) for
|
||||
`GGML_ASSERT|Assertion|terminate called|Segmentation fault|ErrorDeviceLost|ErrorOutOfDeviceMemory|DEVICE_LOST`: none.
|
||||
7. `dmesg` clean (no `ring ... timeout`, no `GPU_HUNG`). No pod restart and no OOM kill:
|
||||
- Restarts, from the workstation: `kubectl get pods -n llama | grep supervisord`; RESTARTS of the
|
||||
Running pod must equal the value you noted before the deploy. A stale `supervisord-...-pbmjq`
|
||||
in state `Unknown` is also listed; it is an old leftover, ignore it.
|
||||
- OOM kills, in the pod: `dmesg | grep -E 'OOM controller|Sending SIGKILL to cgroup'`. The
|
||||
timestamps are seconds since boot; compare them with the first field of `/proc/uptime` to see
|
||||
whether any falls in the deploy window (2026-10-25: the newest was about 19 h old).
|
||||
8. Inkling (FORK.md section 5, F12): judge the answer text, not only HTTP 200. The first request
|
||||
after a fresh load should be a prompt of about 150 tokens (upstream #32778 gives all-'?' output
|
||||
there); then a long prompt (Pan Tadeusz, 8K and 14K tokens, upstream #32755). Send an image
|
||||
and a tool request to both entries: since the nothink macro dropped `--reasoning-budget 0`
|
||||
(FORK.md F12) both answer them; repeated `<|message_model|><|content_thinking|><|end_message|>`
|
||||
from the nothink entry means a budget of 0 is back. Since F13 a scratch server with the
|
||||
production command line plus `-lv 4` must log `post_load_tensors: N views of host weights`
|
||||
(production's own log level hides libllama's info lines, so the llama-swap log never shows it),
|
||||
and the entries no longer set
|
||||
`GGML_OP_OFFLOAD_MIN_BATCH=64` (F13 measured the 32-63-token window correct without it). The
|
||||
placement is `-ot` with the loop layers 6..6+k-1 left on the GPU (FORK.md F13 placement guide);
|
||||
after a change of context or quant, check the fit again (the breakdown must leave room for the
|
||||
mmproj, about 0.8 GiB). The memory breakdown table is printed only at `-lv 4`,
|
||||
which production does not set, so it is not in the llama-swap log: read it from a scratch server
|
||||
started with production's command line plus `-lv 4`. Drivers used on 2026-11-22 (pod only):
|
||||
`/root/arms/carry1122-post.py` (through llama-swap) and `/root/arms/carry1122-ink-direct.py`
|
||||
(scratch server, GPU to itself).
|
||||
|
||||
A wedged arbiter looks like this: requests hang while `/health` still returns 200, and `/proc/locks`
|
||||
shows a process holding WRITE on `token.lock` that is not generating. llama-swap restarts a child
|
||||
that exits (on the next request for its model, FORK.md F7) but never one that wedges. Inspect it with
|
||||
`gdb -p <pid> -batch -ex 'thread apply all bt'` before killing it. gdb is not in
|
||||
the pod image: it was installed by hand and was gone after the 2026-10-24 pod restart, so check
|
||||
`command -v gdb` and reinstall it first if needed.
|
||||
|
||||
## 9. Rollback
|
||||
|
||||
```sh
|
||||
supervisorctl -c /root/supervisord.conf stop llama-swap
|
||||
ps -eo pid,args | grep '[l]lama-server' # SIGTERM any orphan by PID
|
||||
cd /root/llama.cpp
|
||||
git checkout -B <previous-production-branch> "$(head -1 /root/DEPLOY-ROLLBACK-commit-N.txt | cut -d' ' -f1)"
|
||||
HF_UI_VERSION=$(cut -d'|' -f2 build/tools/ui/.ui-stamp) cmake --build build --target llama-server llama-bench -j 12
|
||||
supervisorctl -c /root/supervisord.conf start llama-swap
|
||||
curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/v1/models
|
||||
```
|
||||
|
||||
This is the fallback when no build backup exists. With the deploy step 4 backup, roll back by copy
|
||||
instead (stop, checkout the old commit, replace the build dir, start):
|
||||
|
||||
```sh
|
||||
supervisorctl -c /root/supervisord.conf stop llama-swap
|
||||
ps -eo pid,args | grep '[l]lama-server' # SIGTERM any orphan by PID
|
||||
cd /root/llama.cpp
|
||||
git checkout -B <previous-production-branch> <oldsha>
|
||||
mv build /root/arms/build-failed-<newsha> # or rm -rf build; mind the disk
|
||||
cp -a /root/arms/build-<oldsha> build
|
||||
supervisorctl -c /root/supervisord.conf start llama-swap
|
||||
curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/v1/models
|
||||
```
|
||||
|
||||
Once the new deploy is confirmed stable, delete the copy (`rm -rf /root/arms/build-<oldsha>`; the
|
||||
disk is about 98% full) and any `build-failed-*` directory.
|
||||
|
||||
Then run the post-deploy checks again. Roll back on any failed check; report rather than debug in
|
||||
production. Keep the candidate worktree for a later redeploy.
|
||||
|
||||
If you run the rollback (or the deploy steps 4-7) from a workstation through `kubectl exec`, put the
|
||||
whole sequence in one script on the pod and start it as `setsid nohup sh /root/arms/rollback.sh >
|
||||
/root/arms/rollback.log 2>&1 </dev/null &`. Do not background a `cat > X && sh -n X && setsid ...`
|
||||
chain in the exec shell: it can die with the exec session halfway, leaving llama-swap stopped (section
|
||||
5). A rollback that also undoes a config change (for example removing `--cache-disk-path` from
|
||||
`/root/config.yaml`, F11) needs the config restored first, from its `/root/config.yaml.bak-*`.
|
||||
|
||||
Pointer files 2 to 16 exist; for example commit-8 holds `64543a435`, commit-9 `5751c977b`,
|
||||
commit-10 `6c480f7e7`, commit-11 was used by the failed cb48bf224 deploy, commit-12 holds
|
||||
`2fc9f9c35`, commit-13 holds `adc2aa3ec`, commit-14 holds `3bf6b55b3`, commit-15 holds `3621025e8`, and commit-16 holds `b13c41355`, the rollback target
|
||||
for the current production `f3397cded` (next free: 17; build copy `/root/arms/build-b13c41355`). The formats vary
|
||||
(commit-4 to 6 hold short SHAs, commit-6 has branch and date after the SHA, commit-9 a subject on
|
||||
line 2), hence `head -1 | cut -d' ' -f1`. New pointers: the full SHA alone on line 1.
|
||||
|
||||
## 10. Recovering a hung GPU
|
||||
|
||||
Symptoms: a process that survives SIGKILL (stuck in the driver); `ring sdma1 timeout` or
|
||||
`ring comp_... timeout` in the kernel log; after a reboot the supervisord pod sits in FailedMount with
|
||||
`/dev/kfd is not a character device` / `/dev/dri is not a directory`, and
|
||||
`talosctl get pcidevices` lists no VGA device (the RX 580 and its HDMI audio function are missing).
|
||||
A warm reboot does not bring the card back (classic Polaris failed reset).
|
||||
|
||||
The node (`anapistula-delrosalae`, Talos, bare metal) has a NanoKVM at http://192.168.255.8.
|
||||
Credentials: ask the owner. Procedure (owner's instructions, follow exactly):
|
||||
|
||||
1. Soft shutdown: **one short press** of the power button through the NanoKVM.
|
||||
2. **Never use the reset button.** It does not cut GPU power and is useless here.
|
||||
3. Wait until the power LED goes off.
|
||||
4. Press the power button once more to boot.
|
||||
5. Only if the LED is still on 10+ minutes after the short press: one **long press of 8 seconds** to
|
||||
force power off, wait for the LED to go off, then a short press to boot. With the GPU off the bus the
|
||||
soft shutdown has hung before (Talos services stopped, LED stayed on), and the long press was needed.
|
||||
|
||||
Talos access from the workstation:
|
||||
|
||||
```sh
|
||||
export TALOSCONFIG=/home/user/Projects/klaster/talos/generated/talosconfig
|
||||
nix run nixpkgs#talosctl -- -n anapistula-delrosalae get pcidevices
|
||||
nix run nixpkgs#talosctl -- -n anapistula-delrosalae dmesg
|
||||
```
|
||||
|
||||
Afterwards: pod Running, RX 580 listed, llama-swap RUNNING, `/v1/models` 200, and the 120 W cap active:
|
||||
`cat /sys/class/drm/card*/device/hwmon/hwmon*/power1_cap` reads `120000000` (the
|
||||
`power1_cap ... -> 120000000` line in `/tmp/fan-control.log` may be missing after a pod restart). The two 2026-10-09 hangs both came at the
|
||||
first 4096-token prefill right after a decode phase, before the cap was fixed at 120 W; both a fork
|
||||
and a clean-upstream build hung, so they were treated as hardware/power, not code.
|
||||
|
||||
## 11. Operations reference
|
||||
|
||||
Pod access:
|
||||
|
||||
```sh
|
||||
export KUBECONFIG=/home/user/Projects/klaster/talos/generated/kubeconfig
|
||||
kubectl -n llama exec -i deploy/supervisord -- sh -c '<command>'
|
||||
kubectl get pods -n llama -l app=llama-swap -o name # the pod name changes across restarts
|
||||
```
|
||||
|
||||
Wiring: k8s service `llama:11434` -> pod port 8080 (selector `app=llama-swap`) -> llama-swap `:8080`;
|
||||
a Caddy proxy fronts `llama.lumpiasty.xyz`. llama-swap must listen on `:8080`.
|
||||
|
||||
Files (all under `/root`, a persistent volume):
|
||||
- `/root/llama.cpp`: production tree, built in `build/` (shared libraries, so the real code is in
|
||||
`build/bin/libllama.so` and `libggml-*.so`). Scratch worktrees go in `/root/arms/`.
|
||||
- `/root/supervisord.conf`: supervisord (PID 1). `[program:llama-swap]` runs
|
||||
`/root/llama-swap -config /root/config.yaml -listen :8080 -watch-config`; its `environment=` sets
|
||||
`PATH` and `LD_LIBRARY_PATH` to `/root/llama.cpp/build/bin` and the arbiter variables (FORK.md section 5).
|
||||
- `/root/config.yaml`: llama-swap models, macros (`moe_offload_args`, `moe_offload_gemma`,
|
||||
`common_args`), matrix/exclusivity sets, preload. Backups `/root/config.yaml.bak-*`. Edit only with
|
||||
the owner's approval.
|
||||
- `/root/fan-control.sh`: fan curve and the fixed 120 W cap (separate supervisord program).
|
||||
- `/root/bench/`: benchmark results and pod-side helpers (`/root/bench/moe/swapdown.sh`).
|
||||
- `/root/DEPLOY-ROLLBACK-commit-N.txt`: rollback pointers.
|
||||
- `/root/prompt-cache/` (configured 2026-11-21, `--cache-disk-path /root/prompt-cache
|
||||
--cache-disk-size 4096` in the `moe_offload_args` and `moe_offload_gemma` macros; backup
|
||||
`/root/config.yaml.bak-diskcache-20261121`): disk prompt cache files `*.lpcd` of every model
|
||||
with the flag (FORK.md F11). Production check: a 35B 4.6k-token chat was saved on eviction (278.6
|
||||
MiB, copy 126 ms, write 495 ms), restored after the reload (read 416 ms, restore 33 ms) and turn 2
|
||||
prefilled 32 tokens; a gemma image request works with the flags on and is not cached. Safe to empty while llama-swap is stopped.
|
||||
- `/dev/shm/llama-vram/`: production arbiter arena (`token.lock`, `want.lock`; an inert `doorbell/`
|
||||
directory from the old design may remain). Never point a test at it.
|
||||
|
||||
Reloads: a `config.yaml` change is picked up by `-watch-config`, but a running model keeps its old
|
||||
arguments until it restarts (`supervisorctl -c /root/supervisord.conf restart llama-swap`). An
|
||||
`environment=` change needs `supervisorctl -c /root/supervisord.conf reread && supervisorctl -c
|
||||
/root/supervisord.conf update`.
|
||||
|
||||
Useful commands:
|
||||
- VRAM in use: `cat /sys/class/drm/card*/device/mem_info_vram_used`.
|
||||
- Token holder and waiters: `/proc/locks` filtered by the inode of `token.lock` / `want.lock`
|
||||
(WRITE = holder on token.lock, READ = registered waiters on want.lock).
|
||||
- 120 W cap active: `cat /sys/class/drm/card*/device/hwmon/hwmon*/power1_cap` = `120000000`.
|
||||
- The pod has `curl`, `python3`, `flock`; no `jq`, and no `gdb` unless reinstalled by hand since the
|
||||
last pod restart (`fork-validate.sh` then runs its SIGTERM-in-release case by polling the log).
|
||||
|
||||
Second test box (optional): `ssh user@gaming-pc`, the owner's NixOS desktop with an RX 6600 (RDNA2).
|
||||
Pod-built binaries can run there through the host glibc loader without compiling. Numbers from it do
|
||||
not transfer: about 2.25x the RX 580's prefill, 13% slower decode, and the GCN-only mask_opt change does
|
||||
not apply. Do not change its system config.
|
||||
|
||||
Not a fork problem: OpenWebUI's stop button does not cancel generation (its Redis task-command listener
|
||||
never handles the stop); llama-server and llama-swap both cancel correctly on client disconnect.
|
||||
@@ -109,6 +109,11 @@ extern "C" {
|
||||
GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
||||
GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
||||
|
||||
// Free transient/scratch device memory the backend holds outside of any allocated buffer
|
||||
// (compute preallocations, staging buffers). No-op if the backend does not implement it.
|
||||
// The backend remains usable; scratch is reallocated lazily on the next compute.
|
||||
GGML_API void ggml_backend_free_scratch(ggml_backend_t backend);
|
||||
|
||||
// NOTE: will be removed, use device version instead
|
||||
GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
||||
GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
|
||||
|
||||
@@ -2283,6 +2283,15 @@ extern "C" {
|
||||
struct ggml_tensor * a,
|
||||
struct ggml_tensor * sinks);
|
||||
|
||||
GGML_API bool ggml_flash_attn_ext_has_indexed_bias(const struct ggml_tensor * a);
|
||||
|
||||
// Add table[indices[key, query], head, query] to the mask before its F16 rounding.
|
||||
// Queries from all streams are flattened in stream order. Indices must be in [0, table->ne[0]).
|
||||
GGML_API void ggml_flash_attn_ext_add_indexed_bias(
|
||||
struct ggml_tensor * a,
|
||||
struct ggml_tensor * table,
|
||||
struct ggml_tensor * indices);
|
||||
|
||||
// TODO: needs to be adapted to ggml_flash_attn_ext
|
||||
GGML_API struct ggml_tensor * ggml_flash_attn_back(
|
||||
struct ggml_context * ctx,
|
||||
|
||||
@@ -162,6 +162,11 @@ extern "C" {
|
||||
|
||||
// (optional) sort/optimize the nodes in the graph
|
||||
void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params);
|
||||
|
||||
// (optional) free transient/scratch device memory the backend holds outside of any buffer
|
||||
// (e.g. compute preallocations and staging buffers). The backend stays usable; the scratch
|
||||
// is reallocated lazily on the next compute. Used to shrink an idle model's device footprint.
|
||||
void (*free_scratch) (ggml_backend_t backend);
|
||||
};
|
||||
|
||||
struct ggml_backend {
|
||||
|
||||
@@ -152,6 +152,10 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_
|
||||
static ggml_backend_buffer_type_t ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev);
|
||||
|
||||
static bool ggml_backend_meta_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
|
||||
if (op->op == GGML_OP_FLASH_ATTN_EXT && ggml_flash_attn_ext_has_indexed_bias(op)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
GGML_ASSERT(ggml_backend_dev_is_meta(dev));
|
||||
const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
|
||||
return std::all_of(meta_dev_ctx->simple_devs.begin(), meta_dev_ctx->simple_devs.end(),
|
||||
|
||||
@@ -775,6 +775,15 @@ void ggml_backend_synchronize(ggml_backend_t backend) {
|
||||
backend->iface.synchronize(backend);
|
||||
}
|
||||
|
||||
void ggml_backend_free_scratch(ggml_backend_t backend) {
|
||||
GGML_ASSERT(backend);
|
||||
if (backend->iface.free_scratch == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
backend->iface.free_scratch(backend);
|
||||
}
|
||||
|
||||
ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
|
||||
GGML_ASSERT(backend);
|
||||
GGML_ASSERT(backend->iface.graph_plan_create != NULL);
|
||||
|
||||
@@ -2679,8 +2679,8 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
|
||||
return true;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
{
|
||||
// K/V row lists (ggml_flash_attn_ext_rows) are not supported
|
||||
if (op->src[5] != nullptr) {
|
||||
// K/V row lists and indexed bias are not supported
|
||||
if (op->src[5] != nullptr || ggml_flash_attn_ext_has_indexed_bias(op)) {
|
||||
return false;
|
||||
}
|
||||
#ifdef ASCEND_310P
|
||||
|
||||
@@ -422,6 +422,10 @@ static ggml_backend_buffer_t ggml_backend_cpu_device_buffer_from_host_ptr(ggml_b
|
||||
}
|
||||
|
||||
static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
|
||||
if (op->op == GGML_OP_FLASH_ATTN_EXT && ggml_flash_attn_ext_has_indexed_bias(op)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const struct ggml_tensor * src0 = op->src[0];
|
||||
const struct ggml_tensor * src1 = op->src[1];
|
||||
|
||||
|
||||
@@ -18,6 +18,23 @@
|
||||
// The macro on the following line shifts it by a factor of 2**3=8, as was needed to fix https://github.com/ggml-org/llama.cpp/issues/18606 .
|
||||
#define FATTN_KQ_MAX_OFFSET (3.0f*0.6931f)
|
||||
|
||||
struct fattn_indexed_bias {
|
||||
const float * table;
|
||||
const int32_t * indices;
|
||||
int64_t n_bins;
|
||||
int64_t n_keys;
|
||||
int64_t query_stride;
|
||||
|
||||
__device__ fattn_indexed_bias for_head(int64_t first_query, int head) const {
|
||||
return {table + first_query*query_stride + head*n_bins, indices + first_query*n_keys, n_bins, n_keys, query_stride};
|
||||
}
|
||||
|
||||
__device__ half add(half mask, int query, int key) const {
|
||||
const int bin = indices[query*n_keys + key];
|
||||
return uint32_t(bin) < n_bins ? __float2half(__half2float(mask) + table[query*query_stride + bin]) : mask;
|
||||
}
|
||||
};
|
||||
|
||||
typedef void (* fattn_kernel_t)(
|
||||
const char * __restrict__ Q,
|
||||
const char * __restrict__ K,
|
||||
@@ -40,7 +57,8 @@ typedef void (* fattn_kernel_t)(
|
||||
const int32_t nb11, const int32_t nb12, const int64_t nb13,
|
||||
const int32_t nb21, const int32_t nb22, const int64_t nb23,
|
||||
const int32_t ne31, const int32_t ne32, const int32_t ne33,
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33);
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33,
|
||||
const fattn_indexed_bias bias);
|
||||
|
||||
typedef float (*vec_dot_KQ_t)(
|
||||
const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds);
|
||||
@@ -1170,7 +1188,7 @@ void launch_fattn(
|
||||
// Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.
|
||||
// Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or
|
||||
// multiple sequences of possibly different lengths, or kv_rows lists that end with padding.
|
||||
const bool scan_mask = !use_sparse && mask && mask->ne[2] == 1 && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows);
|
||||
const bool scan_mask = !dst->src[6] && !use_sparse && mask && mask->ne[2] == 1 && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows);
|
||||
if (scan_mask) {
|
||||
const int64_t s31 = mask->nb[1] / sizeof(half2);
|
||||
const int64_t s33 = mask->nb[3] / sizeof(half2);
|
||||
@@ -1324,7 +1342,12 @@ void launch_fattn(
|
||||
K->ne[0], n_kv, K->ne[2], K->ne[3], nb11, nb12, nb13,
|
||||
nb21, nb22, nb23,
|
||||
mask ? mask->ne[1] : 0, mask ? mask->ne[2] : 0, mask ? mask->ne[3] : 0,
|
||||
mask ? mask->nb[1] : 0, mask ? mask->nb[2] : 0, mask ? mask->nb[3] : 0
|
||||
mask ? mask->nb[1] : 0, mask ? mask->nb[2] : 0, mask ? mask->nb[3] : 0,
|
||||
fattn_indexed_bias{dst->src[6] ? (const float *) dst->src[6]->data : nullptr,
|
||||
dst->src[7] ? (const int32_t *) dst->src[7]->data : nullptr,
|
||||
dst->src[6] ? dst->src[6]->ne[0] : 0,
|
||||
dst->src[7] ? dst->src[7]->ne[0] : 0,
|
||||
dst->src[6] ? dst->src[6]->ne[0]*dst->src[6]->ne[1] : 0}
|
||||
);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
|
||||
|
||||
@@ -1960,7 +1960,9 @@ static __global__ void flash_attn_ext_f16(
|
||||
const int32_t nb11, const int32_t nb12, const int64_t nb13,
|
||||
const int32_t nb21, const int32_t nb22, const int64_t nb23,
|
||||
const int32_t ne31, const int32_t ne32, const int32_t ne33,
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33) {
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33,
|
||||
const fattn_indexed_bias bias) {
|
||||
GGML_UNUSED(bias);
|
||||
ggml_cuda_pdl_sync(); // TODO optimize placement
|
||||
#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))
|
||||
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
|
||||
|
||||
@@ -374,9 +374,9 @@ static constexpr __device__ int ggml_cuda_fattn_tile_get_nbatch_K(const int DKQ,
|
||||
}
|
||||
|
||||
// TODO: deduplicate with mma-f16
|
||||
template<int warp_size, int nwarps, int I, int J, int J_padding, bool oob_check>
|
||||
template<int warp_size, int nwarps, int I, int J, int J_padding, bool oob_check, bool use_rows>
|
||||
static __device__ __forceinline__ void flash_attn_tile_load_tile(
|
||||
const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int stride_KV, const int i_sup) {
|
||||
const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int stride_KV, const int i_sup, const int * rows) {
|
||||
constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();
|
||||
constexpr int cpy_ne = cpy_nb / 4;
|
||||
|
||||
@@ -400,6 +400,7 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile(
|
||||
const int i = i0 + threadIdx.y*stride_i + (stride_j == warp_size ? 0 : threadIdx.x / stride_j);
|
||||
|
||||
if (i0 + nwarps*stride_i <= I || i < I) {
|
||||
const int row = use_rows && (!oob_check || i < i_sup) ? rows[i] : i;
|
||||
#pragma unroll
|
||||
for (int j0 = j0_start; j0 < j0_stop; j0 += stride_j) {
|
||||
const int j = j0*cpy_ne + (stride_j == warp_size ? threadIdx.x : threadIdx.x % stride_j)*cpy_ne;
|
||||
@@ -407,7 +408,7 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile(
|
||||
const __align__(16) half2 zero[cpy_ne] = {{0.0f, 0.0f}};
|
||||
ggml_cuda_memcpy_1<cpy_nb>(
|
||||
tile_KV + i*(J/2 + J_padding) + j,
|
||||
!oob_check || i < i_sup ? KV + i*stride_KV + j : zero);
|
||||
(!oob_check || i < i_sup) && (!use_rows || row >= 0) ? KV + int64_t(row)*stride_KV + j : zero);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -424,9 +425,9 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile(
|
||||
ggml_cuda_unroll<7>{}(load);
|
||||
}
|
||||
|
||||
template<int warp_size, int nwarps, int I, int J, int J_padding, bool oob_check>
|
||||
template<int warp_size, int nwarps, int I, int J, int J_padding, bool oob_check, bool use_rows>
|
||||
static __device__ __forceinline__ void flash_attn_tile_load_tile(
|
||||
const half2 * const __restrict__ KV, float * const __restrict__ tile_KV, const int stride_KV, const int i_sup) {
|
||||
const half2 * const __restrict__ KV, float * const __restrict__ tile_KV, const int stride_KV, const int i_sup, const int * rows) {
|
||||
constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();
|
||||
constexpr int cpy_ne = cpy_nb / 4;
|
||||
|
||||
@@ -450,6 +451,7 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile(
|
||||
const int i = i0 + threadIdx.y*stride_i + (stride_j == warp_size ? 0 : threadIdx.x / stride_j);
|
||||
|
||||
if (i0 + nwarps*stride_i <= I || i < I) {
|
||||
const int row = use_rows && (!oob_check || i < i_sup) ? rows[i] : i;
|
||||
#pragma unroll
|
||||
for (int j0 = j0_start; j0 < j0_stop; j0 += stride_j) {
|
||||
const int j = j0*(cpy_ne/2) + (stride_j == warp_size ? threadIdx.x : threadIdx.x % stride_j)*(cpy_ne/2);
|
||||
@@ -457,7 +459,7 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile(
|
||||
const half2 zero[cpy_ne/2] = {{0.0f, 0.0f}};
|
||||
__align__(16) half2 tmp_h2[cpy_ne/2];
|
||||
ggml_cuda_memcpy_1<sizeof(tmp_h2)>(
|
||||
tmp_h2, !oob_check || i < i_sup ? KV + i*stride_KV + j : zero);
|
||||
tmp_h2, (!oob_check || i < i_sup) && (!use_rows || row >= 0) ? KV + int64_t(row)*stride_KV + j : zero);
|
||||
|
||||
__align__(16) float2 tmp_f2[cpy_ne/2];
|
||||
#pragma unroll
|
||||
@@ -481,16 +483,15 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile(
|
||||
|
||||
// Function that performs a single iteration in for the KQ matrix multiplication:
|
||||
template <int warp_size, int nwarps, int ncols1, int ncols2, int DKQ, int nbatch_fa, int nbatch_K,
|
||||
bool use_logit_softcap, bool oob_check, typename T_vec_dot>
|
||||
bool use_logit_softcap, bool oob_check, bool use_rows, typename T_vec_dot>
|
||||
static __device__ __forceinline__ void flash_attn_tile_iter_KQ(
|
||||
T_vec_dot * const Q_tmp,
|
||||
const half2 * const __restrict__ K_h2,
|
||||
T_vec_dot * const KV_tmp,
|
||||
const int stride_K2,
|
||||
const int k_VKQ_0,
|
||||
const int k_VKQ_sup,
|
||||
const int k_KQ_0,
|
||||
float * KQ_acc) {
|
||||
float * KQ_acc, const int * rows) {
|
||||
constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();
|
||||
constexpr int cpy_ne = cpy_nb / 4;
|
||||
|
||||
@@ -498,8 +499,8 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ(
|
||||
constexpr int cpw = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp
|
||||
constexpr int np = nwarps > ncols ? nwarps/ncols : 1; // number of parallel warps per Q column
|
||||
|
||||
flash_attn_tile_load_tile<warp_size, nwarps, nbatch_fa, nbatch_K, cpy_ne, oob_check>
|
||||
(K_h2 + int64_t(k_VKQ_0)*stride_K2 + k_KQ_0/2, KV_tmp, stride_K2, k_VKQ_sup);
|
||||
flash_attn_tile_load_tile<warp_size, nwarps, nbatch_fa, nbatch_K, cpy_ne, oob_check, use_rows>
|
||||
(K_h2 + k_KQ_0/2, KV_tmp, stride_K2, k_VKQ_sup, rows);
|
||||
__syncthreads();
|
||||
|
||||
#ifdef FAST_FP16_AVAILABLE
|
||||
@@ -556,7 +557,7 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ(
|
||||
|
||||
// Function that performs a single iteration of the main loop over up to nbatch_fa tokens.
|
||||
template <int warp_size, int nwarps, int ncols1, int ncols2, int DKQ, int DV, int nbatch_fa, int nbatch_K,
|
||||
bool use_logit_softcap, bool oob_check, typename T_vec_dot, typename T_KQ, typename T_acc>
|
||||
bool use_logit_softcap, bool oob_check, bool use_rows, bool indexed_bias, typename T_vec_dot, typename T_KQ, typename T_acc>
|
||||
static __device__ __forceinline__ void flash_attn_tile_iter(
|
||||
T_vec_dot * const Q_tmp,
|
||||
const half2 * const __restrict__ K_h2,
|
||||
@@ -575,7 +576,8 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
|
||||
T_acc * const VKQ,
|
||||
const int k_VKQ_0,
|
||||
const int k_VKQ_max,
|
||||
const int col_Q_0) {
|
||||
const int col_Q_0,
|
||||
const fattn_indexed_bias bias, const int * rows) {
|
||||
constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();
|
||||
constexpr int cpy_ne = cpy_nb / 4;
|
||||
|
||||
@@ -594,6 +596,9 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
|
||||
#endif // FAST_FP16_AVAILABLE
|
||||
static_assert(cpw % KQ_cs == 0, "bad KQ_cs");
|
||||
const int k_VKQ_sup = k_VKQ_max - k_VKQ_0; // k supremum, only smaller k values have valid KV data
|
||||
const half2 * K_tile = use_rows ? K_h2 : K_h2 + int64_t(k_VKQ_0)*stride_K2;
|
||||
const half2 * V_tile = use_rows ? V_h2 : V_h2 + int64_t(k_VKQ_0)*stride_V2;
|
||||
const int * tile_rows = use_rows ? rows + k_VKQ_0 : nullptr;
|
||||
|
||||
float KQ_max_new[cpw];
|
||||
#pragma unroll
|
||||
@@ -607,13 +612,13 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
|
||||
constexpr int nbatch_K_last = DKQ % nbatch_K;
|
||||
#pragma unroll
|
||||
for (int k_KQ_0 = 0; k_KQ_0 < DKQ - nbatch_K_last; k_KQ_0 += nbatch_K) {
|
||||
flash_attn_tile_iter_KQ<warp_size, nwarps, ncols1, ncols2, DKQ, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>(
|
||||
Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);
|
||||
flash_attn_tile_iter_KQ<warp_size, nwarps, ncols1, ncols2, DKQ, nbatch_fa, nbatch_K, use_logit_softcap, oob_check, use_rows>(
|
||||
Q_tmp, K_tile, KV_tmp, stride_K2, k_VKQ_sup, k_KQ_0, KQ_acc, tile_rows);
|
||||
}
|
||||
if (nbatch_K_last > 0) {
|
||||
constexpr int k_KQ_0 = DKQ - nbatch_K_last;
|
||||
flash_attn_tile_iter_KQ<warp_size, nwarps, ncols1, ncols2, DKQ, nbatch_fa, nbatch_K_last, use_logit_softcap, oob_check>(
|
||||
Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);
|
||||
flash_attn_tile_iter_KQ<warp_size, nwarps, ncols1, ncols2, DKQ, nbatch_fa, nbatch_K_last, use_logit_softcap, oob_check, use_rows>(
|
||||
Q_tmp, K_tile, KV_tmp, stride_K2, k_VKQ_sup, k_KQ_0, KQ_acc, tile_rows);
|
||||
}
|
||||
|
||||
// Apply logit softcap + mask, update KQ_max:
|
||||
@@ -636,8 +641,14 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
|
||||
}
|
||||
|
||||
if (!oob_check || i_KQ < k_VKQ_sup) {
|
||||
KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] += (ncols2 > 1 || mask) ?
|
||||
slope*__half2float(mask[j*stride_mask + k_VKQ_0 + i_KQ]) : 0.0f;
|
||||
const int k = k_VKQ_0 + i_KQ;
|
||||
if (ncols2 > 1 || mask) {
|
||||
half m = mask[int64_t(j)*stride_mask + k];
|
||||
if constexpr (indexed_bias) {
|
||||
m = bias.add(m, j, k);
|
||||
}
|
||||
KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] += slope*__half2float(m);
|
||||
}
|
||||
|
||||
KQ_max_new[jc0] = fmaxf(KQ_max_new[jc0], KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] + FATTN_KQ_MAX_OFFSET);
|
||||
}
|
||||
@@ -718,8 +729,8 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
|
||||
static_assert(nbatch_V % np == 0, "bad nbatch_V");
|
||||
#pragma unroll
|
||||
for (int k0 = 0; k0 < nbatch_fa; k0 += nbatch_V) {
|
||||
flash_attn_tile_load_tile<warp_size, nwarps, nbatch_V, DV, 0, oob_check>
|
||||
(V_h2 + int64_t(k_VKQ_0 + k0)*stride_V2, KV_tmp, stride_V2, k_VKQ_sup - k0);
|
||||
flash_attn_tile_load_tile<warp_size, nwarps, nbatch_V, DV, 0, oob_check, use_rows>
|
||||
(use_rows ? V_tile : V_tile + int64_t(k0)*stride_V2, KV_tmp, stride_V2, k_VKQ_sup - k0, use_rows ? tile_rows + k0 : nullptr);
|
||||
__syncthreads();
|
||||
|
||||
#ifdef FAST_FP16_AVAILABLE
|
||||
@@ -788,7 +799,7 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
|
||||
}
|
||||
}
|
||||
|
||||
template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap> // D == head size
|
||||
template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool indexed_bias, bool use_rows> // D == head size
|
||||
__launch_bounds__(ggml_cuda_fattn_tile_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_tile_get_occupancy(DKQ, DV, ncols1*ncols2))
|
||||
static __global__ void flash_attn_tile(
|
||||
const char * Q_ptr,
|
||||
@@ -812,7 +823,8 @@ static __global__ void flash_attn_tile(
|
||||
const int32_t nb11, const int32_t nb12, const int64_t nb13,
|
||||
const int32_t nb21, const int32_t nb22, const int64_t nb23,
|
||||
const int32_t ne31, const int32_t ne32, const int32_t ne33,
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33) {
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33,
|
||||
const fattn_indexed_bias bias) {
|
||||
#ifdef FLASH_ATTN_AVAILABLE
|
||||
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
|
||||
const char * GGML_CUDA_RESTRICT K = K_ptr;
|
||||
@@ -822,7 +834,6 @@ static __global__ void flash_attn_tile(
|
||||
const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr;
|
||||
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
||||
float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;
|
||||
GGML_UNUSED(KV_rows_ptr);
|
||||
|
||||
// Skip unused kernel variants for faster compilation:
|
||||
|
||||
@@ -853,7 +864,9 @@ static __global__ void flash_attn_tile(
|
||||
const int col_Q_0 = blockIdx.x * ncols1; // Index of the first Q column for this CUDA block to work on.
|
||||
|
||||
const int sequence = blockIdx.z / (ne02/ncols2);
|
||||
const int * rows = use_rows ? KV_rows_ptr + int64_t(sequence)*ne11 : nullptr;
|
||||
const int head0 = blockIdx.z*ncols2 - sequence*ne02; // == blockIdx.z % (ne02/ncols2)
|
||||
const auto head_bias = indexed_bias ? bias.for_head(int64_t(sequence)*ne01.z, head0) : bias;
|
||||
const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
|
||||
const float * Q_f = (const float *) (Q + nb03*sequence + nb02* head0);
|
||||
const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*(head0 / gqa_ratio));
|
||||
@@ -959,24 +972,24 @@ static __global__ void flash_attn_tile(
|
||||
int k_VKQ_0 = blockIdx.y*nbatch_fa;
|
||||
while (k_VKQ_0 <= k_VKQ_max - nbatch_fa) {
|
||||
constexpr bool oob_check = false;
|
||||
flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>
|
||||
flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check, use_rows, indexed_bias>
|
||||
(Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
|
||||
stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
|
||||
stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0, head_bias, rows);
|
||||
k_VKQ_0 += gridDim.y*nbatch_fa;
|
||||
}
|
||||
if (k_VKQ_0 < k_VKQ_max) {
|
||||
constexpr bool oob_check = true;
|
||||
flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>
|
||||
flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check, use_rows, indexed_bias>
|
||||
(Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
|
||||
stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
|
||||
stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0, head_bias, rows);
|
||||
}
|
||||
} else {
|
||||
// Branch without out-of-bounds checks.
|
||||
for (int k_VKQ_0 = blockIdx.y*nbatch_fa; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nbatch_fa) {
|
||||
constexpr bool oob_check = false;
|
||||
flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>
|
||||
flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check, use_rows, indexed_bias>
|
||||
(Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
|
||||
stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
|
||||
stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0, head_bias, rows);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1147,7 +1160,7 @@ static __global__ void flash_attn_tile(
|
||||
#endif // FLASH_ATTN_AVAILABLE
|
||||
}
|
||||
|
||||
template <int DKQ, int DV, int ncols2, bool use_logit_softcap>
|
||||
template <int DKQ, int DV, int ncols2, bool use_logit_softcap, bool indexed_bias = false, bool use_rows = false>
|
||||
static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * Q = dst->src[0];
|
||||
|
||||
@@ -1163,7 +1176,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
constexpr int cols_per_block = 64;
|
||||
const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap, indexed_bias, use_rows>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
@@ -1179,7 +1192,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
constexpr int cols_per_block = 32;
|
||||
const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap, indexed_bias, use_rows>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
@@ -1191,7 +1204,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
constexpr int cols_per_block = 16;
|
||||
const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap, indexed_bias, use_rows>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
@@ -1203,7 +1216,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
constexpr int cols_per_block = 8;
|
||||
const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap, indexed_bias, use_rows>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
@@ -1215,7 +1228,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
constexpr int cols_per_block = 4;
|
||||
const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap, indexed_bias, use_rows>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
@@ -1226,7 +1239,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
|
||||
constexpr int cols_per_block = 2;
|
||||
const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
|
||||
const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
|
||||
fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap, indexed_bias, use_rows>;
|
||||
launch_fattn<DV, cols_per_block/ncols2, ncols2>
|
||||
(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);
|
||||
return;
|
||||
@@ -1255,6 +1268,17 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm
|
||||
// a mask with a head dimension is not shared by the Q heads of a K/V head
|
||||
const bool use_gqa_opt = mask && mask->ne[2] == 1 && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0;
|
||||
|
||||
if constexpr (DKQ == 64 || DKQ == 128) {
|
||||
if (dst->src[6]) {
|
||||
if (dst->src[5]) {
|
||||
launch_fattn_tile_switch_ncols1<DKQ, DV, 1, use_logit_softcap, true, true>(ctx, dst);
|
||||
} else {
|
||||
launch_fattn_tile_switch_ncols1<DKQ, DV, 1, use_logit_softcap, true, false>(ctx, dst);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (DKQ == 320) {
|
||||
// This branch is only used for Mistral Small 4 which has a GQA ratio of 32.
|
||||
// On AMD, simply use that GQA ratio with 32 columns / block since we always have enough SRAM.
|
||||
|
||||
@@ -16,7 +16,7 @@ static constexpr __device__ int ggml_cuda_fattn_vec_get_nthreads_device() {
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wpass-failed"
|
||||
#endif // __clang__
|
||||
template<int D, int ncols, ggml_type type_K, ggml_type type_V, bool use_logit_softcap> // D == head size
|
||||
template<int D, int ncols, ggml_type type_K, ggml_type type_V, bool use_logit_softcap, bool indexed_bias = false, bool use_rows = false> // D == head size
|
||||
__launch_bounds__(ggml_cuda_fattn_vec_get_nthreads_device(), 1)
|
||||
static __global__ void flash_attn_ext_vec(
|
||||
const char * Q_ptr,
|
||||
@@ -40,7 +40,8 @@ static __global__ void flash_attn_ext_vec(
|
||||
const int32_t nb11, const int32_t nb12, const int64_t nb13,
|
||||
const int32_t nb21, const int32_t nb22, const int64_t nb23,
|
||||
const int32_t ne31, const int32_t ne32, const int32_t ne33,
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33) {
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33,
|
||||
const fattn_indexed_bias bias) {
|
||||
ggml_cuda_pdl_lc();
|
||||
#ifdef FLASH_ATTN_AVAILABLE
|
||||
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
|
||||
@@ -51,7 +52,6 @@ static __global__ void flash_attn_ext_vec(
|
||||
const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr;
|
||||
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
||||
float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;
|
||||
GGML_UNUSED(KV_rows_ptr);
|
||||
|
||||
// Skip unused kernel variants for faster compilation:
|
||||
if (use_logit_softcap && !(D == 128 || D == 256)) {
|
||||
@@ -107,6 +107,7 @@ static __global__ void flash_attn_ext_vec(
|
||||
|
||||
const int sequence = blockIdx.z / ne02;
|
||||
const int head = blockIdx.z - sequence*ne02;
|
||||
const auto head_bias = indexed_bias ? bias.for_head(int64_t(sequence)*ne01.z, head) : bias;
|
||||
const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
|
||||
Q += nb03*sequence + nb02* head + nb01*ic0;
|
||||
K += nb13*sequence + nb12*(head / gqa_ratio);
|
||||
@@ -271,12 +272,15 @@ static __global__ void flash_attn_ext_vec(
|
||||
}
|
||||
|
||||
const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11;
|
||||
K += blockIdx.y*nthreads * nb11;
|
||||
V += blockIdx.y*nthreads * nb21;
|
||||
const int * rows = use_rows ? KV_rows_ptr + int64_t(sequence)*ne11 : nullptr;
|
||||
if constexpr (!use_rows) {
|
||||
K += blockIdx.y*nthreads * nb11;
|
||||
V += blockIdx.y*nthreads * nb21;
|
||||
}
|
||||
maskh += blockIdx.y*nthreads;
|
||||
for (int k_VKQ_0 = blockIdx.y*nthreads; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nthreads,
|
||||
// Increment pointers after each loop:
|
||||
K += gridDim.y*nthreads*nb11, V += gridDim.y*nthreads*nb21, maskh += gridDim.y*nthreads) {
|
||||
K += use_rows ? 0 : gridDim.y*nthreads*nb11, V += use_rows ? 0 : gridDim.y*nthreads*nb21, maskh += gridDim.y*nthreads) {
|
||||
|
||||
// Calculate KQ tile and keep track of new maximum KQ values:
|
||||
float KQ_reg[ncols]; // KQ in registers.
|
||||
@@ -290,22 +294,24 @@ static __global__ void flash_attn_ext_vec(
|
||||
#pragma unroll
|
||||
for (int i_KQ_0 = 0; i_KQ_0 < nthreads_KQ; ++i_KQ_0) {
|
||||
const int i_KQ = threadIdx.y*WARP_SIZE + (nthreads_KQ == WARP_SIZE ? 0 : (threadIdx.x & ~(nthreads_KQ-1))) + i_KQ_0;
|
||||
const int row = use_rows ? rows[k_VKQ_0 + i_KQ] : i_KQ;
|
||||
|
||||
#pragma unroll
|
||||
for (int j = 0; j < ncols; ++j) {
|
||||
float sum = vec_dot_KQ(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j]);
|
||||
float sum = vec_dot_KQ(K + (use_rows ? int64_t(max(row, 0))*nb11 : i_KQ*nb11), Q_reg[j], Q_i32[j], Q_ds[j]);
|
||||
sum = warp_reduce_sum<nthreads_KQ>(sum);
|
||||
|
||||
if (use_logit_softcap) {
|
||||
sum = logit_softcap*tanhf(sum);
|
||||
}
|
||||
|
||||
if (mask && (ncols == 1 || ic0 + j < int(ne01.z))) {
|
||||
sum += slope*__half2float(maskh[j*ne11 + i_KQ]);
|
||||
if constexpr (!indexed_bias) {
|
||||
if (mask && (ncols == 1 || ic0 + j < int(ne01.z))) {
|
||||
sum += slope*__half2float(maskh[j*ne11 + i_KQ]);
|
||||
}
|
||||
KQ_max_new[j] = fmaxf(KQ_max_new[j], sum + FATTN_KQ_MAX_OFFSET);
|
||||
}
|
||||
|
||||
KQ_max_new[j] = fmaxf(KQ_max_new[j], sum + FATTN_KQ_MAX_OFFSET);
|
||||
|
||||
if ((nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ) == uint32_t(i_KQ_0)) {
|
||||
KQ_reg[j] = sum;
|
||||
}
|
||||
@@ -314,8 +320,14 @@ static __global__ void flash_attn_ext_vec(
|
||||
|
||||
#pragma unroll
|
||||
for (int j = 0; j < ncols; ++j) {
|
||||
if constexpr (indexed_bias) {
|
||||
const int k = k_VKQ_0 + tid;
|
||||
const half m = use_rows && rows[k] < 0 ? half(-INFINITY) : head_bias.add(maskh[j*ne11 + tid], ic0 + j, k);
|
||||
KQ_reg[j] += slope*__half2float(m);
|
||||
KQ_max_new[j] = fmaxf(KQ_max[j], KQ_reg[j] + FATTN_KQ_MAX_OFFSET);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int offset = nthreads_KQ; offset < WARP_SIZE; offset <<= 1) {
|
||||
for (int offset = indexed_bias ? 1 : nthreads_KQ; offset < WARP_SIZE; offset <<= 1) {
|
||||
KQ_max_new[j] = fmaxf(KQ_max_new[j], ggml_cuda_shfl_xor(KQ_max_new[j], offset, WARP_SIZE));
|
||||
}
|
||||
const float KQ_max_scale = expf(KQ_max[j] - KQ_max_new[j]);
|
||||
@@ -345,6 +357,7 @@ static __global__ void flash_attn_ext_vec(
|
||||
#pragma unroll
|
||||
for (int k0 = 0; k0 < WARP_SIZE; k0 += V_cols_per_iter) {
|
||||
const int k = threadIdx.y*WARP_SIZE + k0 + (nthreads_V == WARP_SIZE ? 0 : threadIdx.x / nthreads_V);
|
||||
const char * V_row = V + (use_rows ? int64_t(max(rows[k_VKQ_0 + k], 0))*nb21 : k*nb21);
|
||||
|
||||
#ifdef V_DOT2_F32_F16_AVAILABLE
|
||||
half2 KQ_k[ncols];
|
||||
@@ -357,14 +370,14 @@ static __global__ void flash_attn_ext_vec(
|
||||
half2 tmp[V_rows_per_thread/2];
|
||||
if constexpr (type_V == GGML_TYPE_BF16) {
|
||||
float2 tmp_f[V_rows_per_thread/2];
|
||||
dequantize_V(V + k*nb21, tmp_f,
|
||||
dequantize_V(V_row, tmp_f,
|
||||
2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread);
|
||||
#pragma unroll
|
||||
for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) {
|
||||
tmp[i_VKQ_1] = __float22half2_rn(tmp_f[i_VKQ_1]);
|
||||
}
|
||||
} else {
|
||||
dequantize_V(V + k*nb21, tmp,
|
||||
dequantize_V(V_row, tmp,
|
||||
2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread);
|
||||
}
|
||||
#pragma unroll
|
||||
@@ -384,7 +397,7 @@ static __global__ void flash_attn_ext_vec(
|
||||
#pragma unroll
|
||||
for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {
|
||||
float2 tmp[V_rows_per_thread/2];
|
||||
dequantize_V(V + k*nb21, tmp,
|
||||
dequantize_V(V_row, tmp,
|
||||
2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread);
|
||||
#pragma unroll
|
||||
for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) {
|
||||
@@ -545,7 +558,7 @@ static __global__ void flash_attn_ext_vec(
|
||||
nb11, nb12, nb13,
|
||||
nb21, nb22, nb23,
|
||||
ne31, ne32, ne33,
|
||||
nb31, nb32, nb33);
|
||||
nb31, nb32, nb33, bias);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // FLASH_ATTN_AVAILABLE
|
||||
}
|
||||
@@ -560,6 +573,12 @@ void ggml_cuda_flash_attn_ext_vec_case_impl(ggml_backend_cuda_context & ctx, ggm
|
||||
const int nthreads = ggml_cuda_fattn_vec_get_nthreads_host(cc);
|
||||
const int nwarps = nthreads / WARP_SIZE;
|
||||
fattn_kernel_t fattn_kernel = flash_attn_ext_vec<D, cols_per_block, type_K, type_V, use_logit_softcap>;
|
||||
if constexpr (cols_per_block == 1 && (D == 64 || D == 128) && !use_logit_softcap) {
|
||||
if (dst->src[6]) {
|
||||
fattn_kernel = dst->src[5] ? flash_attn_ext_vec<D, cols_per_block, type_K, type_V, use_logit_softcap, true, true> :
|
||||
flash_attn_ext_vec<D, cols_per_block, type_K, type_V, use_logit_softcap, true, false>;
|
||||
}
|
||||
}
|
||||
const bool need_f16_K = type_K == GGML_TYPE_F16;
|
||||
const bool need_f16_V = type_V == GGML_TYPE_F16;
|
||||
constexpr size_t nbytes_shared = 0;
|
||||
|
||||
@@ -669,6 +669,18 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
|
||||
if (ggml_flash_attn_ext_has_indexed_bias(dst)) {
|
||||
const float softcap = ggml_get_op_params_f32(dst, 2);
|
||||
if (Q->ne[0] != 128 && !(Q->ne[0] == 64 && softcap == 0.0f)) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
if (softcap == 0.0f && Q->ne[1] == 1 && n_kv % FATTN_KQ_STRIDE == 0 &&
|
||||
ggml_cuda_get_fattn_vec_case(Q->ne[0], K->type, V->type)) {
|
||||
return BEST_FATTN_KERNEL_VEC;
|
||||
}
|
||||
return BEST_FATTN_KERNEL_TILE;
|
||||
}
|
||||
|
||||
if (mask && mask->ne[2] != 1) {
|
||||
// the kv_rows mask is compacted per sequence, it has no head dimension
|
||||
if (kv_rows) {
|
||||
|
||||
@@ -1277,6 +1277,7 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
||||
(op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && op->src[2] &&
|
||||
(op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr && op->src[5] == nullptr &&
|
||||
!ggml_flash_attn_ext_has_indexed_bias(op) &&
|
||||
ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0])) {
|
||||
float max_bias = 0.0f;
|
||||
float logit_softcap = 0.0f;
|
||||
|
||||
@@ -355,8 +355,8 @@ static bool htp_kparams_solve_flash_attn(
|
||||
return false;
|
||||
}
|
||||
|
||||
// K/V row lists (ggml_flash_attn_ext_rows) are not supported
|
||||
if (op->src[5] != nullptr) {
|
||||
// K/V row lists and indexed bias are not supported
|
||||
if (op->src[5] != nullptr || ggml_flash_attn_ext_has_indexed_bias(op)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -1744,6 +1744,9 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
case GGML_OP_ROLL:
|
||||
return ggml_is_contiguous(op->src[0]);
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
if (ggml_flash_attn_ext_has_indexed_bias(op)) {
|
||||
return false;
|
||||
}
|
||||
// K/V row lists (ggml_flash_attn_ext_rows): one list of cache rows per q slice
|
||||
if (op->src[5] != NULL) {
|
||||
const struct ggml_tensor * kv_rows = op->src[5];
|
||||
|
||||
@@ -12426,8 +12426,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
case GGML_OP_MEAN:
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_FLASH_ATTN_EXT: {
|
||||
// K/V row lists (ggml_flash_attn_ext_rows) are not supported
|
||||
if (op->src[5] != nullptr) {
|
||||
// K/V row lists and indexed bias are not supported
|
||||
if (op->src[5] != nullptr || ggml_flash_attn_ext_has_indexed_bias(op)) {
|
||||
return false;
|
||||
}
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
|
||||
@@ -1905,6 +1905,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
return {false, "FLASH_ATTN_EXT gemma3n pattern on GPU is not supported"};
|
||||
}
|
||||
|
||||
if (ggml_flash_attn_ext_has_indexed_bias(op)) {
|
||||
return {false, "FLASH_ATTN_EXT with indexed bias is not supported"};
|
||||
}
|
||||
|
||||
// K/V row lists (ggml_flash_attn_ext_rows) are not supported
|
||||
if (op->src[5] != nullptr) {
|
||||
return {false, "FLASH_ATTN_EXT with K/V row lists is not supported"};
|
||||
|
||||
@@ -3113,9 +3113,8 @@ static ggml_backend_buffer_type_t ggml_backend_rpc_device_get_buffer_type(ggml_b
|
||||
|
||||
static bool ggml_backend_rpc_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
|
||||
GGML_UNUSED(dev);
|
||||
GGML_UNUSED(op);
|
||||
//TODO: call the remote backend and cache the results
|
||||
return true;
|
||||
return op->op != GGML_OP_FLASH_ATTN_EXT || !ggml_flash_attn_ext_has_indexed_bias(op);
|
||||
}
|
||||
|
||||
static bool ggml_backend_rpc_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
|
||||
|
||||
@@ -7741,8 +7741,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
return op->src[0]->ne[0] <= SYCL_SOLVE_TRI_MAX_N && op->src[1]->ne[0] <= SYCL_SOLVE_TRI_MAX_K;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
// K/V row lists (ggml_flash_attn_ext_rows) are not supported
|
||||
return op->src[5] == nullptr && ggml_sycl_flash_attn_ext_supported(device, op);
|
||||
// K/V row lists and indexed bias are not supported
|
||||
return op->src[5] == nullptr && !ggml_flash_attn_ext_has_indexed_bias(op) && ggml_sycl_flash_attn_ext_supported(device, op);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -30,6 +30,10 @@ static void ggml_backend_remoting_device_get_memory(ggml_backend_dev_t dev, size
|
||||
}
|
||||
|
||||
static bool ggml_backend_remoting_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
|
||||
if (op->op == GGML_OP_FLASH_ATTN_EXT && ggml_flash_attn_ext_has_indexed_bias(op)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((op->op == GGML_OP_ROPE || op->op == GGML_OP_ROPE_BACK) &&
|
||||
!ggml_rope_get_truncate(op)) {
|
||||
return false;
|
||||
|
||||
@@ -1256,6 +1256,10 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device,
|
||||
|
||||
result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4);
|
||||
|
||||
// Staging K/V through shared memory is a loss on AMD GCN: measured -6.7% at depth 16k and
|
||||
// -7.4% at 32k on Polaris (head size 128). The kvsh stride of D/4+1 dwords is 4 mod 32, which
|
||||
// costs an 8-way LDS bank conflict on wave64, and the extra shared memory cuts into occupancy
|
||||
// that this shader is already short of. Keep it to NVIDIA.
|
||||
result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0;
|
||||
|
||||
if (!reduce_block_rows && result.block_rows > 1 && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
||||
@@ -2333,7 +2337,12 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
const void * spv_data = nullptr;
|
||||
size_t spv_size = 0;
|
||||
const char *name = nullptr;
|
||||
if (bf16_kv) {
|
||||
const bool kv_rows = (fa.first.flags & (1u << 31)) != 0;
|
||||
if (kv_rows) {
|
||||
spv_data = device->fp16 ? flash_attn_f32_f16_rows_data : flash_attn_f32_f16_rows_fp32_data;
|
||||
spv_size = device->fp16 ? flash_attn_f32_f16_rows_len : flash_attn_f32_f16_rows_fp32_len;
|
||||
name = "flash_attn_f32_f16_rows";
|
||||
} else if (bf16_kv) {
|
||||
spv_data = flash_attn_f32_f16_fp32_data;
|
||||
spv_size = flash_attn_f32_f16_fp32_len;
|
||||
name = aligned ? "flash_attn_f32_bf16_aligned" : "flash_attn_f32_bf16";
|
||||
@@ -2363,7 +2372,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
}
|
||||
name = aligned ? "flash_attn_f32_f16_aligned" : "flash_attn_f32_f16";
|
||||
}
|
||||
ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8,
|
||||
ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", kv_rows ? 9 : 8,
|
||||
sizeof(vk_flash_attn_push_constants), {Br, 1, 1},
|
||||
get_fa_spec_constants(fa.first), aligned ? Bc : 1, true,
|
||||
!fa_ds, !fa_ds ? fa_sgs : 0);
|
||||
@@ -9025,10 +9034,12 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
const uint32_t nem2 = mask ? mask->ne[2] : 0;
|
||||
const uint32_t nem3 = mask ? mask->ne[3] : 0;
|
||||
|
||||
const ggml_tensor * bias = dst->src[6];
|
||||
const uint32_t HSK = nek0;
|
||||
const uint32_t HSV = nev0;
|
||||
uint32_t N = neq1;
|
||||
const uint32_t KV = nek1;
|
||||
const ggml_tensor * rows = dst->src[5];
|
||||
const uint32_t KV = rows ? rows->ne[0] : nek1;
|
||||
|
||||
GGML_ASSERT(ne0 == HSV);
|
||||
GGML_ASSERT(ne2 == N);
|
||||
@@ -9085,7 +9096,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
const uint32_t cm2_min_reuse = KV >= 8192 ? 4096 : 8192;
|
||||
// Query reuse must cover the conversion and synchronization cost.
|
||||
const bool cm2_dequant_kv = KV >= 4096 && neq1 >= cm2_min_queries && (uint64_t) neq1 * qk_ratio >= cm2_min_reuse;
|
||||
const bool use_dequant_kv = ((k_quant && v_quant) || kv_raw_strided) && neq1 >= 64 &&
|
||||
const bool use_dequant_kv = !rows && ((k_quant && v_quant) || kv_raw_strided) && neq1 >= 64 &&
|
||||
is_dense_kv_cache(k) && is_dense_kv_cache(v) &&
|
||||
(uint64_t)ggml_nelements(k) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange &&
|
||||
(uint64_t)ggml_nelements(v) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange &&
|
||||
@@ -9105,7 +9116,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
vk_fa_tuning_params tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, 32, KV, k_type_eff, v_type_eff, f32acc, qk_ratio);
|
||||
const uint32_t max_gqa = std::min(tuning_params.block_rows, 32u);
|
||||
|
||||
if (N <= 8 && qk_ratio > 1 && qk_ratio <= max_gqa &&
|
||||
if (!bias && N <= 8 && qk_ratio > 1 && qk_ratio <= max_gqa &&
|
||||
qk_ratio * nek2 == neq2 && nek2 == nev2 && nem2 <= 1) {
|
||||
// grouped query attention - make the N dimension equal to gqa_ratio, reduce
|
||||
// workgroups proportionally in y dimension. The shader will detect gqa_ratio > 1
|
||||
@@ -9115,7 +9126,8 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
workgroups_y /= gqa_ratio;
|
||||
}
|
||||
|
||||
tuning_params = get_fa_tuning_params(ctx->device, HSK, HSV, N, KV, k_type_eff, v_type_eff, f32acc, qk_ratio);
|
||||
tuning_params = bias ? get_fa_tuning_params_scalar(ctx->device, HSK, HSV, N, KV, k_type_eff, v_type_eff, f32acc)
|
||||
: get_fa_tuning_params(ctx->device, HSK, HSV, N, KV, k_type_eff, v_type_eff, f32acc, qk_ratio);
|
||||
|
||||
float scale = 1.0f;
|
||||
float max_bias = 0.0f;
|
||||
@@ -9141,7 +9153,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
return (t->type != GGML_TYPE_F16 && t->type != GGML_TYPE_BF16) ||
|
||||
(t->nb[1] | t->nb[2] | t->nb[3]) % (4 * sizeof(ggml_fp16_t)) == 0;
|
||||
};
|
||||
const bool use_sparse = !disable_sparse && n_kv_max > 0 && mask &&
|
||||
const bool use_sparse = !bias && !disable_sparse && n_kv_max > 0 && mask &&
|
||||
max_bias == 0.0f && logit_softcap == 0.0f &&
|
||||
(tuning_params.path != FA_COOPMAT2 || (sparse_gather_aligned(k) && sparse_gather_aligned(v))) &&
|
||||
nem0 == KV &&
|
||||
@@ -9200,11 +9212,20 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
}
|
||||
|
||||
// Only use mask opt when the mask is fairly large. This hasn't been tuned extensively.
|
||||
bool use_mask_opt = mask && !use_sparse && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16
|
||||
&& (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256);
|
||||
// GCN with large head size (>= 256) benefits in high-context prefill (skipping the
|
||||
// per-block mask add on fully-visible blocks dominates), so enable it there too.
|
||||
bool use_mask_opt = !bias && mask && !use_sparse && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16
|
||||
&& (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK >= 256 || HSV >= 256);
|
||||
vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc,
|
||||
mask != nullptr, use_mask_opt, logit_softcap != 0, use_sparse, k_type_eff, v_type_eff);
|
||||
|
||||
if (bias) {
|
||||
// Bind the table and indices in the two unused mask optimization slots.
|
||||
fa_pipeline_state.flags |= (uint32_t) bias->ne[0] << 8;
|
||||
if (rows) {
|
||||
fa_pipeline_state.flags |= 1u << 31;
|
||||
}
|
||||
}
|
||||
vk_pipeline pipeline = nullptr;
|
||||
|
||||
auto tensor_aligned_with = [&](const ggml_tensor * t, uint32_t in) {
|
||||
@@ -9219,7 +9240,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
tensor_aligned_with(k, 16u) && tensor_aligned_with(v, 16u);
|
||||
bool xe_fa_supported_dtype = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 &&
|
||||
mask != nullptr && mask->type == GGML_TYPE_F16;
|
||||
xe_fa_opt = xe_fa_supported_platform && xe_fa_supported_usage && xe_fa_supported_dtype;
|
||||
xe_fa_opt = !bias && xe_fa_supported_platform && xe_fa_supported_usage && xe_fa_supported_dtype;
|
||||
std::pair<vk_pipeline, vk_pipeline> xe_fa_pipeline_dual_phases = { nullptr , nullptr };
|
||||
vk_pipeline xe_fa_pipeline = nullptr;
|
||||
size_t size_q_staging = 0;
|
||||
@@ -9433,6 +9454,10 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
vk_subbuffer sinks_buf = sinks ? ggml_vk_tensor_subbuffer(ctx, sinks) : q_buf;
|
||||
vk_subbuffer mask_opt_buf = use_mask_opt ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf;
|
||||
vk_subbuffer sparse_buf = use_sparse ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf;
|
||||
if (bias) {
|
||||
mask_opt_buf = ggml_vk_tensor_subbuffer(ctx, bias);
|
||||
sparse_buf = ggml_vk_tensor_subbuffer(ctx, dst->src[7]);
|
||||
}
|
||||
if (use_mask_opt || use_sparse) {
|
||||
// the mask opt bits and the sparse index list overwrite a matmul input converted into prealloc_y
|
||||
ctx->prealloc_y_last_pipeline_used = nullptr;
|
||||
@@ -9548,6 +9573,16 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
mask_n_head_log2, m0, m1,
|
||||
gqa_ratio, split_kv, split_k };
|
||||
|
||||
auto dispatch_fa = [&](const vk_subbuffer & out, std::array<uint32_t, 3> groups) {
|
||||
if (rows) {
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
|
||||
{q_buf, k_buf, v_buf, mask_buf, sinks_buf, out, mask_opt_buf, sparse_buf, ggml_vk_tensor_subbuffer(ctx, rows)}, pc, groups);
|
||||
} else {
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
|
||||
{q_buf, k_buf, v_buf, mask_buf, sinks_buf, out, mask_opt_buf, sparse_buf}, pc, groups);
|
||||
}
|
||||
};
|
||||
|
||||
if (xe_fa_opt) {
|
||||
uint32_t head_stride_q = uint32_t(nbq2 / ggml_type_size(q->type));
|
||||
uint32_t head_stride_k = uint32_t(nbk2 / ggml_type_size(k->type));
|
||||
@@ -9619,9 +9654,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
}
|
||||
|
||||
vk_subbuffer split_k_buf = ggml_vk_subbuffer(ctx, ctx->prealloc_split_k, 0);
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
|
||||
{q_buf, k_buf, v_buf, mask_buf, sinks_buf, split_k_buf, mask_opt_buf, sparse_buf},
|
||||
pc, { dispatch_x, workgroups_y, workgroups_z });
|
||||
dispatch_fa(split_k_buf, { dispatch_x, workgroups_y, workgroups_z });
|
||||
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
const vk_op_flash_attn_split_k_reduce_push_constants pc2 = { HSV, (uint32_t)ne1, (uint32_t)ne2, (uint32_t)ne3, split_k, (sinks != nullptr) };
|
||||
@@ -9634,9 +9667,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
// When using gqa, we want one actual workgroup per batch, so cancel out wg_denoms
|
||||
workgroups_x *= pipeline->wg_denoms[0];
|
||||
}
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
|
||||
{q_buf, k_buf, v_buf, mask_buf, sinks_buf, dst_buf, mask_opt_buf, sparse_buf},
|
||||
pc, { workgroups_x, workgroups_y, workgroups_z });
|
||||
dispatch_fa(dst_buf, { workgroups_x, workgroups_y, workgroups_z });
|
||||
}
|
||||
|
||||
if (use_dequant_kv) {
|
||||
@@ -9743,7 +9774,21 @@ static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> &
|
||||
return pipelines[2];
|
||||
}
|
||||
|
||||
// fork: GGML_VK_CONCAT_FASTPATH=0 forces the generic concat kernel (validation / A-B knob, see ggml_vk_concat)
|
||||
static bool ggml_vk_concat_transpose_enabled() {
|
||||
static const char * env = getenv("GGML_VK_CONCAT_FASTPATH");
|
||||
static const bool off = env != nullptr && std::string(env) == "0";
|
||||
return !off;
|
||||
}
|
||||
|
||||
static bool ggml_vk_concat_is_transposed(const vk_device & device, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) {
|
||||
if (!ggml_vk_concat_transpose_enabled()) {
|
||||
return false;
|
||||
}
|
||||
// below one tile (decode, n_tokens < 32) the tiled path is pure overhead, the generic kernel is faster (FORK.md F3)
|
||||
if (src1->ne[0] < 32) {
|
||||
return false;
|
||||
}
|
||||
if (ggml_get_op_params_i32(dst, 0) != 0) { // dim 0 only
|
||||
return false;
|
||||
}
|
||||
@@ -11595,9 +11640,120 @@ void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, con
|
||||
ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_OPT_STEP_SGD, { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f });
|
||||
}
|
||||
|
||||
// Fast path for concat along dim 0 where one source is stored "transposed"
|
||||
// (nb[1] == type_size, dim1 innermost) and the other source + dst are
|
||||
// contiguous along dim 0. The generic concat shader reads the transposed
|
||||
// source with a catastrophic uncoalesced stride; here we instead copy the
|
||||
// contiguous source with copy.comp and transpose the other source into the
|
||||
// matching dst sub-region with the tiled copy_transpose shader (shared-memory
|
||||
// transpose, coalesced read+write). Mirrors the precedent in
|
||||
// ggml_vk_cpy_to_contiguous: direct dispatch with custom push constants.
|
||||
static void ggml_vk_concat_transpose_fastpath(ggml_backend_vk_context * ctx, vk_context& subctx,
|
||||
const ggml_tensor * ctg, const ggml_tensor * trp,
|
||||
ggml_tensor * dst, uint32_t off_ctg, uint32_t off_trp) {
|
||||
const uint32_t ts = ggml_type_size(dst->type);
|
||||
|
||||
vk_pipeline pipeline_cpy = (ts == 4) ? ctx->device->pipeline_cpy_32
|
||||
: ctx->device->pipeline_cpy_16;
|
||||
vk_pipeline pipeline_trp = (ts == 4) ? ctx->device->pipeline_cpy_transpose_32
|
||||
: ctx->device->pipeline_cpy_transpose_16;
|
||||
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline_cpy, 1);
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline_trp, 1);
|
||||
|
||||
vk_subbuffer ctg_buf = ggml_vk_tensor_subbuffer(ctx, ctg, true);
|
||||
vk_subbuffer trp_buf = ggml_vk_tensor_subbuffer(ctx, trp, true);
|
||||
vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true);
|
||||
|
||||
const uint32_t a_misalign_ctg = get_misalign_bytes(ctx, ctg) / ts;
|
||||
const uint32_t a_misalign_trp = get_misalign_bytes(ctx, trp) / ts;
|
||||
const uint32_t d_misalign = get_misalign_bytes(ctx, dst) / ts;
|
||||
|
||||
// Dispatch A: contiguous copy of `ctg` into dst[off_ctg : off_ctg + ctg->ne[0], :]
|
||||
if (ctg->ne[0] > 0) {
|
||||
const uint32_t ne_ctg = (uint32_t) ggml_nelements(ctg);
|
||||
vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(ctg, dst, ne_ctg);
|
||||
pc.ne10 = (uint32_t) ctg->ne[0]; // only ctg's columns in dst
|
||||
pc.misalign_offsets = (a_misalign_ctg << 16) | (d_misalign + off_ctg);
|
||||
init_pushconst_fastdiv(pc);
|
||||
std::array<uint32_t, 3> el = ne_ctg > 262144 ? std::array<uint32_t,3>{512, 512, CEIL_DIV(ne_ctg, 262144)}
|
||||
: ne_ctg > 512 ? std::array<uint32_t,3>{512, CEIL_DIV(ne_ctg, 512), 1}
|
||||
: std::array<uint32_t,3>{ne_ctg, 1, 1};
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline_cpy, { ctg_buf, dst_buf }, pc, el);
|
||||
}
|
||||
|
||||
// Dispatch B: tiled transpose of `trp` into dst[off_trp : off_trp + trp->ne[0], :]
|
||||
if (trp->ne[0] > 0) {
|
||||
vk_op_unary_push_constants pc = vk_op_unary_push_constants_init(trp, dst, ggml_nelements(trp));
|
||||
pc.ne10 = (uint32_t) trp->ne[0]; // dst bound = trp columns (NOT dst->ne[0])
|
||||
pc.misalign_offsets = (a_misalign_trp << 16) | (d_misalign + off_trp);
|
||||
init_pushconst_fastdiv(pc);
|
||||
std::array<uint32_t, 3> el = {
|
||||
(uint32_t) CEIL_DIV(trp->ne[0], 32),
|
||||
(uint32_t) CEIL_DIV(trp->ne[1], 32),
|
||||
(uint32_t) (trp->ne[2] * trp->ne[3]),
|
||||
};
|
||||
el[0] = std::min(el[0], (uint32_t) ctx->device->properties.limits.maxComputeWorkGroupCount[0]);
|
||||
el[1] = std::min(el[1], (uint32_t) ctx->device->properties.limits.maxComputeWorkGroupCount[1]);
|
||||
el[2] = std::min(el[2], (uint32_t) ctx->device->properties.limits.maxComputeWorkGroupCount[2]);
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline_trp, { trp_buf, dst_buf }, pc, el);
|
||||
}
|
||||
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
}
|
||||
|
||||
void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
int * op_params = (int *)dst->op_params;
|
||||
|
||||
// Fast path: concat along dim 0 with one source "transposed" (nb[1]==type_size,
|
||||
// dim1 innermost) and the other source + dst contiguous along dim 0. The generic
|
||||
// shader reads the transposed source uncoalesced (~5.6ms on RX 580 vs ~130us for
|
||||
// a coalesced copy); here we instead use a tiled shared-memory transpose.
|
||||
auto src_transposed_2d = [&](const ggml_tensor * s) {
|
||||
const uint32_t ts = ggml_type_size(s->type);
|
||||
return s->nb[1] == ts // dim1 innermost
|
||||
&& s->nb[0] == (size_t) s->ne[1] * ts // consistent 2D transpose
|
||||
&& s->ne[0] >= 32 // at least one full transpose tile (TILE_DIM); at 1 token the
|
||||
// tiled path is pure overhead, the generic copy is faster
|
||||
&& s->ne[2] == 1 && s->ne[3] == 1;
|
||||
};
|
||||
const uint32_t dst_ts = ggml_type_size(dst->type);
|
||||
const bool dim0_ok = (op_params[0] == 0) && (dst->nb[0] == dst_ts);
|
||||
// the dispatches below are raw f32/f16 element copies, so quantized and integer
|
||||
// types stay on the generic path
|
||||
const bool types_ok = (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)
|
||||
&& (src0->type == src1->type && src0->type == dst->type);
|
||||
const bool shapes_ok = (src0->ne[1] == src1->ne[1] && src1->ne[1] == dst->ne[1])
|
||||
&& (src0->ne[2] == src1->ne[2] && src0->ne[2] == dst->ne[2])
|
||||
&& (src0->ne[3] == src1->ne[3] && src0->ne[3] == dst->ne[3])
|
||||
&& (src0->ne[0] + src1->ne[0] == dst->ne[0]);
|
||||
// doffset is 16-bit for unary shaders; guard against overflow
|
||||
const bool off_fits = ((get_misalign_bytes(ctx, dst)/dst_ts + dst->ne[0])) < 0xFFFFu;
|
||||
|
||||
const bool s1_trans = dim0_ok && types_ok && shapes_ok && off_fits
|
||||
&& src_transposed_2d(src1) && ggml_is_contiguous(src0);
|
||||
const bool s0_trans = dim0_ok && types_ok && shapes_ok && off_fits
|
||||
&& src_transposed_2d(src0) && ggml_is_contiguous(src1);
|
||||
|
||||
// validation/A-B knob: GGML_VK_CONCAT_FASTPATH=0 forces the generic path, =log reports every
|
||||
// fast-path dispatch on stderr
|
||||
static const char * fastpath_env = getenv("GGML_VK_CONCAT_FASTPATH");
|
||||
static const bool fastpath_off = fastpath_env != nullptr && std::string(fastpath_env) == "0";
|
||||
static const bool fastpath_log = fastpath_env != nullptr && std::string(fastpath_env) == "log";
|
||||
|
||||
if (!fastpath_off && (s1_trans || s0_trans)) {
|
||||
const ggml_tensor * ctg = s1_trans ? src0 : src1; // contiguous source
|
||||
const ggml_tensor * trp = s1_trans ? src1 : src0; // transposed source
|
||||
if (fastpath_log) {
|
||||
std::cerr << "ggml_vk_concat: transpose fast path, transposed ne0=" << trp->ne[0] << std::endl;
|
||||
}
|
||||
// dst offsets (in elements) where each source region begins
|
||||
const uint32_t off_ctg = s1_trans ? 0u : (uint32_t) src1->ne[0];
|
||||
const uint32_t off_trp = s1_trans ? (uint32_t) src0->ne[0] : 0u;
|
||||
ggml_vk_concat_transpose_fastpath(ctx, subctx, ctg, trp, dst, off_ctg, off_trp);
|
||||
return;
|
||||
}
|
||||
|
||||
const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);
|
||||
const uint32_t units_per_block = ggml_type_size(dst->type) / unit_size;
|
||||
const uint32_t block_size = ggml_blck_size(dst->type);
|
||||
@@ -14402,6 +14558,35 @@ static const char * ggml_backend_vk_name(ggml_backend_t backend) {
|
||||
return ctx->name.c_str();
|
||||
}
|
||||
|
||||
// Free the compute-scratch device buffers (prealloc_* and the transfer staging buffer) without
|
||||
// tearing down the backend (pipelines, command pools, fences and device stay alive). These buffers
|
||||
// are reallocated lazily by ggml_vk_preallocate_buffers() on the next compute, so this just shrinks
|
||||
// an idle model's device footprint. Mirrors the buffer-freeing subset of ggml_vk_cleanup().
|
||||
static void ggml_backend_vk_free_scratch(ggml_backend_t backend) {
|
||||
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
|
||||
VK_LOG_DEBUG("ggml_backend_vk_free_scratch(" << ctx->name << ")");
|
||||
|
||||
// discard any unsubmitted command buffer and wait for in-flight work before freeing
|
||||
ctx->compute_ctx.reset();
|
||||
ggml_vk_synchronize(ctx);
|
||||
|
||||
ggml_vk_destroy_buffer(ctx->prealloc_x);
|
||||
ggml_vk_destroy_buffer(ctx->prealloc_y);
|
||||
ggml_vk_destroy_buffer(ctx->prealloc_split_k);
|
||||
ggml_vk_destroy_buffer(ctx->prealloc_add_rms_partials);
|
||||
ggml_vk_destroy_buffer(ctx->sync_staging);
|
||||
|
||||
ctx->prealloc_y_last_pipeline_used = nullptr;
|
||||
ctx->prealloc_y_last_tensor_used = nullptr;
|
||||
ctx->prealloc_y_last_k_padded = false;
|
||||
|
||||
ctx->prealloc_size_x = 0;
|
||||
ctx->prealloc_size_y = 0;
|
||||
ctx->prealloc_size_split_k = 0;
|
||||
ctx->prealloc_size_add_rms_partials = 0;
|
||||
ctx->prealloc_size_add_rms_partials_offset = 0;
|
||||
}
|
||||
|
||||
void ggml_backend_vk_free(ggml_backend_t backend) {
|
||||
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
|
||||
VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
|
||||
@@ -16421,6 +16606,7 @@ static ggml_backend_i ggml_backend_vk_interface = {
|
||||
/* .event_record = */ ggml_backend_vk_event_record,
|
||||
/* .event_wait = */ ggml_backend_vk_event_wait,
|
||||
/* .graph_optimize = */ ggml_vk_graph_optimize,
|
||||
/* .free_scratch = */ ggml_backend_vk_free_scratch,
|
||||
};
|
||||
|
||||
static ggml_guid_t ggml_backend_vk_guid() {
|
||||
@@ -16772,8 +16958,16 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
}
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
{
|
||||
// K/V row lists (ggml_flash_attn_ext_rows) are not supported
|
||||
if (op->src[5] != nullptr) {
|
||||
// Only the indexed-bias scalar kernel reads row lists.
|
||||
const bool indexed_bias = ggml_flash_attn_ext_has_indexed_bias(op);
|
||||
if (op->src[5] && (!indexed_bias || device->properties.limits.maxPerStageDescriptorStorageBuffers < 9 ||
|
||||
device->properties.limits.maxDescriptorSetStorageBuffers < 9)) {
|
||||
return false;
|
||||
}
|
||||
if (indexed_bias && (op->src[6]->ne[0] > 0x7fffff || !device->subgroup_shuffle || !device->subgroup_vote ||
|
||||
ggml_nelements(op->src[6]) > UINT32_MAX || ggml_nelements(op->src[7]) > UINT32_MAX ||
|
||||
op->src[1]->type != GGML_TYPE_F16 || op->src[2]->type != GGML_TYPE_F16 ||
|
||||
(op->src[1]->ne[0] != 64 && op->src[1]->ne[0] != 128) || op->src[1]->ne[0] != op->src[2]->ne[0])) {
|
||||
return false;
|
||||
}
|
||||
bool coopmat2 = device->coopmat2;
|
||||
@@ -17046,7 +17240,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
case GGML_OP_SLEEP:
|
||||
return device->shader_clock;
|
||||
case GGML_OP_SCALE:
|
||||
return ggml_is_contiguous(op->src[0]) && ggml_vk_is_float_elementwise(op);
|
||||
return ggml_vk_is_float_elementwise(op);
|
||||
case GGML_OP_PAD:
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
case GGML_OP_ROLL:
|
||||
|
||||
@@ -212,9 +212,23 @@ void main() {
|
||||
uint32_t r = (idx + tid) / Bc;
|
||||
if (idx + tid < Bc * Br) {
|
||||
uint32_t kcol;
|
||||
#ifdef KV_ROWS
|
||||
kcol = j * Bc + c;
|
||||
bool kv_active = kcol < KV;
|
||||
#else
|
||||
bool kv_active = fa_kv_index(j * Bc + c, kcol);
|
||||
#endif
|
||||
if (kv_active && (!nem1_bounds_check || i * Br + r < p.nem1)) {
|
||||
FLOAT_TYPE m = FLOAT_TYPE(data_m[m_offset + (i * Br + r) * m_stride + kcol]);
|
||||
const uint n_bins = (Flags & 0x7fffffffu) >> 8;
|
||||
if (n_bins != 0) {
|
||||
uint query = iq3 * p.N + i * Br + r;
|
||||
uint bin = uint(data_sparse[query * KV + kcol]);
|
||||
if (bin < n_bins) {
|
||||
float bias = uintBitsToFloat(data_mask_opt[(query * p.neq2 + iq2) * n_bins + bin]);
|
||||
m = FLOAT_TYPE(float16_t(float(m) + bias));
|
||||
}
|
||||
}
|
||||
masksh[c * masksh_stride + r] = m;
|
||||
max_mask = max(max_mask, float(m));
|
||||
} else {
|
||||
|
||||
@@ -88,6 +88,10 @@ layout (binding = 5) writeonly buffer OV4 {D_TYPEV4 data_ov4[];};
|
||||
|
||||
layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];};
|
||||
|
||||
#ifdef KV_ROWS
|
||||
layout (binding = 8) readonly buffer KR {int32_t data_kv_rows[];};
|
||||
#endif
|
||||
|
||||
layout (binding = 7) readonly buffer SP {int32_t data_sparse[];};
|
||||
|
||||
// Use -FLT_MAX/2 rather than -inf to reduce the possibility of NaNs, e.g. when computing Mold-M.
|
||||
@@ -237,6 +241,15 @@ void init_indices()
|
||||
|
||||
// Resolve a linear KV slot to a real column; false for inactive (sparse padding/-1, or dense OOB).
|
||||
bool fa_kv_index(uint lin, out uint kv_col) {
|
||||
#ifdef KV_ROWS
|
||||
if (lin >= KV) {
|
||||
kv_col = 0;
|
||||
return false;
|
||||
}
|
||||
int row = data_kv_rows[iq3 * KV + lin];
|
||||
kv_col = uint(max(row, 0));
|
||||
return row >= 0;
|
||||
#endif
|
||||
if (USE_SPARSE) {
|
||||
if (lin >= p.split_kv) {
|
||||
kv_col = 0;
|
||||
|
||||
@@ -18,7 +18,7 @@ void main() {
|
||||
continue;
|
||||
}
|
||||
|
||||
data_d[get_doffset() + idx] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2));
|
||||
data_d[get_doffset() + dst_idx(idx)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(idx)]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2));
|
||||
idx += num_threads;
|
||||
}
|
||||
}
|
||||
@@ -749,6 +749,11 @@ void process_shaders() {
|
||||
#endif
|
||||
}
|
||||
|
||||
if (!f16acc) {
|
||||
string_to_spv("flash_attn_f32_f16_rows", "flash_attn.comp",
|
||||
merge_maps(fa_base_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"D_TYPEV4", "vec4"}, {"KV_ROWS", "1"}}), fp16, false, false, false);
|
||||
}
|
||||
|
||||
string_to_spv("flash_attn_f32_f16", "flash_attn.comp",
|
||||
merge_maps(fa_base_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"D_TYPEV4", "vec4"}}), fp16, false, false, f16acc);
|
||||
|
||||
|
||||
@@ -5080,8 +5080,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
|
||||
{
|
||||
// conservative support checks for whether the more resource-intensive shader paths
|
||||
// can be used, to avoid cases where flash_attn is assigned to the CPU later on
|
||||
// K/V row lists (ggml_flash_attn_ext_rows) are not supported
|
||||
supports_op = op->src[5] == nullptr && src0->type == GGML_TYPE_F32 &&
|
||||
// K/V row lists and indexed bias are not supported
|
||||
supports_op = op->src[5] == nullptr && !ggml_flash_attn_ext_has_indexed_bias(op) &&
|
||||
src0->type == GGML_TYPE_F32 &&
|
||||
(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 ||
|
||||
src1->type == GGML_TYPE_Q4_0 || src1->type == GGML_TYPE_Q8_0) &&
|
||||
(src2->type == GGML_TYPE_F32 || src2->type == GGML_TYPE_F16 ||
|
||||
|
||||
@@ -21,7 +21,7 @@ bool ggml_zdnn_supports_flash_attn_ext(const ggml_backend_zdnn_device_context *
|
||||
memcpy(&logit_softcap, (const float *) op->op_params + 2, sizeof(float));
|
||||
|
||||
// sinks, alibi, softcap, sparse k/v sets and k/v row lists are not implemented
|
||||
if (op->src[4] != nullptr || op->src[5] != nullptr || max_bias != 0.0f || logit_softcap != 0.0f || ggml_get_op_params_i32(op, 4) != 0) {
|
||||
if (op->src[4] != nullptr || op->src[5] != nullptr || ggml_flash_attn_ext_has_indexed_bias(op) || max_bias != 0.0f || logit_softcap != 0.0f || ggml_get_op_params_i32(op, 4) != 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -5526,6 +5526,34 @@ void ggml_flash_attn_ext_add_sinks(
|
||||
a->src[4] = sinks;
|
||||
}
|
||||
|
||||
bool ggml_flash_attn_ext_has_indexed_bias(const struct ggml_tensor * a) {
|
||||
GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT);
|
||||
return ggml_get_op_params_i32(a, 5) != 0;
|
||||
}
|
||||
|
||||
void ggml_flash_attn_ext_add_indexed_bias(
|
||||
struct ggml_tensor * a,
|
||||
struct ggml_tensor * table,
|
||||
struct ggml_tensor * indices) {
|
||||
GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT);
|
||||
GGML_ASSERT((table == NULL) == (indices == NULL));
|
||||
a->src[6] = table;
|
||||
a->src[7] = indices;
|
||||
ggml_set_op_params_i32(a, 5, table != NULL);
|
||||
if (!table) {
|
||||
return;
|
||||
}
|
||||
|
||||
const struct ggml_tensor * q = a->src[0];
|
||||
const struct ggml_tensor * mask = a->src[3];
|
||||
GGML_ASSERT(mask && mask->ne[0] == (a->src[5] ? a->src[5]->ne[0] : a->src[1]->ne[1]) && mask->ne[1] == q->ne[1]);
|
||||
GGML_ASSERT(mask->ne[2] == 1 && mask->ne[3] == q->ne[3]);
|
||||
GGML_ASSERT(table->type == GGML_TYPE_F32 && ggml_is_contiguous(table));
|
||||
GGML_ASSERT(table->ne[1] == q->ne[2] && table->ne[0] > 0 && table->ne[2] == q->ne[1]*q->ne[3] && table->ne[3] == 1);
|
||||
GGML_ASSERT(indices->type == GGML_TYPE_I32 && ggml_is_contiguous(indices) && ggml_is_matrix(indices));
|
||||
GGML_ASSERT(indices->ne[0] == mask->ne[0] && indices->ne[1] == table->ne[2]);
|
||||
}
|
||||
|
||||
// ggml_flash_attn_back
|
||||
|
||||
struct ggml_tensor * ggml_flash_attn_back(
|
||||
|
||||
@@ -582,6 +582,23 @@ extern "C" {
|
||||
|
||||
LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);
|
||||
LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx);
|
||||
|
||||
// On-demand device (VRAM) residency: free the model's GPU weight buffers (keeping a host
|
||||
// shadow so no reload is needed) to hand VRAM to another model, and rebuild them on demand.
|
||||
// Any subsequent llama_decode auto-restores. If evict_kv is true, the KV cache device buffers
|
||||
// are also evicted to a host shadow (for when the KV would not fit alongside the other model),
|
||||
// at the cost of a D2H/H2D copy of the live KV each cycle; otherwise the KV stays resident.
|
||||
// Intended for time-sharing a single GPU between several always-loaded models.
|
||||
LLAMA_API void llama_context_release_device(struct llama_context * ctx, bool evict_kv);
|
||||
LLAMA_API void llama_context_restore_device(struct llama_context * ctx);
|
||||
// Free all device buffers (weights, KV, compute) for good, without host copies, e.g. to hand
|
||||
// the VRAM to another process before exit. The context must not be used for compute afterwards.
|
||||
LLAMA_API void llama_context_discard_device(struct llama_context * ctx);
|
||||
// Before exit, after llama_context_release_device(ctx, true): point the evicted KV/recurrent state
|
||||
// at its host copy (no device use, no extra copy) so llama_state_seq_get_data can read it. The
|
||||
// context must not be used for compute afterwards.
|
||||
LLAMA_API bool llama_context_restore_memory_host(struct llama_context * ctx);
|
||||
LLAMA_API bool llama_context_weights_resident(const struct llama_context * ctx);
|
||||
LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
|
||||
|
||||
LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,247 @@
|
||||
# rx580-bench
|
||||
|
||||
Prompt-processing benchmark harness for the RX 580 (Polaris) serving pod.
|
||||
|
||||
It exists so that no future session has to rediscover how to measure this
|
||||
machine. It encodes the production config, fixed corpus slices, the repeat
|
||||
and median discipline, and the noise floor.
|
||||
|
||||
## What it measures
|
||||
|
||||
Two things, deliberately:
|
||||
|
||||
1. **Corpus prompt processing** (the number we actually care about). Real
|
||||
Polish prose from `pan-tadeusz.txt` pushed through `llama-server`'s
|
||||
`/completion` endpoint with `n_predict: 1`, `cache_prompt: false`. This
|
||||
mimics agentic tool-result parsing: a big blob of real text arriving cold.
|
||||
Reported as `timings.prompt_per_second` at prompt lengths of about 4k, 16k
|
||||
and 32k tokens.
|
||||
2. **`llama-bench` sweep** (the controlled cross-check). Synthetic random
|
||||
tokens: prefill `pp2048`, `pp8192` at depth 0, and decode `tg256@d4096`
|
||||
(256 generated tokens on top of a 4096-token context, `-p 0 -n 256
|
||||
-d 4096`). The prefill cells are the continuity metric with prior work on
|
||||
this box. Decode used to be `tg32` at the default depth; that is not a valid
|
||||
metric here (see "Reading the numbers"), and old `tg32` rows in
|
||||
`results.txt` are history only.
|
||||
|
||||
They do not measure the same thing and they do not agree. `llama-bench`
|
||||
pp2048 is a cold depth-0 batch; the corpus numbers include the cost of a
|
||||
growing KV cache, so they fall off with prompt length. Both are useful.
|
||||
|
||||
## How to run it
|
||||
|
||||
One command from the workstation:
|
||||
|
||||
```
|
||||
./scripts/rx580-bench/run.sh --build /root/arms/new-clean/build --label new-clean
|
||||
```
|
||||
|
||||
Interleaved A/B, which is the only valid way to compare two builds:
|
||||
|
||||
```
|
||||
./scripts/rx580-bench/run.sh \
|
||||
--build /root/arms/new-clean/build --label new-clean \
|
||||
--build-b /root/arms/new-fork/build --label-b new-fork \
|
||||
--rounds 3
|
||||
```
|
||||
|
||||
Summarize whatever has accumulated:
|
||||
|
||||
```
|
||||
./scripts/rx580-bench/run.sh --summarize
|
||||
```
|
||||
|
||||
`run.sh` copies the harness onto the pod and execs it there. The wrapper is
|
||||
thin on purpose: all the methodology lives in the pod-side scripts, so it can
|
||||
also be driven directly on the pod if kubectl is inconvenient.
|
||||
|
||||
The run is detached on the pod and then tailed, so Ctrl-C on the wrapper does
|
||||
not orphan a half-finished run. `bench.sh` restarts `llama-swap` from an EXIT
|
||||
trap on success, failure and interrupt alike.
|
||||
|
||||
### Pod access
|
||||
|
||||
```
|
||||
export KUBECONFIG=/home/user/Projects/klaster/talos/generated/kubeconfig
|
||||
kubectl -n llama exec -i deploy/supervisord -- sh -c '<command>'
|
||||
```
|
||||
|
||||
The pod has `curl` and `python3` but **no `jq`**; the harness uses python3 for
|
||||
all JSON.
|
||||
|
||||
## Files
|
||||
|
||||
| file | side | what |
|
||||
|---|---|---|
|
||||
| `run.sh` | workstation | one-command wrapper; installs and execs the rest |
|
||||
| `bench.sh` | pod | orchestrator: stops llama-swap, waits for idle GPU, runs both harnesses, restarts llama-swap via trap |
|
||||
| `ppbench.py` | pod | the corpus prompt-processing harness |
|
||||
| `lbsweep.sh` | pod | the llama-bench sweep |
|
||||
| `summarize.py` | pod | turns `results.txt` into a median/min-max table with delta significance |
|
||||
|
||||
Results append to `/root/bench/results.txt` as parseable `RESULT` /
|
||||
`SUMMARY` / `LBRESULT` lines. Server logs land in `/root/bench/server-*.log`.
|
||||
|
||||
## The config it assumes
|
||||
|
||||
Production serving config, matching `/root/config.yaml`:
|
||||
|
||||
```
|
||||
-t 6 -ngl 99 --n-cpu-moe 40 -b 2048 -ub 2048 -fa 1 -lm none
|
||||
--ctx-size 40960 --no-warmup
|
||||
```
|
||||
|
||||
Model:
|
||||
|
||||
```
|
||||
/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
||||
```
|
||||
|
||||
Qwen3.6-35B-A3B, 20.8 GiB, 40 layers, 256 experts of which 8 are used.
|
||||
Machine: AMD RX 580 8 GB (Polaris/GCN, RADV, no fp16 accel), PCIe 3.0 x16,
|
||||
12 CPU cores, 62 GB RAM.
|
||||
|
||||
The `llama-bench` equivalent of that config is:
|
||||
|
||||
```
|
||||
-t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -p 2048,8192 -n 0 -r 3
|
||||
-t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none -p 0 -n 256 -d 4096 -r 3
|
||||
```
|
||||
|
||||
`lbsweep.sh` runs these as two invocations (prefill, then decode). Override
|
||||
with `LB_P` (prefill sizes), `LB_N` / `LB_D` (decode tokens / depth; `LB_N=0`
|
||||
skips decode) and `LB_R` (repetitions).
|
||||
|
||||
## The corpus slices are constants, not recalibrated
|
||||
|
||||
`pan-tadeusz.txt` is 447334 chars / 482907 bytes of Polish text. Against this
|
||||
model's tokenizer it runs **2.6702 chars/token**, and `/completion` reports
|
||||
`prompt_n` identical to `/tokenize` (BOS offset 0).
|
||||
|
||||
The slices are `corpus[0:NCHARS]` and are baked into `ppbench.py`:
|
||||
|
||||
| target tokens | chars | utf-8 bytes |
|
||||
|---|---|---|
|
||||
| 4096 | 10776 | 11608 |
|
||||
| 16384 | 43858 | 47175 |
|
||||
| 32768 | 87165 | 93843 |
|
||||
|
||||
These hit the target `prompt_n` exactly. Every build is therefore measured on
|
||||
byte-identical input. `ppbench.py` warns loudly if the reported `prompt_n`
|
||||
ever stops matching the target, which is the signal that the table is stale.
|
||||
Only recalibrate (`ppbench.py <build> <label> --calibrate`) if the model file
|
||||
or the corpus changes.
|
||||
|
||||
## Reading the numbers
|
||||
|
||||
* `prompt_n` is the token count actually prefilled. It must equal the target.
|
||||
* `prompt_ms` is wall-clock prefill time.
|
||||
* `tps` / `prompt_per_second` is `prompt_n / prompt_ms * 1000`. Higher better.
|
||||
* `pp2048` / `pp8192` are llama-bench prefill throughput at depth 0. A depth
|
||||
suffix (`pp2048@d16384`) means the prompt is processed on top of a context
|
||||
of that many tokens.
|
||||
* `tg256@d4096` is the decode metric: 256 generated tokens at depth 4096
|
||||
(`-p 0 -n 256 -d 4096`). Decode is CPU-bound with `--n-cpu-moe 40` and
|
||||
swings with node CPU contention (about 60 co-tenant pods), so compare
|
||||
medians across interleaved rounds and expect occasional catastrophic
|
||||
samples.
|
||||
* `tg32` (old rows, no depth suffix) is token generation at the DEFAULT depth
|
||||
(n_ctx 32). **It is NOT a valid decode metric on this box.** At that size
|
||||
decode is launch/barrier bound, not compute bound, and measures a regime
|
||||
serving never reaches: it read -36% between two upstream commits that are at
|
||||
parity at tg256 (the cause was the view-alias fix in
|
||||
ggml_vk_graph_optimize). tg is also NOT monotonic in n_ctx here - on one
|
||||
binary n_ctx 256 measured slower than both 32 and 544 - so only ever compare
|
||||
same-depth cells. `summarize.py` flags any depth-less tg row.
|
||||
|
||||
## Noise floor: deltas under about 5 percent are unproven
|
||||
|
||||
This is the single most important thing in this directory.
|
||||
|
||||
`llama-bench`'s within-run error bars understate **cross-invocation** variance
|
||||
by roughly **14x**. The same config measured `276.7 +/- 0.9` in one invocation
|
||||
and `296.4 +/- 0.6` in another, 7 percent apart, with error bars that claimed
|
||||
0.3 percent precision.
|
||||
|
||||
Therefore:
|
||||
|
||||
* Any comparison you want to draw a conclusion from must live **inside a
|
||||
single `llama-bench` invocation**, using comma-separated sweeps.
|
||||
* Two different *builds* cannot share an invocation, so run them
|
||||
**interleaved A/B/A/B/A/B across at least 3 rounds** and compare medians.
|
||||
That is what `--build-b` does.
|
||||
* State explicitly that any cross-build delta under about 5 percent is
|
||||
unproven. `summarize.py` labels them `unproven` for you.
|
||||
|
||||
## Traps
|
||||
|
||||
* **`-tb` is not a `llama-bench` flag.** It is a `llama-server` flag. Passing
|
||||
it makes `llama-bench` print usage and exit silently, mid-sweep, which looks
|
||||
exactly like a run that produced nothing. Always check the row count.
|
||||
* **A ubatch larger than the prompt never fills.** With `-ub 2048`, prompt
|
||||
lengths must be multiples of 2048 or the sweep measures nothing meaningful.
|
||||
* **Nothing else may touch the GPU.** `bench.sh` stops `llama-swap` and
|
||||
refuses to start if any `llama-*` process is still alive.
|
||||
* **Never leave the pod not serving.** `bench.sh` restarts `llama-swap` from
|
||||
an EXIT trap. If you bypass the script, restart it by hand:
|
||||
`supervisorctl -c /root/supervisord.conf start llama-swap`.
|
||||
* **Do not run `llama-cli` detached with closed stdin** on this box; it loops
|
||||
forever. Use the server harness or `llama-bench -d` for depth timing.
|
||||
* **`--mmap` / `--no-mmap` / `--mlock` and llama-bench's `-mmp` are gone**
|
||||
upstream (#28334). Use `-lm none` (`--load-mode none`) on both
|
||||
`llama-server` and `llama-bench`. The old flags now fail: the server refuses
|
||||
to start, `llama-bench` silently produces no rows. `/root/config.yaml` must
|
||||
be updated to `-lm none` as well before deploying a build past that commit.
|
||||
|
||||
## Optional: the MoE expert cache
|
||||
|
||||
Upstream #29887 added `--moe-cache-mib N` (default 0, off): a device cache of
|
||||
the most recently used experts of the CPU-resident (`--n-cpu-moe`) layers. It
|
||||
mostly helps decode and small batches; large prefill batches only read from it.
|
||||
It is a server option only - `llama-bench` has no flag for it - so measure it
|
||||
with the corpus harness, e.g.
|
||||
`LLAMA_ARG_MOE_CACHE_MIB=1024 bench.sh --corpus-only --build DIR --label NAME`
|
||||
(the environment reaches the spawned server). The variable applies to every
|
||||
arm of the run, so an on/off comparison spans two runs and the noise floor
|
||||
above applies in full. The cache takes VRAM from the same 8 GB as the KV
|
||||
cache and compute buffers.
|
||||
|
||||
## Known-good reference values
|
||||
|
||||
If a fresh run disagrees with these by much more than the noise floor,
|
||||
something is wrong (something else on the GPU, a thermal problem, a bad
|
||||
build) before you believe you found a speedup.
|
||||
|
||||
Model Qwen3.6-35B-A3B-UD-Q4_K_XL, production config.
|
||||
|
||||
llama-bench, current production fork (20261024 rebase, `6c480f7e7`) and the
|
||||
previous production fork (`5751c977b`), measured 2026-10-24 by
|
||||
`fork-validate.sh --perf` (2 interleaved arms each, `-r 2` prefill, `-r 3`
|
||||
decode):
|
||||
|
||||
| metric | about |
|
||||
|---|---|
|
||||
| pp2048 (depth 0) | 274-286 t/s (274 current, 286 previous; the -4% is inside the noise floor) |
|
||||
| pp2048@d16384 | 201-206 t/s |
|
||||
| pp2048@d32768 | 158-160 t/s |
|
||||
| tg256@d4096 | 17.8 t/s |
|
||||
|
||||
Corpus harness, measured 2026-09-08/09 on the then-current upstream
|
||||
(`f3f1a8f27`) and the fork deployed then (not re-measured since):
|
||||
|
||||
| metric | clean upstream | fork |
|
||||
|---|---|---|
|
||||
| corpus pp @ 4k | about 240 t/s | about 255 t/s |
|
||||
| corpus pp @ 16k | about 206 t/s | about 225 t/s |
|
||||
| corpus pp @ 32k | about 171 t/s | about 195 t/s |
|
||||
| llama-bench pp8192 | about 222 t/s | about 246 t/s |
|
||||
|
||||
Invalid-metric history: the 2026-09 runs recorded `tg32` (7-15 t/s, swinging
|
||||
by run) and older notes mention "tg32 about 19 t/s". Neither is a decode
|
||||
reference; see the tg32 warning above.
|
||||
|
||||
Model load with `-lm none` takes about 26 s warm, longer cold.
|
||||
A full 3-round A/B run of both harnesses takes roughly 100 minutes;
|
||||
add about 16 minutes per extra arm per round. The tg256@d4096 decode cell
|
||||
costs about 1.2 minutes per arm per round (model load, the 4096-token depth
|
||||
prefill and 3 x 256 tokens), measured 2026-10-24.
|
||||
Executable
+167
@@ -0,0 +1,167 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# RX 580 prompt-processing benchmark orchestrator. Runs ON THE POD.
|
||||
#
|
||||
# Stops llama-swap, waits for the GPU to go idle, runs the real-corpus
|
||||
# prompt-processing harness and the llama-bench sweep for every arm, appends
|
||||
# parseable results, and ALWAYS restarts llama-swap on exit (success, error
|
||||
# or Ctrl-C).
|
||||
#
|
||||
# Usage:
|
||||
# bench.sh --build DIR --label NAME
|
||||
# [--build-b DIR --label-b NAME] second arm, interleaved
|
||||
# [--arm DIR:LABEL ...] extra arms, repeatable (N-way)
|
||||
# [--rounds N] default 3 multi-arm, 1 otherwise
|
||||
# [--reps N] corpus repeats per size, default 3
|
||||
# [--sizes 4096,16384,32768]
|
||||
# [--corpus-only | --lb-only]
|
||||
# [--results PATH] default /root/bench/results.txt
|
||||
# [--keep-swap-down] do not restart llama-swap at exit
|
||||
#
|
||||
set -e
|
||||
|
||||
BENCH_DIR=/root/bench
|
||||
SUPCONF=/root/supervisord.conf
|
||||
RESULTS="$BENCH_DIR/results.txt"
|
||||
MODEL=/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
|
||||
|
||||
ARMS="" # space separated DIR|LABEL entries
|
||||
BUILD_A=""; LABEL_A=""
|
||||
BUILD_B=""; LABEL_B=""
|
||||
ROUNDS=""
|
||||
REPS=3
|
||||
SIZES=4096,16384,32768
|
||||
DO_CORPUS=1
|
||||
DO_LB=1
|
||||
KEEP_SWAP_DOWN=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--build) BUILD_A="$2"; shift 2 ;;
|
||||
--label) LABEL_A="$2"; shift 2 ;;
|
||||
--build-b) BUILD_B="$2"; shift 2 ;;
|
||||
--label-b) LABEL_B="$2"; shift 2 ;;
|
||||
--arm) ARMS="$ARMS ${2%%:*}|${2#*:}"; shift 2 ;;
|
||||
--rounds) ROUNDS="$2"; shift 2 ;;
|
||||
--reps) REPS="$2"; shift 2 ;;
|
||||
--sizes) SIZES="$2"; shift 2 ;;
|
||||
--results) RESULTS="$2"; shift 2 ;;
|
||||
--corpus-only) DO_LB=0; shift ;;
|
||||
--lb-only) DO_CORPUS=0; shift ;;
|
||||
--keep-swap-down) KEEP_SWAP_DOWN=1; shift ;;
|
||||
-h|--help) sed -n '2,20p' "$0"; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -n "$BUILD_A" ] && [ -z "$LABEL_A" ]; then
|
||||
echo "error: --build needs --label" >&2; exit 2
|
||||
fi
|
||||
if [ -n "$BUILD_B" ] && [ -z "$LABEL_B" ]; then
|
||||
echo "error: --build-b needs --label-b" >&2; exit 2
|
||||
fi
|
||||
# --build/--label and --build-b/--label-b are sugar for the first two arms.
|
||||
if [ -n "$BUILD_A" ]; then ARMS="$BUILD_A|$LABEL_A $ARMS"; fi
|
||||
if [ -n "$BUILD_B" ]; then ARMS="$ARMS $BUILD_B|$LABEL_B"; fi
|
||||
ARMS=$(echo $ARMS)
|
||||
if [ -z "$ARMS" ]; then
|
||||
echo "error: need at least one arm (--build/--label or --arm DIR:LABEL)" >&2
|
||||
exit 2
|
||||
fi
|
||||
NARMS=$(echo "$ARMS" | wc -w)
|
||||
if [ -z "$ROUNDS" ]; then
|
||||
if [ "$NARMS" -gt 1 ]; then ROUNDS=3; else ROUNDS=1; fi
|
||||
fi
|
||||
for e in $ARMS; do
|
||||
d=${e%%|*}
|
||||
if [ ! -x "$d/bin/llama-server" ] || [ ! -x "$d/bin/llama-bench" ]; then
|
||||
echo "error: $d lacks bin/llama-server or bin/llama-bench" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
cat <<'WARN'
|
||||
-------------------------------------------------------------------------
|
||||
RX 580 benchmark. Read this before trusting any number.
|
||||
|
||||
* Cross-invocation variance on this box is about 7 percent. llama-bench
|
||||
within-run error bars understate it by roughly 14x. Any cross-build
|
||||
delta under about 5 percent is UNPROVEN noise.
|
||||
* Comparisons you care about must live inside ONE llama-bench invocation
|
||||
(comma-separated sweeps), or be interleaved across at least 3 rounds.
|
||||
That is what multiple arms plus --rounds does.
|
||||
* Nothing else may touch the GPU while this runs. llama-swap is stopped
|
||||
for the duration and restarted on exit, including on error or Ctrl-C.
|
||||
* Traps: -tb is NOT a llama-bench flag. A ubatch larger than -p never
|
||||
fills, so prompt lengths must be multiples of the 2048 ubatch. A bad
|
||||
flag makes llama-bench print usage and exit silently mid-sweep.
|
||||
-------------------------------------------------------------------------
|
||||
WARN
|
||||
|
||||
echo "[bench] $NARMS arm(s), $ROUNDS round(s): $ARMS"
|
||||
|
||||
restore_swap() {
|
||||
rc=$?
|
||||
if [ "$KEEP_SWAP_DOWN" -eq 0 ]; then
|
||||
echo ""
|
||||
echo "[bench] restarting llama-swap"
|
||||
supervisorctl -c "$SUPCONF" start llama-swap || true
|
||||
supervisorctl -c "$SUPCONF" status llama-swap || true
|
||||
else
|
||||
echo "[bench] --keep-swap-down: llama-swap left STOPPED"
|
||||
fi
|
||||
exit $rc
|
||||
}
|
||||
trap restore_swap EXIT INT TERM
|
||||
|
||||
echo "[bench] stopping llama-swap"
|
||||
supervisorctl -c "$SUPCONF" stop llama-swap || true
|
||||
|
||||
wait_gpu_idle() {
|
||||
i=0
|
||||
while [ $i -lt 60 ]; do
|
||||
if ! pgrep -f 'bin/llama-server|bin/llama-bench|bin/llama-cli' >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
i=$((i+1))
|
||||
done
|
||||
echo "error: something is still on the GPU:" >&2
|
||||
pgrep -af 'bin/llama-server|bin/llama-bench|bin/llama-cli' >&2
|
||||
exit 3
|
||||
}
|
||||
|
||||
echo "[bench] waiting for the GPU to go idle"
|
||||
wait_gpu_idle
|
||||
echo "[bench] GPU idle"
|
||||
|
||||
run_corpus() {
|
||||
[ "$DO_CORPUS" -eq 1 ] || return 0
|
||||
echo "===== corpus $2 ====="
|
||||
wait_gpu_idle
|
||||
python3 "$BENCH_DIR/ppbench.py" "$1" "$2" \
|
||||
--sizes "$SIZES" --reps "$REPS" --results "$RESULTS"
|
||||
}
|
||||
|
||||
run_lb() {
|
||||
[ "$DO_LB" -eq 1 ] || return 0
|
||||
echo "===== llama-bench $2 ====="
|
||||
wait_gpu_idle
|
||||
RESULTS="$RESULTS" MODEL="$MODEL" sh "$BENCH_DIR/lbsweep.sh" "$1" "$2"
|
||||
}
|
||||
|
||||
# Interleave: every arm is measured once per round, so slow drift in the
|
||||
# machine hits all arms roughly equally instead of biasing whichever ran first.
|
||||
r=1
|
||||
while [ "$r" -le "$ROUNDS" ]; do
|
||||
echo ""
|
||||
echo "########## ROUND $r / $ROUNDS ##########"
|
||||
for e in $ARMS; do run_corpus "${e%%|*}" "${e#*|}"; done
|
||||
for e in $ARMS; do run_lb "${e%%|*}" "${e#*|}"; done
|
||||
r=$((r+1))
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "ALL_RUNS_DONE"
|
||||
echo "[bench] raw results appended to $RESULTS"
|
||||
python3 "$BENCH_DIR/summarize.py" "$RESULTS" || true
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# llama-bench sweep, the controlled cross-check for the corpus harness.
|
||||
# Runs ON THE POD. Normally invoked by bench.sh.
|
||||
#
|
||||
# Usage: lbsweep.sh <build_dir> <label>
|
||||
# Env overrides: RESULTS, MODEL, LB_P, LB_N, LB_D, LB_R
|
||||
#
|
||||
# Two llama-bench invocations per call:
|
||||
# prefill -p $LB_P -n 0 at depth 0 (default pp2048,pp8192)
|
||||
# decode -p 0 -n $LB_N -d $LB_D (default tg256 @ d4096)
|
||||
# LB_N=0 skips the decode invocation.
|
||||
#
|
||||
# Everything that must be compared lives inside ONE llama-bench invocation
|
||||
# (-p takes a comma-separated list), because cross-invocation variance on this
|
||||
# box is about 7 percent while the within-run error bars are about 0.3 percent.
|
||||
# Prefill and decode cells are never compared with each other, so they may
|
||||
# live in separate invocations.
|
||||
#
|
||||
# Decode is measured at an explicit depth on purpose: tg32 at the default
|
||||
# depth (n_ctx 32) is launch/barrier bound on this card and is NOT a valid
|
||||
# decode metric (it once read -36% between two builds that were at parity at
|
||||
# tg256). Decode rows are labelled with their depth (tg256@d4096); old tg32
|
||||
# rows in results.txt are invalid-metric history.
|
||||
#
|
||||
# Flag traps, learned the hard way:
|
||||
# * -tb is NOT a llama-bench flag. It exists on llama-server only.
|
||||
# * -ncmoe is the llama-bench spelling of --n-cpu-moe.
|
||||
# * -lm none is the load mode the server runs with. -mmp was removed upstream
|
||||
# (#28334) and, like any bad flag, now makes the sweep silently empty.
|
||||
# * A bad flag makes llama-bench print usage and exit silently, which looks
|
||||
# exactly like a sweep that produced no rows. Always check row count.
|
||||
# * A ubatch larger than -p never fills, so keep -p a multiple of -ub.
|
||||
#
|
||||
set -e
|
||||
BUILD_DIR="$1"
|
||||
LABEL="$2"
|
||||
if [ -z "$BUILD_DIR" ] || [ -z "$LABEL" ]; then
|
||||
echo "usage: $0 <build_dir> <label>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
RESULTS="${RESULTS:-/root/bench/results.txt}"
|
||||
MODEL="${MODEL:-/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf}"
|
||||
LB_P="${LB_P:-2048,8192}"
|
||||
LB_N="${LB_N:-256}"
|
||||
LB_D="${LB_D:-4096}"
|
||||
LB_R="${LB_R:-3}"
|
||||
|
||||
STAMP=$(date -u +%Y-%m-%dT%H:%M:%S)
|
||||
export LD_LIBRARY_PATH="$BUILD_DIR/bin"
|
||||
export LABEL BUILD_DIR STAMP RESULTS
|
||||
|
||||
lb() {
|
||||
"$BUILD_DIR/bin/llama-bench" \
|
||||
-m "$MODEL" \
|
||||
-t 6 -ngl 99 -ncmoe 40 -b 2048 -ub 2048 -fa 1 -lm none \
|
||||
-r "$LB_R" -o json "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
record() { # $1 = what, stdin = llama-bench JSON
|
||||
WHAT="$1" python3 -c '
|
||||
import json, sys, os
|
||||
label = os.environ["LABEL"]
|
||||
build = os.environ["BUILD_DIR"]
|
||||
stamp = os.environ["STAMP"]
|
||||
results = os.environ["RESULTS"]
|
||||
what = os.environ["WHAT"]
|
||||
text = sys.stdin.read()
|
||||
if not text.strip():
|
||||
sys.stderr.write("ERROR: llama-bench produced no output for %s %s (bad flag?)\n" % (label, what))
|
||||
sys.exit(3)
|
||||
rows = json.loads(text)
|
||||
if not rows:
|
||||
sys.stderr.write("ERROR: llama-bench returned zero rows for %s %s\n" % (label, what))
|
||||
sys.exit(3)
|
||||
out = open(results, "a")
|
||||
for r in rows:
|
||||
test = "pp%d" % r["n_prompt"] if r["n_prompt"] else "tg%d" % r["n_gen"]
|
||||
if r.get("n_depth", 0):
|
||||
test += "@d%d" % r["n_depth"]
|
||||
line = ("LBRESULT label=%s build=%s test=%s tps=%.2f stddev=%.2f "
|
||||
"samples=%s time=%s"
|
||||
% (label, build, test, r["avg_ts"], r["stddev_ts"],
|
||||
r["samples_ts"], stamp))
|
||||
out.write(line + "\n")
|
||||
print(line)
|
||||
out.close()
|
||||
'
|
||||
}
|
||||
|
||||
lb -p "$LB_P" -n 0 | record prefill
|
||||
if [ "$LB_N" -gt 0 ]; then
|
||||
lb -p 0 -n "$LB_N" -d "$LB_D" | record decode
|
||||
fi
|
||||
Executable
+239
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Real-corpus prompt-processing benchmark for llama-server on the RX 580 pod.
|
||||
|
||||
Runs ON THE POD. Normally invoked by bench.sh, not directly.
|
||||
|
||||
Starts llama-server from a given build directory with the production config,
|
||||
waits for /health, POSTs fixed slices of the Pan Tadeusz corpus to /completion,
|
||||
records timings.prompt_n / prompt_ms / prompt_per_second, then shuts the server
|
||||
down cleanly.
|
||||
|
||||
Prompt slices are FIXED CONSTANTS (see SLICES below), calibrated once against
|
||||
the Qwen3.6-35B-A3B tokenizer, so that every build is measured on byte-identical
|
||||
input. Do not re-calibrate for a normal run; use --calibrate only if the model
|
||||
or the corpus changes.
|
||||
|
||||
Usage:
|
||||
ppbench.py <build_dir> <label> [--sizes 4096,16384,32768] [--reps 3]
|
||||
[--results PATH] [--calibrate]
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from statistics import median
|
||||
|
||||
BENCH_DIR = "/root/bench"
|
||||
CORPUS = os.path.join(BENCH_DIR, "pan-tadeusz.txt")
|
||||
DEFAULT_RESULTS = os.path.join(BENCH_DIR, "results.txt")
|
||||
|
||||
MODEL = ("/root/.cache/huggingface/hub/models--unsloth--Qwen3.6-35B-A3B-GGUF/"
|
||||
"snapshots/a483e9e6cbd595906af30beda3187c2663a1118c/"
|
||||
"Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf")
|
||||
|
||||
PORT = 8099
|
||||
BASE = "http://127.0.0.1:%d" % PORT
|
||||
|
||||
# Production serving config. Keep in sync with /root/config.yaml (llama-swap).
|
||||
# Traps encoded here on purpose:
|
||||
# -b / -ub 2048 : a ubatch larger than the prompt never fills, so every
|
||||
# prompt length below must stay a multiple of 2048.
|
||||
# -lm none : what --no-mmap used to be; --mmap/--no-mmap/--mlock were
|
||||
# removed upstream (#28334), so the old spelling now fails.
|
||||
SERVER_ARGS = ["-t", "6", "-ngl", "99", "--n-cpu-moe", "40",
|
||||
"-b", "2048", "-ub", "2048", "-fa", "1", "-lm", "none",
|
||||
"--ctx-size", "40960", "--no-warmup",
|
||||
"--port", str(PORT), "-m", MODEL]
|
||||
|
||||
# Calibrated once on 2026-09-08 against Qwen3.6-35B-A3B-UD-Q4_K_XL:
|
||||
# Polish text of pan-tadeusz.txt runs 2.6702 chars/token, and /completion
|
||||
# reports prompt_n identical to /tokenize (BOS offset 0).
|
||||
# Each slice is corpus[0:NCHARS] read as UTF-8 text; NBYTES is the resulting
|
||||
# UTF-8 byte length, recorded so the slice can be reproduced with byte tools.
|
||||
# These produce EXACTLY the target prompt_n. Do not edit without recalibrating.
|
||||
SLICES = {
|
||||
4096: {"chars": 10776, "bytes": 11608},
|
||||
16384: {"chars": 43858, "bytes": 47175},
|
||||
32768: {"chars": 87165, "bytes": 93843},
|
||||
}
|
||||
|
||||
DEFAULT_SIZES = [4096, 16384, 32768]
|
||||
|
||||
|
||||
def post(path, payload, timeout=2400):
|
||||
req = urllib.request.Request(
|
||||
BASE + path,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
|
||||
|
||||
def health_ok():
|
||||
try:
|
||||
with urllib.request.urlopen(BASE + "/health", timeout=5) as r:
|
||||
return json.loads(r.read().decode("utf-8")).get("status") == "ok"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def start_server(build_dir, logpath):
|
||||
env = dict(os.environ)
|
||||
env["LD_LIBRARY_PATH"] = os.path.join(build_dir, "bin")
|
||||
binary = os.path.join(build_dir, "bin", "llama-server")
|
||||
if not os.path.exists(binary):
|
||||
raise RuntimeError("no llama-server at %s" % binary)
|
||||
log = open(logpath, "wb")
|
||||
p = subprocess.Popen([binary] + SERVER_ARGS, stdout=log,
|
||||
stderr=subprocess.STDOUT, env=env,
|
||||
start_new_session=True)
|
||||
deadline = time.time() + 1200
|
||||
while time.time() < deadline:
|
||||
if p.poll() is not None:
|
||||
raise RuntimeError("llama-server exited early rc=%s, see %s"
|
||||
% (p.returncode, logpath))
|
||||
if health_ok():
|
||||
return p
|
||||
time.sleep(2)
|
||||
stop_server(p)
|
||||
raise RuntimeError("llama-server not healthy within 1200s, see %s" % logpath)
|
||||
|
||||
|
||||
def stop_server(p):
|
||||
if p is None or p.poll() is not None:
|
||||
return
|
||||
try:
|
||||
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
|
||||
except Exception:
|
||||
p.terminate()
|
||||
for _ in range(120):
|
||||
if p.poll() is not None:
|
||||
return
|
||||
time.sleep(1)
|
||||
try:
|
||||
os.killpg(os.getpgid(p.pid), signal.SIGKILL)
|
||||
except Exception:
|
||||
p.kill()
|
||||
p.wait()
|
||||
|
||||
|
||||
def n_tokens(text):
|
||||
"""Exact token count via /tokenize. Cheap: no prefill."""
|
||||
return len(post("/tokenize", {"content": text}, timeout=300)["tokens"])
|
||||
|
||||
|
||||
def calibrate(corpus, targets):
|
||||
"""Recompute the SLICES table. Only needed if model or corpus changes."""
|
||||
probe = corpus[:2000]
|
||||
tk = n_tokens(probe)
|
||||
r = post("/completion", {"prompt": probe, "n_predict": 1,
|
||||
"cache_prompt": False, "temperature": 0})
|
||||
offset = r["timings"]["prompt_n"] - tk
|
||||
cpt = len(probe) / float(tk)
|
||||
sys.stderr.write("calibrate: %.4f chars/token, prompt_n offset %d\n"
|
||||
% (cpt, offset))
|
||||
for target in targets:
|
||||
want = target - offset
|
||||
lo, hi = 1, len(corpus)
|
||||
guess = min(len(corpus), max(1, int(want * cpt)))
|
||||
best = None
|
||||
for _ in range(60):
|
||||
got = n_tokens(corpus[:guess])
|
||||
if got == want:
|
||||
best = guess
|
||||
break
|
||||
if got < want:
|
||||
lo = guess + 1
|
||||
else:
|
||||
hi = guess - 1
|
||||
if lo > hi:
|
||||
break
|
||||
guess = (lo + hi) // 2
|
||||
if best is None:
|
||||
best = guess
|
||||
nbytes = len(corpus[:best].encode("utf-8"))
|
||||
sys.stderr.write("calibrate: %d tokens -> chars=%d bytes=%d\n"
|
||||
% (target, best, nbytes))
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
sys.stderr.write(__doc__)
|
||||
return 2
|
||||
build_dir = os.path.abspath(sys.argv[1])
|
||||
label = sys.argv[2]
|
||||
sizes = list(DEFAULT_SIZES)
|
||||
reps = 3
|
||||
results_path = DEFAULT_RESULTS
|
||||
do_calibrate = "--calibrate" in sys.argv
|
||||
args = sys.argv[3:]
|
||||
for i, a in enumerate(args):
|
||||
if a == "--sizes":
|
||||
sizes = [int(x) for x in args[i + 1].split(",")]
|
||||
elif a == "--reps":
|
||||
reps = int(args[i + 1])
|
||||
elif a == "--results":
|
||||
results_path = args[i + 1]
|
||||
|
||||
corpus = open(CORPUS, encoding="utf-8").read()
|
||||
logpath = os.path.join(BENCH_DIR, "server-%s-%d.log" % (label, int(time.time())))
|
||||
|
||||
p = None
|
||||
try:
|
||||
sys.stderr.write("[%s] starting llama-server from %s\n" % (label, build_dir))
|
||||
t0 = time.time()
|
||||
p = start_server(build_dir, logpath)
|
||||
sys.stderr.write("[%s] healthy after %.1fs\n" % (label, time.time() - t0))
|
||||
|
||||
if do_calibrate:
|
||||
calibrate(corpus, sizes)
|
||||
return 0
|
||||
|
||||
stamp = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
out = open(results_path, "a")
|
||||
for target in sizes:
|
||||
if target not in SLICES:
|
||||
sys.stderr.write("no calibrated slice for %d tokens, skipping\n" % target)
|
||||
continue
|
||||
nchars = SLICES[target]["chars"]
|
||||
prompt = corpus[:nchars]
|
||||
rates, mss, ns = [], [], set()
|
||||
for rep in range(1, reps + 1):
|
||||
r = post("/completion", {"prompt": prompt, "n_predict": 1,
|
||||
"cache_prompt": False, "temperature": 0})
|
||||
t = r["timings"]
|
||||
rates.append(t["prompt_per_second"])
|
||||
mss.append(t["prompt_ms"])
|
||||
ns.add(t["prompt_n"])
|
||||
line = ("RESULT corpus label=%s build=%s target=%d chars=%d "
|
||||
"prompt_n=%d rep=%d prompt_ms=%.2f tps=%.2f time=%s"
|
||||
% (label, build_dir, target, nchars, t["prompt_n"], rep,
|
||||
t["prompt_ms"], t["prompt_per_second"], stamp))
|
||||
out.write(line + "\n")
|
||||
out.flush()
|
||||
sys.stderr.write(line + "\n")
|
||||
if ns != {target}:
|
||||
sys.stderr.write("WARNING: prompt_n %s != target %d; slice table "
|
||||
"is stale, rerun with --calibrate\n" % (sorted(ns), target))
|
||||
s = ("SUMMARY corpus label=%s target=%d prompt_n=%s median_tps=%.2f "
|
||||
"min_tps=%.2f max_tps=%.2f median_ms=%.1f time=%s"
|
||||
% (label, target, sorted(ns), median(rates), min(rates),
|
||||
max(rates), median(mss), stamp))
|
||||
out.write(s + "\n")
|
||||
out.flush()
|
||||
sys.stderr.write(s + "\n")
|
||||
out.close()
|
||||
finally:
|
||||
stop_server(p)
|
||||
sys.stderr.write("[%s] server stopped\n" % label)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# One-command entry point for the RX 580 prompt-processing benchmark.
|
||||
#
|
||||
# Runs from your workstation. Copies the pod-side harness into the pod and
|
||||
# execs it there over kubectl. All the real work happens on the pod; this is
|
||||
# a thin wrapper so there is exactly one command to remember.
|
||||
#
|
||||
# Examples:
|
||||
# ./scripts/rx580-bench/run.sh --build /root/llama.cpp/build --label fork-before
|
||||
# ./scripts/rx580-bench/run.sh --build /root/arms/new-clean/build --label new-clean \
|
||||
# --build-b /root/arms/new-fork/build --label-b new-fork
|
||||
# ./scripts/rx580-bench/run.sh --summarize
|
||||
# ./scripts/rx580-bench/run.sh --install-only
|
||||
#
|
||||
# Every argument other than the wrapper-only flags below is passed straight
|
||||
# through to bench.sh on the pod. See bench.sh --help.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
KUBECONFIG_PATH="${KUBECONFIG:-/home/user/Projects/klaster/talos/generated/kubeconfig}"
|
||||
NS="${RX580_NS:-llama}"
|
||||
DEPLOY="${RX580_DEPLOY:-deploy/supervisord}"
|
||||
POD_BENCH_DIR=/root/bench
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
export KUBECONFIG="$KUBECONFIG_PATH"
|
||||
|
||||
kx() { kubectl -n "$NS" exec -i "$DEPLOY" -- sh -c "$1"; }
|
||||
|
||||
install_harness() {
|
||||
echo "[run] installing harness into $POD_BENCH_DIR"
|
||||
kx "mkdir -p $POD_BENCH_DIR"
|
||||
for f in bench.sh ppbench.py lbsweep.sh summarize.py; do
|
||||
kx "cat > $POD_BENCH_DIR/$f" < "$HERE/$f"
|
||||
done
|
||||
kx "chmod +x $POD_BENCH_DIR/bench.sh $POD_BENCH_DIR/lbsweep.sh $POD_BENCH_DIR/ppbench.py $POD_BENCH_DIR/summarize.py"
|
||||
# The corpus is large and rarely changes; only push it if missing.
|
||||
if ! kx "test -s $POD_BENCH_DIR/pan-tadeusz.txt" 2>/dev/null; then
|
||||
if [ -f "$HERE/pan-tadeusz.txt" ]; then
|
||||
echo "[run] uploading corpus"
|
||||
kx "cat > $POD_BENCH_DIR/pan-tadeusz.txt" < "$HERE/pan-tadeusz.txt"
|
||||
else
|
||||
echo "[run] ERROR: $POD_BENCH_DIR/pan-tadeusz.txt missing on the pod and" >&2
|
||||
echo " no local copy at $HERE/pan-tadeusz.txt to upload." >&2
|
||||
exit 4
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
ARGS=()
|
||||
INSTALL_ONLY=0
|
||||
SUMMARIZE=0
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--install-only) INSTALL_ONLY=1 ;;
|
||||
--summarize) SUMMARIZE=1 ;;
|
||||
*) ARGS+=("$a") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
install_harness
|
||||
|
||||
if [ "$INSTALL_ONLY" -eq 1 ]; then
|
||||
echo "[run] harness installed; not running anything"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$SUMMARIZE" -eq 1 ]; then
|
||||
kx "python3 $POD_BENCH_DIR/summarize.py ${ARGS[*]:-}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ${#ARGS[@]} -eq 0 ]; then
|
||||
kx "sh $POD_BENCH_DIR/bench.sh --help"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# A full 3-round A/B run takes hours; run detached on the pod and tail it, so a
|
||||
# dropped kubectl connection cannot orphan a half-finished run with llama-swap
|
||||
# still down. bench.sh restarts llama-swap from its own EXIT trap either way.
|
||||
STAMP="$(date -u +%Y%m%d-%H%M%S)"
|
||||
LOG="$POD_BENCH_DIR/run-$STAMP.log"
|
||||
echo "[run] starting detached run on the pod, log: $LOG"
|
||||
kx "cd $POD_BENCH_DIR && nohup sh $POD_BENCH_DIR/bench.sh ${ARGS[*]} > $LOG 2>&1 & echo started"
|
||||
|
||||
echo "[run] tailing until ALL_RUNS_DONE (safe to Ctrl-C: the pod keeps running)"
|
||||
kx "i=0; while [ \$i -lt 100000 ]; do
|
||||
if grep -qE 'ALL_RUNS_DONE|Traceback|error:' $LOG; then break; fi
|
||||
sleep 15; i=\$((i+1));
|
||||
done; cat $LOG"
|
||||
Executable
+101
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Summarize a results.txt produced by bench.sh into a per-label table.
|
||||
|
||||
Usage: summarize.py [results.txt]
|
||||
|
||||
Prints median and min-max of every metric for every label, and flags
|
||||
cross-label deltas against the ~5 percent noise floor of this machine.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from statistics import median
|
||||
|
||||
NOISE_FLOOR_PCT = 5.0
|
||||
|
||||
RE_CORPUS = re.compile(
|
||||
r"^RESULT corpus label=(\S+) build=\S+ target=(\d+) chars=\d+ "
|
||||
r"prompt_n=(\d+) rep=\d+ prompt_ms=([\d.]+) tps=([\d.]+)")
|
||||
RE_LB = re.compile(
|
||||
r"^LBRESULT label=(\S+) build=\S+ test=(\S+) tps=([\d.]+)")
|
||||
RE_TG_NODEPTH = re.compile(r"^tg\d+-llama-bench$")
|
||||
|
||||
|
||||
def main():
|
||||
path = sys.argv[1] if len(sys.argv) > 1 else "/root/bench/results.txt"
|
||||
# data[metric][label] = list of tps
|
||||
data = {}
|
||||
order = []
|
||||
labels = []
|
||||
for line in open(path):
|
||||
m = RE_CORPUS.match(line)
|
||||
if m:
|
||||
label, target, _pn, _ms, tps = m.groups()
|
||||
metric = "pp%s-corpus" % target
|
||||
else:
|
||||
m = RE_LB.match(line)
|
||||
if not m:
|
||||
continue
|
||||
label, test, tps = m.groups()
|
||||
metric = "%s-llama-bench" % test
|
||||
if metric not in data:
|
||||
data[metric] = {}
|
||||
order.append(metric)
|
||||
data[metric].setdefault(label, []).append(float(tps))
|
||||
if label not in labels:
|
||||
labels.append(label)
|
||||
|
||||
if not data:
|
||||
print("no parseable results in %s" % path)
|
||||
return 1
|
||||
|
||||
def sort_key(m):
|
||||
n = re.search(r"(\d+)", m)
|
||||
return (0 if "corpus" in m else 1, int(n.group(1)) if n else 0)
|
||||
order.sort(key=sort_key)
|
||||
|
||||
w = max(len(l) for l in labels) + 2
|
||||
head = "metric".ljust(26) + "".join(l.ljust(max(w, 24)) for l in labels)
|
||||
print("")
|
||||
print("t/s, median (min-max), n samples")
|
||||
print(head)
|
||||
print("-" * len(head))
|
||||
for metric in order:
|
||||
row = metric.ljust(26)
|
||||
for label in labels:
|
||||
vals = data[metric].get(label)
|
||||
if not vals:
|
||||
row += "-".ljust(max(w, 24))
|
||||
else:
|
||||
cell = "%.1f (%.1f-%.1f) n=%d" % (
|
||||
median(vals), min(vals), max(vals), len(vals))
|
||||
row += cell.ljust(max(w, 24))
|
||||
print(row)
|
||||
if any(RE_TG_NODEPTH.match(m) for m in order):
|
||||
print("note: tgN rows without @d<depth> ran at n_ctx ~N (old default);"
|
||||
" INVALID decode metric on this box, history only")
|
||||
|
||||
if len(labels) >= 2:
|
||||
base = labels[0]
|
||||
print("")
|
||||
print("deltas vs %s (noise floor %.0f%%, anything under it is UNPROVEN)"
|
||||
% (base, NOISE_FLOOR_PCT))
|
||||
for other in labels[1:]:
|
||||
print(" %s vs %s:" % (other, base))
|
||||
for metric in order:
|
||||
a = data[metric].get(base)
|
||||
b = data[metric].get(other)
|
||||
if not a or not b:
|
||||
continue
|
||||
ma, mb = median(a), median(b)
|
||||
pct = (mb - ma) / ma * 100.0
|
||||
verdict = "SIGNIFICANT" if abs(pct) >= NOISE_FLOOR_PCT else "unproven"
|
||||
if RE_TG_NODEPTH.match(metric):
|
||||
verdict = "INVALID metric (tg at n_ctx ~N), ignore"
|
||||
print(" %-26s %+6.1f%% %s" % (metric, pct, verdict))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+156
-21
@@ -635,7 +635,8 @@ llama_context::~llama_context() {
|
||||
synchronize();
|
||||
|
||||
// when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
|
||||
if (!model.no_alloc() && !opt_ctx) {
|
||||
// the scheduler is also gone if the context is destroyed while cold (on-demand VRAM eviction, see release_device)
|
||||
if (sched && !model.no_alloc() && !opt_ctx) {
|
||||
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
||||
ggml_backend_t backend = backend_ptrs[i];
|
||||
ggml_backend_buffer_type_t buft = backend_buft[i];
|
||||
@@ -980,6 +981,86 @@ void llama_context::synchronize() {
|
||||
t_compute_start_us = 0;
|
||||
}
|
||||
|
||||
bool llama_context::device_resident() const {
|
||||
return model.weights_resident() && (!moe_cache || moe_cache->resident());
|
||||
}
|
||||
|
||||
void llama_context::release_device(bool evict_kv) {
|
||||
if (!device_resident() && !(evict_kv && !kv_device_evicted)) {
|
||||
return;
|
||||
}
|
||||
// ensure no compute is in flight before freeing the device buffers
|
||||
synchronize();
|
||||
model.release_device_weights();
|
||||
// the banks are emptied, not shadowed: restored slot maps must not point at zeroed experts
|
||||
if (moe_cache) {
|
||||
moe_cache->release_device();
|
||||
}
|
||||
if (evict_kv && memory && !kv_device_evicted) {
|
||||
memory->release_device_buffers();
|
||||
kv_device_evicted = true;
|
||||
// also free the scheduler and its worst-case compute buffer (hundreds of MiB) so a cold
|
||||
// model holds essentially no VRAM. Rebuilt lazily by sched_reserve() on restore.
|
||||
sched.reset();
|
||||
sched_need_reserve = true;
|
||||
// finally, free each backend's own compute-scratch (Vulkan prealloc/staging buffers, which
|
||||
// are owned by the backend and survive sched.reset()). Reallocated lazily on next compute.
|
||||
for (auto & backend : backends) {
|
||||
ggml_backend_free_scratch(backend.get());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void llama_context::discard_device() {
|
||||
// same as release_device(true), minus the host copies of the weights and the KV
|
||||
synchronize();
|
||||
model.release_device_weights(/*keep_shadow =*/ false);
|
||||
if (moe_cache) {
|
||||
moe_cache->release_device();
|
||||
}
|
||||
if (memory) {
|
||||
memory->release_device_buffers(/*keep_shadow =*/ false);
|
||||
}
|
||||
kv_device_evicted = true;
|
||||
device_discarded = true;
|
||||
sched.reset();
|
||||
sched_need_reserve = true;
|
||||
for (auto & backend : backends) {
|
||||
ggml_backend_free_scratch(backend.get());
|
||||
}
|
||||
}
|
||||
|
||||
bool llama_context::restore_memory_host() {
|
||||
device_discarded = true;
|
||||
if (!kv_device_evicted || !memory) {
|
||||
return true;
|
||||
}
|
||||
return memory->restore_device_buffers(/*to_host =*/ true);
|
||||
}
|
||||
|
||||
void llama_context::restore_device() {
|
||||
GGML_ASSERT(!device_discarded && "device buffers were discarded");
|
||||
if (device_resident() && !kv_device_evicted && sched) {
|
||||
return;
|
||||
}
|
||||
model.restore_device_weights();
|
||||
if (moe_cache) {
|
||||
moe_cache->restore_device();
|
||||
}
|
||||
if (kv_device_evicted && memory) {
|
||||
memory->restore_device_buffers();
|
||||
kv_device_evicted = false;
|
||||
}
|
||||
// rebuild the scheduler + compute buffer if they were freed on release (evict_kv mode)
|
||||
if (!sched) {
|
||||
sched_reserve();
|
||||
}
|
||||
// make sure all weight/KV uploads have completed before any compute reads them
|
||||
if (sched) {
|
||||
ggml_backend_sched_synchronize(sched.get());
|
||||
}
|
||||
}
|
||||
|
||||
const llama_model & llama_context::get_model() const {
|
||||
return model;
|
||||
}
|
||||
@@ -2164,6 +2245,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// on-demand VRAM: if the weights were released while this model was idle, bring them back
|
||||
// to the device before building the compute graph
|
||||
if (!device_resident()) {
|
||||
restore_device();
|
||||
}
|
||||
|
||||
const auto & vocab = model.vocab;
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
@@ -3152,16 +3239,40 @@ bool llama_context::sched_copy_experts(ggml_backend_t backend, const ggml_tensor
|
||||
const size_t expert_size = src->nb[2];
|
||||
|
||||
if (ids != st.ids || (int64_t) st.used.size() != n_expert) {
|
||||
st.ids_data.resize(ggml_nbytes(ids)/sizeof(int32_t));
|
||||
ggml_backend_tensor_get_async(backend, ids, st.ids_data.data(), 0, ggml_nbytes(ids));
|
||||
ggml_backend_synchronize(backend);
|
||||
// Reading the ids back forces a full pipeline flush, since the ids are
|
||||
// produced on the same device we are about to upload to. Once the batch
|
||||
// draws enough experts, essentially all of them come back used and the
|
||||
// readback has bought nothing, so skip it and copy the whole tensor as one
|
||||
// range. Copying an expert that no id selects cannot change the result:
|
||||
// mul_mat_id only reads the rows the ids point at, so this stays exact.
|
||||
// With the MoE cache on, cached experts are still copied device-to-device.
|
||||
// validation/A-B knob: LLAMA_MOE_IDS_SKIP=0 always reads the ids back (the upstream path),
|
||||
// =log reports every skip on stderr
|
||||
static const char * skip_env = getenv("LLAMA_MOE_IDS_SKIP");
|
||||
static const bool skip_off = skip_env != nullptr && strcmp(skip_env, "0") == 0;
|
||||
static const bool skip_log = skip_env != nullptr && strcmp(skip_env, "log") == 0;
|
||||
|
||||
st.used.assign(n_expert, false);
|
||||
for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) {
|
||||
for (int64_t i0 = 0; i0 < ids->ne[0]; i0++) {
|
||||
const int32_t id = st.ids_data[i1*ids->nb[1]/sizeof(int32_t) + i0*ids->nb[0]/sizeof(int32_t)];
|
||||
GGML_ASSERT(id >= 0 && id < n_expert);
|
||||
st.used[id] = true;
|
||||
const bool all_experts_used = !skip_off && ggml_nelements(ids) >= 4*n_expert;
|
||||
|
||||
if (all_experts_used) {
|
||||
if (skip_log) {
|
||||
fprintf(stderr, "sched_copy_experts: ids readback skipped, n_ids=%" PRId64 " n_expert=%" PRId64 "\n",
|
||||
ggml_nelements(ids), n_expert);
|
||||
}
|
||||
st.ids_data.clear();
|
||||
st.used.assign(n_expert, true);
|
||||
} else {
|
||||
st.ids_data.resize(ggml_nbytes(ids)/sizeof(int32_t));
|
||||
ggml_backend_tensor_get_async(backend, ids, st.ids_data.data(), 0, ggml_nbytes(ids));
|
||||
ggml_backend_synchronize(backend);
|
||||
|
||||
st.used.assign(n_expert, false);
|
||||
for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) {
|
||||
for (int64_t i0 = 0; i0 < ids->ne[0]; i0++) {
|
||||
const int32_t id = st.ids_data[i1*ids->nb[1]/sizeof(int32_t) + i0*ids->nb[0]/sizeof(int32_t)];
|
||||
GGML_ASSERT(id >= 0 && id < n_expert);
|
||||
st.used[id] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4124,17 +4235,21 @@ llama_memory_breakdown llama_context::memory_breakdown() const {
|
||||
ret[buft].context += size;
|
||||
}
|
||||
}
|
||||
if (model.no_alloc()) {
|
||||
for (size_t i = 0; i < backends.size(); ++i) {
|
||||
ggml_backend_t backend = backends[i].get();
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
||||
ret[buft].compute += backend_buf_exp_size[i];
|
||||
}
|
||||
} else {
|
||||
for (const auto & backend_ptr : backends) {
|
||||
ggml_backend_t backend = backend_ptr.get();
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
||||
ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend);
|
||||
// the scheduler (and its compute buffers) may have been freed while the model is cold
|
||||
// (on-demand VRAM eviction, see release_device); it contributes no compute VRAM then.
|
||||
if (sched) {
|
||||
if (model.no_alloc()) {
|
||||
for (size_t i = 0; i < backends.size(); ++i) {
|
||||
ggml_backend_t backend = backends[i].get();
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
||||
ret[buft].compute += backend_buf_exp_size[i];
|
||||
}
|
||||
} else {
|
||||
for (const auto & backend_ptr : backends) {
|
||||
ggml_backend_t backend = backend_ptr.get();
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
|
||||
ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
@@ -4652,6 +4767,26 @@ void llama_synchronize(llama_context * ctx) {
|
||||
ctx->synchronize();
|
||||
}
|
||||
|
||||
void llama_context_release_device(llama_context * ctx, bool evict_kv) {
|
||||
ctx->release_device(evict_kv);
|
||||
}
|
||||
|
||||
void llama_context_restore_device(llama_context * ctx) {
|
||||
ctx->restore_device();
|
||||
}
|
||||
|
||||
void llama_context_discard_device(llama_context * ctx) {
|
||||
ctx->discard_device();
|
||||
}
|
||||
|
||||
bool llama_context_restore_memory_host(llama_context * ctx) {
|
||||
return ctx->restore_memory_host();
|
||||
}
|
||||
|
||||
bool llama_context_weights_resident(const llama_context * ctx) {
|
||||
return ctx->device_resident();
|
||||
}
|
||||
|
||||
float * llama_get_logits(llama_context * ctx) {
|
||||
ctx->synchronize();
|
||||
|
||||
|
||||
@@ -60,6 +60,23 @@ struct llama_context {
|
||||
|
||||
void synchronize();
|
||||
|
||||
// on-demand device (VRAM) residency: free / rebuild the model's GPU weight buffers so that
|
||||
// VRAM can be time-shared with another model. release_device() synchronizes first; decode()
|
||||
// auto-restores when it finds the weights have been released. If evict_kv is set, the KV cache
|
||||
// device buffers are also evicted to a host shadow (for when the KV would not fit alongside the
|
||||
// other model) - this adds a D2H/H2D copy of the live KV on each cycle.
|
||||
void release_device(bool evict_kv = false);
|
||||
void restore_device();
|
||||
// free every device buffer for good, without host copies (before exit). The context must not
|
||||
// compute again: restore_device() aborts.
|
||||
void discard_device();
|
||||
// before exit: bring an evicted KV/recurrent state back in host memory so the state API can read
|
||||
// it without the device. The context must not compute again: restore_device() aborts.
|
||||
bool restore_memory_host();
|
||||
|
||||
// the model weights and the MoE cache are on the device
|
||||
bool device_resident() const;
|
||||
|
||||
const llama_model & get_model() const;
|
||||
const llama_cparams & get_cparams() const;
|
||||
|
||||
@@ -431,6 +448,11 @@ private:
|
||||
int32_t n_reallocs_same = 0;
|
||||
int32_t n_reallocs_diff = 0;
|
||||
|
||||
// on-demand device residency: true while the KV cache device buffers have been evicted to host
|
||||
bool kv_device_evicted = false;
|
||||
// set by discard_device(): the device buffers are gone and cannot be restored
|
||||
bool device_discarded = false;
|
||||
|
||||
ggml_backend_t backend_cpu = nullptr;
|
||||
std::vector<ggml_backend_ptr> backends;
|
||||
|
||||
|
||||
+54
-32
@@ -1415,7 +1415,7 @@ void llm_graph_input_mem_hybrid_iswa_rel::set_input(const llama_ubatch * ubatch)
|
||||
}
|
||||
|
||||
if (self_rel_pos->buffer) {
|
||||
attn_ctx->get_base()->set_input_relative_pos(self_rel_pos, ubatch, extent);
|
||||
attn_ctx->get_base()->set_input_relative_pos(self_rel_pos, inp_attn->self_attn_mask.rows, ubatch, extent);
|
||||
}
|
||||
|
||||
// swa tensors may not be allocated if there are no SWA attention layers
|
||||
@@ -1429,7 +1429,7 @@ void llm_graph_input_mem_hybrid_iswa_rel::set_input(const llama_ubatch * ubatch)
|
||||
}
|
||||
|
||||
if (self_rel_pos_swa->buffer) {
|
||||
attn_ctx->get_swa()->set_input_relative_pos(self_rel_pos_swa, ubatch, extent_swa);
|
||||
attn_ctx->get_swa()->set_input_relative_pos(self_rel_pos_swa, inp_attn->self_attn_mask_swa.rows, ubatch, extent_swa);
|
||||
}
|
||||
|
||||
const int64_t n_rs = mctx->get_recr()->get_n_rs();
|
||||
@@ -1454,16 +1454,13 @@ bool llm_graph_input_mem_hybrid_iswa_rel::can_reuse(const llm_graph_params & par
|
||||
|
||||
const auto * attn_ctx = mctx->get_attn();
|
||||
|
||||
// the masks are always dense, see build_inp_mem_hybrid_iswa_rel()
|
||||
|
||||
// base tensors may not be allocated if there are no non-SWA attention layers
|
||||
if (inp_attn->self_k_idxs && inp_attn->self_k_idxs->buffer) {
|
||||
res &= inp_attn->self_k_idxs->ne[0] == params.ubatch.n_tokens;
|
||||
}
|
||||
|
||||
res &= can_reuse_kq_mask(inp_attn->self_attn_mask.raw, attn_ctx->get_base(), params.ubatch, params.cparams);
|
||||
res &= can_reuse_attn_mask(inp_attn->self_attn_mask, attn_ctx->get_base(), params.ubatch, params.cparams);
|
||||
|
||||
res &= self_rel_pos->ne[0] == attn_ctx->get_base()->get_n_kv();
|
||||
res &= self_rel_pos->ne[1] == params.ubatch.n_tokens;
|
||||
|
||||
// swa tensors may not be allocated if there are no SWA attention layers
|
||||
@@ -1471,9 +1468,8 @@ bool llm_graph_input_mem_hybrid_iswa_rel::can_reuse(const llm_graph_params & par
|
||||
res &= inp_attn->self_k_idxs_swa->ne[0] == params.ubatch.n_tokens;
|
||||
}
|
||||
|
||||
res &= can_reuse_kq_mask(inp_attn->self_attn_mask_swa.raw, attn_ctx->get_swa(), params.ubatch, params.cparams);
|
||||
res &= can_reuse_attn_mask(inp_attn->self_attn_mask_swa, attn_ctx->get_swa(), params.ubatch, params.cparams);
|
||||
|
||||
res &= self_rel_pos_swa->ne[0] == attn_ctx->get_swa()->get_n_kv();
|
||||
res &= self_rel_pos_swa->ne[1] == params.ubatch.n_tokens;
|
||||
|
||||
res &= inp_rs->s_copy->ne[0] == mctx->get_recr()->get_n_rs();
|
||||
@@ -3114,7 +3110,9 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
ggml_tensor * v_mla,
|
||||
int64_t n_kv_max,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
int il,
|
||||
ggml_tensor * bias_table,
|
||||
ggml_tensor * bias_indices) const {
|
||||
const bool v_trans = v->nb[1] > v->nb[2];
|
||||
|
||||
// split the batch into streams if needed
|
||||
@@ -3129,6 +3127,21 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
k = ggml_permute(ctx0, k, 0, 2, 1, 3);
|
||||
v = ggml_permute(ctx0, v, 0, 2, 1, 3);
|
||||
|
||||
auto dense_bias_mask = [&]() {
|
||||
ggml_tensor * table = ggml_cont(ctx0, ggml_permute(ctx0, bias_table, 1, 0, 2, 3));
|
||||
ggml_tensor * bias = ggml_get_rows(ctx0, table, bias_indices);
|
||||
bias = ggml_reshape_4d(ctx0, bias, q->ne[2], bias_indices->ne[0], q->ne[1], n_stream);
|
||||
bias = ggml_cont(ctx0, ggml_permute(ctx0, bias, 2, 0, 1, 3));
|
||||
cb(bias, "kq_b", il);
|
||||
ggml_tensor * mask = kq_mask->type == GGML_TYPE_F32 ? kq_mask : ggml_cast(ctx0, kq_mask, GGML_TYPE_F32);
|
||||
mask = ggml_add(ctx0, bias, mask);
|
||||
if (cparams.flash_attn) {
|
||||
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
|
||||
}
|
||||
cb(mask, "kq_mask_rel_pos", il);
|
||||
return mask;
|
||||
};
|
||||
|
||||
ggml_tensor * cur;
|
||||
|
||||
const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr;
|
||||
@@ -3162,6 +3175,23 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
GGML_ASSERT(n_kv_max >= 0 && n_kv_max <= INT32_MAX);
|
||||
ggml_flash_attn_ext_set_n_kv_max(cur, static_cast<int32_t>(n_kv_max));
|
||||
ggml_prec_set_acc(cur, GGML_PREC_F32);
|
||||
if (bias_table) {
|
||||
ggml_flash_attn_ext_add_indexed_bias(cur, bias_table, bias_indices);
|
||||
const auto dev = model.dev_layer(il);
|
||||
if (!cparams.offload_kqv || !dev || !ggml_backend_dev_supports_op(dev, cur)) {
|
||||
ggml_flash_attn_ext_add_indexed_bias(cur, nullptr, nullptr);
|
||||
cur->src[3] = dense_bias_mask();
|
||||
} else if (kv_rows) {
|
||||
// Looped layers can project Q and the bias on a different device from their KV cache.
|
||||
for (int i = 0; i < ggml_backend_sched_get_n_backends(sched); ++i) {
|
||||
auto backend = ggml_backend_sched_get_backend(sched, i);
|
||||
if (ggml_backend_get_device(backend) == dev) {
|
||||
ggml_backend_sched_set_tensor_backend(sched, cur, backend);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (v_mla) {
|
||||
#if 0
|
||||
@@ -3182,6 +3212,9 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
|
||||
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);
|
||||
} else {
|
||||
if (bias_table) {
|
||||
kq_mask = dense_bias_mask();
|
||||
}
|
||||
ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
|
||||
cb(kq, "kq", il);
|
||||
|
||||
@@ -3691,7 +3724,9 @@ static ggml_tensor * build_attn_iswa_impl(
|
||||
ggml_tensor * sinks,
|
||||
ggml_tensor * v_mla,
|
||||
float kq_scale,
|
||||
int il) {
|
||||
int il,
|
||||
ggml_tensor * bias_table = nullptr,
|
||||
ggml_tensor * bias_indices = nullptr) {
|
||||
ggml_context * ctx0 = gctx.ctx0;
|
||||
ggml_cgraph * gf = gctx.gf;
|
||||
|
||||
@@ -3757,7 +3792,7 @@ static ggml_tensor * build_attn_iswa_impl(
|
||||
ggml_tensor * k = use_kv_cur ? k_cur : mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = use_kv_cur ? v_cur : mctx_cur->get_v(ctx0, il);
|
||||
|
||||
ggml_tensor * cur = gctx.build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il);
|
||||
ggml_tensor * cur = gctx.build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il, bias_table, bias_indices);
|
||||
cur = build_attn_scale_kv_output(ctx0, mctx_cur, cur, il);
|
||||
gctx.cb(cur, "kqv_out", il);
|
||||
|
||||
@@ -4351,19 +4386,18 @@ llm_graph_input_mem_hybrid_iswa_rel * llm_graph_context::build_inp_mem_hybrid_is
|
||||
|
||||
auto inp_attn = std::make_unique<llm_graph_input_attn_kv_iswa>(hparams, cparams, attn_ctx);
|
||||
|
||||
// dense masks: the bias is gathered by the distance to each cell of the K/V view
|
||||
{
|
||||
inp_attn->self_k_idxs = attn_ctx->get_base()->build_input_k_idxs(ctx0, ubatch);
|
||||
inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch);
|
||||
|
||||
inp_attn->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_base(), ubatch, cparams);
|
||||
inp_attn->self_attn_mask = build_attn_inp_attn_mask(ctx0, attn_ctx->get_base(), ubatch, cparams);
|
||||
}
|
||||
|
||||
{
|
||||
inp_attn->self_k_idxs_swa = attn_ctx->get_swa()->build_input_k_idxs(ctx0, ubatch);
|
||||
inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch);
|
||||
|
||||
inp_attn->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams);
|
||||
inp_attn->self_attn_mask_swa = build_attn_inp_attn_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams);
|
||||
}
|
||||
|
||||
inp_attn->self_k_rot = attn_ctx->get_base()->get_k_rot();
|
||||
@@ -4374,11 +4408,11 @@ llm_graph_input_mem_hybrid_iswa_rel * llm_graph_context::build_inp_mem_hybrid_is
|
||||
|
||||
auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa_rel>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur, extent, extent_swa);
|
||||
|
||||
inp->self_rel_pos = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, attn_ctx->get_base()->get_n_kv(), ubatch.n_tokens);
|
||||
inp->self_rel_pos = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, inp->get_attn()->self_attn_mask.raw->ne[0], ubatch.n_tokens);
|
||||
ggml_set_input(inp->self_rel_pos);
|
||||
ggml_set_name(inp->self_rel_pos, "attn_inp_rel_pos");
|
||||
|
||||
inp->self_rel_pos_swa = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, attn_ctx->get_swa()->get_n_kv(), ubatch.n_tokens);
|
||||
inp->self_rel_pos_swa = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, inp->get_attn()->self_attn_mask_swa.raw->ne[0], ubatch.n_tokens);
|
||||
ggml_set_input(inp->self_rel_pos_swa);
|
||||
ggml_set_name(inp->self_rel_pos_swa, "attn_inp_rel_pos_swa");
|
||||
|
||||
@@ -4393,7 +4427,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * q_cur,
|
||||
ggml_tensor * k_cur,
|
||||
ggml_tensor * v_cur,
|
||||
ggml_tensor * kq_b,
|
||||
ggml_tensor * bias_table,
|
||||
ggml_tensor * sinks,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
@@ -4403,22 +4437,10 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
|
||||
const auto & attn_mask = is_swa ? inp_attn->self_attn_mask_swa : inp_attn->self_attn_mask;
|
||||
|
||||
// TODO: support kv_rows - fill the distances over the kv_rows cell lists and pass the rows to build_attn_mha,
|
||||
// needs a kv_rows flash attention with a per-head mask (CUDA has none)
|
||||
GGML_ASSERT(attn_mask.rows == nullptr);
|
||||
ggml_tensor * indices = is_swa ? inp->self_rel_pos_swa : inp->self_rel_pos;
|
||||
|
||||
// a mask per head, in the dense layout of the input mask, which flash attention takes in F16
|
||||
ggml_tensor * kq_mask = attn_mask.raw;
|
||||
if (kq_mask->type != GGML_TYPE_F32) {
|
||||
kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F32);
|
||||
}
|
||||
kq_mask = ggml_add(ctx0, kq_b, kq_mask);
|
||||
if (cparams.flash_attn) {
|
||||
kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
|
||||
}
|
||||
cb(kq_mask, "kq_mask_rel_pos", il);
|
||||
|
||||
return build_attn_iswa_impl(*this, inp_attn, wo, wo_b, wo_s, q_cur, k_cur, v_cur, nullptr, kq_mask, nullptr, sinks, nullptr, kq_scale, il);
|
||||
return build_attn_iswa_impl(*this, inp_attn, wo, wo_b, wo_s, q_cur, k_cur, v_cur, nullptr, attn_mask.raw, attn_mask.rows,
|
||||
sinks, nullptr, kq_scale, il, bias_table, indices);
|
||||
}
|
||||
|
||||
void llm_graph_context::build_dense_out(
|
||||
|
||||
+6
-5
@@ -774,8 +774,7 @@ public:
|
||||
};
|
||||
|
||||
// hybrid iswa memory with a per-head attention bias that depends on the distance from the query to the cell (Inkling)
|
||||
// the KQ masks are always dense (rows == nullptr) and are filled from the same cells as the distances, so that the bias
|
||||
// gathered by distance lines up with the mask
|
||||
// The visibility mask and distances use the same cache cell order.
|
||||
class llm_graph_input_mem_hybrid_iswa_rel : public llm_graph_input_i {
|
||||
public:
|
||||
llm_graph_input_mem_hybrid_iswa_rel(
|
||||
@@ -1336,7 +1335,9 @@ struct llm_graph_context {
|
||||
ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]
|
||||
int64_t n_kv_max,
|
||||
float kq_scale,
|
||||
int il) const;
|
||||
int il,
|
||||
ggml_tensor * bias_table = nullptr,
|
||||
ggml_tensor * bias_indices = nullptr) const;
|
||||
|
||||
llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const;
|
||||
|
||||
@@ -1534,7 +1535,7 @@ struct llm_graph_context {
|
||||
uint32_t extent,
|
||||
uint32_t extent_swa) const;
|
||||
|
||||
// kq_b is added to the KQ mask, after the KQ scale
|
||||
// Gather the per-head bias by distance and add it to the visibility mask.
|
||||
ggml_tensor * build_attn(
|
||||
llm_graph_input_mem_hybrid_iswa_rel * inp,
|
||||
ggml_tensor * wo,
|
||||
@@ -1543,7 +1544,7 @@ struct llm_graph_context {
|
||||
ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
|
||||
ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
|
||||
ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
|
||||
ggml_tensor * kq_b, // [n_kv, n_tokens/n_stream, n_head_q, n_stream]
|
||||
ggml_tensor * bias_table, // [n_bins, n_head_q, n_tokens]
|
||||
ggml_tensor * sinks, // [n_head_q]
|
||||
float kq_scale,
|
||||
int il) const;
|
||||
|
||||
@@ -264,6 +264,17 @@ void llama_kv_cache_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id,
|
||||
kv_swa ->state_read(io, seq_id, flags);
|
||||
}
|
||||
|
||||
void llama_kv_cache_iswa::release_device_buffers(bool keep_shadow) {
|
||||
kv_base->release_device_buffers(keep_shadow);
|
||||
kv_swa->release_device_buffers(keep_shadow);
|
||||
}
|
||||
|
||||
bool llama_kv_cache_iswa::restore_device_buffers(bool to_host) {
|
||||
bool ok = kv_base->restore_device_buffers(to_host);
|
||||
ok = kv_swa->restore_device_buffers(to_host) && ok;
|
||||
return ok;
|
||||
}
|
||||
|
||||
llama_kv_cache * llama_kv_cache_iswa::get_base() const {
|
||||
return kv_base.get();
|
||||
}
|
||||
|
||||
@@ -83,6 +83,9 @@ public:
|
||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers(bool to_host = false) override;
|
||||
|
||||
//
|
||||
// llama_kv_cache_iswa specific API
|
||||
//
|
||||
|
||||
+96
-6
@@ -429,7 +429,9 @@ void llama_kv_cache::clear(bool data) {
|
||||
|
||||
if (data) {
|
||||
for (auto & [_, buf] : ctxs_bufs) {
|
||||
ggml_backend_buffer_clear(buf.get(), 0);
|
||||
if (buf) { // may be null if evicted for on-demand VRAM sharing
|
||||
ggml_backend_buffer_clear(buf.get(), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2007,14 +2009,18 @@ bool llama_kv_cache::has_cell_ext() const {
|
||||
return hparams.n_pos_per_embd() > 1 || hparams.ple_n_heads > 0;
|
||||
}
|
||||
|
||||
void llama_kv_cache::set_input_relative_pos(ggml_tensor * dst, const llama_ubatch * ubatch, uint32_t extent) const {
|
||||
void llama_kv_cache::set_input_relative_pos(ggml_tensor * dst, const ggml_tensor * rows, const llama_ubatch * ubatch, uint32_t extent) const {
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_I32);
|
||||
const int64_t n_kv = dst->ne[0];
|
||||
GGML_ASSERT(!rows || (rows->ne[0] == n_kv && ggml_backend_buffer_is_host(rows->buffer)));
|
||||
const int64_t n_tps = ubatch->n_tokens / (rows ? rows->ne[1] : 1);
|
||||
std::vector<int32_t> data(n_kv * ubatch->n_tokens);
|
||||
for (uint32_t i = 0; i < ubatch->n_tokens; ++i) {
|
||||
const auto & cells = v_cells.at(seq_to_stream.at(ubatch->seq_id[i][0]));
|
||||
const int32_t * list = rows ? (const int32_t *) rows->data + (i/n_tps)*n_kv : nullptr;
|
||||
for (int64_t j = 0; j < n_kv; ++j) {
|
||||
const int64_t distance = cells.is_empty(j) ? -1 : int64_t(ubatch->pos[i]) - cells.pos_get(j);
|
||||
const int64_t cell = list ? list[j] : j;
|
||||
const int64_t distance = cell < 0 || cells.is_empty(cell) ? -1 : int64_t(ubatch->pos[i]) - cells.pos_get(cell);
|
||||
data[i*n_kv + j] = distance >= 0 && distance < extent ? distance : extent;
|
||||
}
|
||||
}
|
||||
@@ -2084,12 +2090,96 @@ size_t llama_kv_cache::total_size() const {
|
||||
size_t size = 0;
|
||||
|
||||
for (const auto & [_, buf] : ctxs_bufs) {
|
||||
size += ggml_backend_buffer_get_size(buf.get());
|
||||
if (buf) { // may be null if evicted for on-demand VRAM sharing
|
||||
size += ggml_backend_buffer_get_size(buf.get());
|
||||
}
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
void llama_kv_cache::release_device_buffers(bool keep_shadow) {
|
||||
// NOTE: the caller must have synchronized the backend so nothing references these buffers.
|
||||
// The KV cache is read-write, so its host shadow is (re)captured fresh on every release.
|
||||
if (dev_released) {
|
||||
return;
|
||||
}
|
||||
dev_shadows.assign(ctxs_bufs.bufs.size(), device_buffer_shadow{});
|
||||
size_t freed = 0;
|
||||
for (size_t i = 0; i < ctxs_bufs.bufs.size(); ++i) {
|
||||
ggml_context * ctx = ctxs_bufs.bufs[i].first.get();
|
||||
ggml_backend_buffer_t buf = ctxs_bufs.bufs[i].second.get();
|
||||
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
|
||||
continue; // only real device (VRAM) buffers are evictable
|
||||
}
|
||||
auto & sh = dev_shadows[i];
|
||||
sh.releasable = true;
|
||||
sh.buft = ggml_backend_buffer_get_type(buf);
|
||||
|
||||
// capture live contents compactly in stable iteration order (skip views, which alias a base)
|
||||
if (keep_shadow) {
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||
}
|
||||
sh.data.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, sh.data.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
}
|
||||
|
||||
freed += ggml_backend_buffer_get_size(buf);
|
||||
ctxs_bufs.bufs[i].second.reset(); // free the device buffer
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
t->buffer = nullptr;
|
||||
t->data = nullptr;
|
||||
}
|
||||
}
|
||||
dev_released = true;
|
||||
if (freed > 0) {
|
||||
LLAMA_LOG_INFO("%s: released %.2f MiB of KV cache from device\n", __func__, freed / 1024.0 / 1024.0);
|
||||
}
|
||||
}
|
||||
|
||||
bool llama_kv_cache::restore_device_buffers(bool to_host) {
|
||||
if (!dev_released) {
|
||||
return true;
|
||||
}
|
||||
for (size_t i = 0; i < ctxs_bufs.bufs.size(); ++i) {
|
||||
auto & sh = dev_shadows[i];
|
||||
if (!sh.releasable) {
|
||||
continue;
|
||||
}
|
||||
ggml_context * ctx = ctxs_bufs.bufs[i].first.get();
|
||||
if (to_host) {
|
||||
ctxs_bufs.bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data));
|
||||
continue;
|
||||
}
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft);
|
||||
if (buf == nullptr) {
|
||||
LLAMA_LOG_ERROR("%s: failed to reallocate KV cache device buffer (out of VRAM?)\n", __func__);
|
||||
return false;
|
||||
}
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_set(t, sh.data.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
ctxs_bufs.bufs[i].second.reset(buf);
|
||||
}
|
||||
dev_released = false;
|
||||
if (!to_host) {
|
||||
dev_shadows.clear(); // recaptured on next release; to_host buffers point into the shadows
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t llama_kv_cache::size_k_bytes() const {
|
||||
size_t size_k_bytes = 0;
|
||||
|
||||
@@ -3124,8 +3214,8 @@ void llama_kv_cache_context::set_input_pos_bucket(ggml_tensor * dst, const llama
|
||||
kv->set_input_pos_bucket(dst, ubatch);
|
||||
}
|
||||
|
||||
void llama_kv_cache_context::set_input_relative_pos(ggml_tensor * dst, const llama_ubatch * ubatch, uint32_t extent) const {
|
||||
kv->set_input_relative_pos(dst, ubatch, extent);
|
||||
void llama_kv_cache_context::set_input_relative_pos(ggml_tensor * dst, const ggml_tensor * rows, const llama_ubatch * ubatch, uint32_t extent) const {
|
||||
kv->set_input_relative_pos(dst, rows, ubatch, extent);
|
||||
}
|
||||
|
||||
void llama_kv_cache_context::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const {
|
||||
|
||||
+16
-2
@@ -153,6 +153,10 @@ public:
|
||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
// on-demand device (VRAM) residency (see llama_memory_i)
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers(bool to_host = false) override;
|
||||
|
||||
//
|
||||
// llama_kv_cache specific API
|
||||
//
|
||||
@@ -249,7 +253,7 @@ public:
|
||||
|
||||
void set_input_attn_mask (const llm_attn_mask & attn_mask, const llama_ubatch * ubatch, bool causal_attn) const;
|
||||
void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const;
|
||||
void set_input_relative_pos(ggml_tensor * dst, const llama_ubatch * ubatch, uint32_t extent) const;
|
||||
void set_input_relative_pos(ggml_tensor * dst, const ggml_tensor * rows, const llama_ubatch * ubatch, uint32_t extent) const;
|
||||
|
||||
// true if llama_kv_cell_ext holds information that has to survive a state save/restore
|
||||
bool has_cell_ext() const;
|
||||
@@ -320,6 +324,16 @@ private:
|
||||
// ggml contexts for the KV cache along with the allocated backend buffers:
|
||||
llama_memory_bufs ctxs_bufs;
|
||||
|
||||
// on-demand device eviction: host shadow of each device buffer's live contents (re-captured on
|
||||
// every release since the KV is read-write), plus the buffer type needed to reallocate it.
|
||||
struct device_buffer_shadow {
|
||||
ggml_backend_buffer_type_t buft = nullptr;
|
||||
bool releasable = false; // device (VRAM) buffer that was freed
|
||||
std::vector<uint8_t> data; // captured tensor bytes (compact, iteration order)
|
||||
};
|
||||
std::vector<device_buffer_shadow> dev_shadows; // parallel to ctxs_bufs
|
||||
bool dev_released = false;
|
||||
|
||||
// the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
|
||||
// note: this is not part of the KV state and it's only used to speed-up the find_slot() method
|
||||
std::vector<uint32_t> v_heads;
|
||||
@@ -454,7 +468,7 @@ public:
|
||||
void set_input_k_shift (ggml_tensor * dst) const;
|
||||
void set_input_attn_mask (const llm_attn_mask & attn_mask, const llama_ubatch * ubatch, bool causal_attn) const;
|
||||
void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const;
|
||||
void set_input_relative_pos(ggml_tensor * dst, const llama_ubatch * ubatch, uint32_t extent) const;
|
||||
void set_input_relative_pos(ggml_tensor * dst, const ggml_tensor * rows, const llama_ubatch * ubatch, uint32_t extent) const;
|
||||
|
||||
// see llama_kv_cache::get_prev_tokens()
|
||||
void get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const;
|
||||
|
||||
@@ -206,6 +206,18 @@ void llama_memory_hybrid::state_read(llama_io_read_i & io, llama_seq_id seq_id,
|
||||
}
|
||||
}
|
||||
|
||||
void llama_memory_hybrid::release_device_buffers(bool keep_shadow) {
|
||||
// evict both the attention KV (grows with context) and the recurrent/SSM state
|
||||
mem_attn->release_device_buffers(keep_shadow);
|
||||
mem_recr->release_device_buffers(keep_shadow);
|
||||
}
|
||||
|
||||
bool llama_memory_hybrid::restore_device_buffers(bool to_host) {
|
||||
bool ok = mem_attn->restore_device_buffers(to_host);
|
||||
ok = mem_recr->restore_device_buffers(to_host) && ok;
|
||||
return ok;
|
||||
}
|
||||
|
||||
llama_kv_cache * llama_memory_hybrid::get_mem_attn() const {
|
||||
return mem_attn.get();
|
||||
}
|
||||
|
||||
@@ -76,6 +76,9 @@ public:
|
||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers(bool to_host = false) override;
|
||||
|
||||
//
|
||||
// llama_memory_hybrid specific API
|
||||
//
|
||||
|
||||
@@ -156,7 +156,9 @@ void llama_memory_recurrent::clear(bool data) {
|
||||
|
||||
if (data) {
|
||||
for (auto & [_, buf] : ctxs_bufs) {
|
||||
ggml_backend_buffer_clear(buf.get(), 0);
|
||||
if (buf) { // may be null if evicted for on-demand VRAM sharing
|
||||
ggml_backend_buffer_clear(buf.get(), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -738,12 +740,95 @@ bool llama_memory_recurrent::is_empty() const {
|
||||
size_t llama_memory_recurrent::total_size() const {
|
||||
size_t size = 0;
|
||||
for (const auto & [_, buf] : ctxs_bufs) {
|
||||
size += ggml_backend_buffer_get_size(buf.get());
|
||||
if (buf) { // may be null if evicted for on-demand VRAM sharing
|
||||
size += ggml_backend_buffer_get_size(buf.get());
|
||||
}
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
void llama_memory_recurrent::release_device_buffers(bool keep_shadow) {
|
||||
// Same mechanism as llama_kv_cache: the recurrent (SSM/conv) state is read-write, so its host
|
||||
// shadow is (re)captured on every release. The caller must have synchronized the backend.
|
||||
if (dev_released) {
|
||||
return;
|
||||
}
|
||||
dev_shadows.assign(ctxs_bufs.bufs.size(), device_buffer_shadow{});
|
||||
size_t freed = 0;
|
||||
for (size_t i = 0; i < ctxs_bufs.bufs.size(); ++i) {
|
||||
ggml_context * ctx = ctxs_bufs.bufs[i].first.get();
|
||||
ggml_backend_buffer_t buf = ctxs_bufs.bufs[i].second.get();
|
||||
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
|
||||
continue;
|
||||
}
|
||||
auto & sh = dev_shadows[i];
|
||||
sh.releasable = true;
|
||||
sh.buft = ggml_backend_buffer_get_type(buf);
|
||||
|
||||
if (keep_shadow) {
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||
}
|
||||
sh.data.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, sh.data.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
}
|
||||
|
||||
freed += ggml_backend_buffer_get_size(buf);
|
||||
ctxs_bufs.bufs[i].second.reset();
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
t->buffer = nullptr;
|
||||
t->data = nullptr;
|
||||
}
|
||||
}
|
||||
dev_released = true;
|
||||
if (freed > 0) {
|
||||
LLAMA_LOG_INFO("%s: released %.2f MiB of recurrent state from device\n", __func__, freed / 1024.0 / 1024.0);
|
||||
}
|
||||
}
|
||||
|
||||
bool llama_memory_recurrent::restore_device_buffers(bool to_host) {
|
||||
if (!dev_released) {
|
||||
return true;
|
||||
}
|
||||
for (size_t i = 0; i < ctxs_bufs.bufs.size(); ++i) {
|
||||
auto & sh = dev_shadows[i];
|
||||
if (!sh.releasable) {
|
||||
continue;
|
||||
}
|
||||
ggml_context * ctx = ctxs_bufs.bufs[i].first.get();
|
||||
if (to_host) {
|
||||
ctxs_bufs.bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data));
|
||||
continue;
|
||||
}
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft);
|
||||
if (buf == nullptr) {
|
||||
LLAMA_LOG_ERROR("%s: failed to reallocate recurrent device buffer (out of VRAM?)\n", __func__);
|
||||
return false;
|
||||
}
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_set(t, sh.data.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
ctxs_bufs.bufs[i].second.reset(buf);
|
||||
}
|
||||
dev_released = false;
|
||||
if (!to_host) {
|
||||
dev_shadows.clear(); // to_host buffers point into the shadows
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t llama_memory_recurrent::size_r_bytes() const {
|
||||
size_t size_r_bytes = 0;
|
||||
|
||||
|
||||
@@ -67,6 +67,10 @@ public:
|
||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
// on-demand device (VRAM) residency (see llama_memory_i)
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers(bool to_host = false) override;
|
||||
|
||||
uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot())
|
||||
uint32_t size = 0; // total number of cells, shared across all sequences
|
||||
uint32_t used = 0; // used cells (i.e. at least one seq_id)
|
||||
@@ -131,6 +135,16 @@ private:
|
||||
// true if no layers - can happen if the layer filter removes all layers
|
||||
bool is_empty() const;
|
||||
|
||||
// on-demand device eviction (see release_device_buffers): host shadow of each device buffer's
|
||||
// live contents (recaptured on every release since the recurrent state is read-write)
|
||||
struct device_buffer_shadow {
|
||||
ggml_backend_buffer_type_t buft = nullptr;
|
||||
bool releasable = false;
|
||||
std::vector<uint8_t> data;
|
||||
};
|
||||
std::vector<device_buffer_shadow> dev_shadows; // parallel to ctxs_bufs
|
||||
bool dev_released = false;
|
||||
|
||||
size_t total_size() const;
|
||||
|
||||
size_t size_r_bytes() const;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "llama-ext.h"
|
||||
#include "llama-io.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <set>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
@@ -100,6 +101,9 @@ ggml_backend_buffer_t llama_memory_bufs::emplace(ggml_context_ptr ctx, ggml_back
|
||||
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_bufs::memory_breakdown() const {
|
||||
std::map<ggml_backend_buffer_type_t, size_t> ret;
|
||||
for (const auto & [ctx, buf] : bufs) {
|
||||
if (!buf) { // may be null if evicted for on-demand VRAM sharing
|
||||
continue;
|
||||
}
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
|
||||
|
||||
if (no_alloc) {
|
||||
@@ -151,3 +155,22 @@ std::vector<ggml_tensor *> llama_memory_get_tensors(const llama_memory_t mem) {
|
||||
|
||||
return std::vector<ggml_tensor *>(writer.tensors.begin(), writer.tensors.end());
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data) {
|
||||
// the buffer base must be aligned to 32 bytes; the bytes before data.data() are never accessed
|
||||
uint8_t * base = (uint8_t *) ((uintptr_t) data.data() & ~(uintptr_t) 31);
|
||||
ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr(base, data.size() + (data.data() - base));
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src == nullptr) {
|
||||
ggml_backend_tensor_alloc(buf, t, data.data() + off);
|
||||
off += ggml_nbytes(t);
|
||||
}
|
||||
}
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) {
|
||||
ggml_backend_view_init(t);
|
||||
}
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
@@ -156,6 +156,22 @@ struct llama_memory_i {
|
||||
|
||||
virtual void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const = 0;
|
||||
virtual void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) = 0;
|
||||
|
||||
//
|
||||
// on-demand device (VRAM) residency
|
||||
//
|
||||
|
||||
// Free this memory's device (VRAM) buffers to a host shadow and rebuild them on demand, to
|
||||
// hand VRAM to another model while keeping the cached data (no re-prefill). The caller must
|
||||
// have synchronized the backend first. Default: no-op (the memory stays resident).
|
||||
// keep_shadow = false frees the buffers for good (before exit): no host copy, no restore.
|
||||
// to_host = true points them at the host shadow instead, without a copy (before exit, to read a released state).
|
||||
virtual void release_device_buffers(bool /*keep_shadow*/ = true) {}
|
||||
virtual bool restore_device_buffers(bool /*to_host*/ = false) { return true; }
|
||||
};
|
||||
|
||||
// wrap a compact host shadow (tensor bytes in iteration order, views skipped) as the buffer of the tensors of ctx, without a copy
|
||||
// the shadow must outlive the buffer
|
||||
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data);
|
||||
|
||||
using llama_memory_ptr = std::unique_ptr<llama_memory_i>;
|
||||
@@ -652,6 +652,19 @@ size_t llama_file::size() const { return pimpl->size; }
|
||||
size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
|
||||
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
|
||||
|
||||
void llama_file::drop_cache(size_t offset, size_t len) const {
|
||||
#ifdef __linux__
|
||||
// only a hint: the pages are clean, a failure just leaves them cached
|
||||
const int err = posix_fadvise(file_id(), (off_t) offset, (off_t) len, POSIX_FADV_DONTNEED);
|
||||
if (err != 0) {
|
||||
LLAMA_LOG_DEBUG("%s: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", __func__, strerror(err));
|
||||
}
|
||||
#else
|
||||
GGML_UNUSED(offset);
|
||||
GGML_UNUSED(len);
|
||||
#endif
|
||||
}
|
||||
|
||||
int llama_file::file_id() const {
|
||||
#ifdef _WIN32
|
||||
return _fileno(pimpl->fp);
|
||||
|
||||
@@ -39,6 +39,9 @@ struct llama_file {
|
||||
|
||||
size_t read_alignment() const;
|
||||
bool has_direct_io() const;
|
||||
|
||||
// hint the kernel to drop the file's page cache in [offset, offset + len), len 0 = to the end (Linux only)
|
||||
void drop_cache(size_t offset = 0, size_t len = 0) const;
|
||||
private:
|
||||
struct impl;
|
||||
std::unique_ptr<impl> pimpl;
|
||||
|
||||
@@ -1925,6 +1925,11 @@ bool llama_model_loader::load_all_data(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// the tensor is copied: drop its page cache as we go, unless the file is mapped (lazy tensors)
|
||||
if (mappings.empty()) {
|
||||
file->drop_cache(weight->offs, n_size);
|
||||
}
|
||||
}
|
||||
|
||||
size_done += n_size;
|
||||
@@ -1966,6 +1971,12 @@ bool llama_model_loader::load_all_data(
|
||||
}
|
||||
}
|
||||
}
|
||||
// copied files: also drop what the per-tensor drops leave (metadata, partial pages at tensor edges)
|
||||
if (mappings.empty()) {
|
||||
for (const auto & file : files) {
|
||||
file->drop_cache();
|
||||
}
|
||||
}
|
||||
if (progress_callback) {
|
||||
// Even though the model is done loading, we still honor
|
||||
// cancellation since we need to free allocations.
|
||||
|
||||
+132
-1
@@ -1388,6 +1388,16 @@ struct llama_model::impl {
|
||||
// contexts where the model tensors metadata is stored as well as the corresponding buffers:
|
||||
std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;
|
||||
|
||||
// on-demand device (VRAM) weight residency: per-ctxs_bufs slot describing whether the
|
||||
// device weight buffer can be freed and rebuilt from a host shadow (see release/restore_device_weights)
|
||||
struct weight_release_slot {
|
||||
ggml_backend_buffer_type_t buft = nullptr; // buffer type to reallocate on restore
|
||||
bool releasable = false; // single-buffer alloc-path device (VRAM) buffer
|
||||
bool resident = true; // currently allocated on device
|
||||
std::vector<uint8_t> shadow; // host copy, captured lazily on first release
|
||||
};
|
||||
std::vector<weight_release_slot> weight_release; // parallel to ctxs_bufs
|
||||
|
||||
buft_list_t cpu_buft_list;
|
||||
std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;
|
||||
|
||||
@@ -2129,8 +2139,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
|
||||
// a lazy context is mapped whatever the load mode, but the memory-fit pass maps nothing
|
||||
const bool is_lazy_mapped = ctx_key.lazy && !ml.no_alloc;
|
||||
const bool host_ptr_path = (ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft;
|
||||
|
||||
if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
|
||||
if (host_ptr_path) {
|
||||
GGML_ASSERT(!ml.no_alloc);
|
||||
const size_t max_size = ggml_get_max_tensor_size(ctx);
|
||||
|
||||
@@ -2195,6 +2206,25 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
|
||||
pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
|
||||
|
||||
// record on-demand release metadata (parallel to ctxs_bufs). Only the single-buffer
|
||||
// alloc path on a device (non-host) buffer can be freed and rebuilt from a host shadow;
|
||||
// the host_ptr path (mmap / lazy), the no_alloc dummy buffers and host (CPU) buffers are
|
||||
// left resident.
|
||||
{
|
||||
llama_model::impl::weight_release_slot slot;
|
||||
auto & last_bufs = pimpl->ctxs_bufs.back().second;
|
||||
if (!host_ptr_path && !ml.no_alloc && last_bufs.size() == 1) {
|
||||
ggml_backend_buffer_t b = last_bufs[0].get();
|
||||
// CPU extra buffers (CPU_REPACK) are not host buffers, but they are in RAM and have no get_tensor
|
||||
ggml_backend_dev_t buft_dev = ggml_backend_buft_get_device(buft);
|
||||
if (b != nullptr && !ggml_backend_buffer_is_host(b) && (buft_dev == nullptr || ggml_backend_dev_type(buft_dev) != GGML_BACKEND_DEVICE_TYPE_CPU)) {
|
||||
slot.releasable = true;
|
||||
slot.buft = buft;
|
||||
}
|
||||
}
|
||||
pimpl->weight_release.push_back(std::move(slot));
|
||||
}
|
||||
|
||||
ctx_bufs.emplace_back(ctx, std::move(buf_ptrs));
|
||||
}
|
||||
|
||||
@@ -2223,6 +2253,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
}
|
||||
|
||||
if (ml.no_alloc) {
|
||||
post_load_tensors(ml);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -2253,6 +2284,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
|
||||
load_tensors_hadamard();
|
||||
|
||||
post_load_tensors(ml);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -2523,6 +2556,7 @@ void llama_model_base::load_tensors_hadamard() {
|
||||
std::vector<ggml_backend_buffer_ptr> buffers;
|
||||
buffers.emplace_back(std::move(buffer));
|
||||
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
|
||||
pimpl->weight_release.emplace_back(); // small, left resident
|
||||
rotations.push_back({ block_size, buft, rotation });
|
||||
it = std::prev(rotations.end());
|
||||
}
|
||||
@@ -2568,6 +2602,7 @@ void llama_model_base::load_tensors_hadamard() {
|
||||
std::vector<ggml_backend_buffer_ptr> buffers;
|
||||
buffers.emplace_back(std::move(buffer));
|
||||
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
|
||||
pimpl->weight_release.emplace_back(); // small, left resident
|
||||
st = sign_tensors.emplace(key, signs).first;
|
||||
}
|
||||
sign_tensor = st->second;
|
||||
@@ -2598,6 +2633,97 @@ ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM
|
||||
tn, ne, flags);
|
||||
}
|
||||
|
||||
void llama_model::release_device_weights(bool keep_shadow) const {
|
||||
// NOTE: the caller is responsible for synchronizing the backend scheduler first, so that no
|
||||
// compute is in flight referencing these buffers when they are freed.
|
||||
size_t freed = 0;
|
||||
for (size_t i = 0; i < pimpl->ctxs_bufs.size(); ++i) {
|
||||
auto & slot = pimpl->weight_release[i];
|
||||
if (!slot.releasable || !slot.resident) {
|
||||
continue;
|
||||
}
|
||||
ggml_context * ctx = pimpl->ctxs_bufs[i].first.get();
|
||||
ggml_backend_buffer_t buf = pimpl->ctxs_bufs[i].second[0].get();
|
||||
const size_t sz = ggml_backend_buffer_get_size(buf);
|
||||
|
||||
// lazily capture a host shadow of the weights (read-only, so captured only once).
|
||||
// Store tensor bytes compactly in stable iteration order (independent of buffer layout /
|
||||
// alignment padding) so restore does not depend on the re-allocated offsets matching.
|
||||
// View tensors alias their base, so they are skipped (restored implicitly via their base).
|
||||
if (keep_shadow && slot.shadow.empty()) {
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) continue;
|
||||
total += ggml_nbytes(t);
|
||||
}
|
||||
slot.shadow.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) continue;
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, slot.shadow.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
}
|
||||
|
||||
// free the device (VRAM) buffer and clear the now-dangling tensor pointers so that
|
||||
// ggml_backend_alloc_ctx_tensors_from_buft reallocates them cleanly on restore
|
||||
pimpl->ctxs_bufs[i].second[0].reset();
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
t->buffer = nullptr;
|
||||
t->data = nullptr;
|
||||
}
|
||||
slot.resident = false;
|
||||
freed += sz;
|
||||
}
|
||||
if (freed > 0) {
|
||||
LLAMA_LOG_INFO("%s: released %.2f MiB of device weights\n", __func__, freed / 1024.0 / 1024.0);
|
||||
}
|
||||
}
|
||||
|
||||
bool llama_model::restore_device_weights() const {
|
||||
size_t restored = 0;
|
||||
for (size_t i = 0; i < pimpl->ctxs_bufs.size(); ++i) {
|
||||
auto & slot = pimpl->weight_release[i];
|
||||
if (!slot.releasable || slot.resident) {
|
||||
continue;
|
||||
}
|
||||
ggml_context * ctx = pimpl->ctxs_bufs[i].first.get();
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, slot.buft);
|
||||
if (buf == nullptr) {
|
||||
LLAMA_LOG_ERROR("%s: failed to reallocate device weight buffer (out of VRAM?)\n", __func__);
|
||||
return false;
|
||||
}
|
||||
ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
|
||||
// re-upload weights from the compact host shadow, using the same stable iteration order
|
||||
// and view-skipping as the capture above
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) continue;
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_set(t, slot.shadow.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
pimpl->ctxs_bufs[i].second[0].reset(buf);
|
||||
slot.resident = true;
|
||||
restored += ggml_backend_buffer_get_size(buf);
|
||||
}
|
||||
if (restored > 0) {
|
||||
LLAMA_LOG_INFO("%s: restored %.2f MiB of device weights\n", __func__, restored / 1024.0 / 1024.0);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool llama_model::weights_resident() const {
|
||||
for (const auto & slot : pimpl->weight_release) {
|
||||
if (slot.releasable && !slot.resident) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string llama_model::arch_name() const {
|
||||
return llm_arch_name(arch);
|
||||
}
|
||||
@@ -2659,6 +2785,11 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_model::memory_breakdown() con
|
||||
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
|
||||
} else {
|
||||
for (const auto & buf : bufs) {
|
||||
// buf may be null if the device weights were released for on-demand VRAM sharing
|
||||
// (see release_device_weights); skip it in the breakdown
|
||||
if (!buf) {
|
||||
continue;
|
||||
}
|
||||
// GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base
|
||||
ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
|
||||
}
|
||||
|
||||
@@ -1055,6 +1055,14 @@ struct llama_model {
|
||||
// same, for the keys of the lightning indexer
|
||||
virtual llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const;
|
||||
|
||||
// on-demand device (VRAM) weight residency: free the GPU weight buffers (keeping a host
|
||||
// shadow) and rebuild them on demand. Used to time-share VRAM between models without
|
||||
// reloading or losing the KV cache. release_device_weights() requires the caller to have
|
||||
// synchronized the backend scheduler first.
|
||||
void release_device_weights(bool keep_shadow = true) const;
|
||||
bool restore_device_weights() const;
|
||||
bool weights_resident() const;
|
||||
|
||||
float get_rope_freq_base (const llama_cparams & cparams, int il) const;
|
||||
float get_rope_freq_scale(const llama_cparams & cparams, int il) const;
|
||||
|
||||
@@ -1083,6 +1091,9 @@ struct llama_model {
|
||||
// optional, runs after the scale pass
|
||||
virtual void post_arch_tensors(llama_model_loader &) {}
|
||||
|
||||
// optional, runs after the weight buffers are allocated and loaded (with no_alloc: after the dummy buffers are set)
|
||||
virtual void post_load_tensors(llama_model_loader &) {}
|
||||
|
||||
protected:
|
||||
llama_model_params params;
|
||||
|
||||
|
||||
+98
-1
@@ -369,6 +369,7 @@ struct llama_moe_cache::impl {
|
||||
static constexpr int64_t max_batch = 32;
|
||||
|
||||
int32_t n_expert_used;
|
||||
bool no_alloc;
|
||||
|
||||
stats stats_small; // up to 8 tokens per ubatch
|
||||
stats stats_large;
|
||||
@@ -400,7 +401,7 @@ struct llama_moe_cache::impl {
|
||||
ggml_context_ptr ctx_views;
|
||||
|
||||
impl(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size, float cpu_share) :
|
||||
n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()), cpu_share(std::clamp(cpu_share, 0.0f, 1.0f)) {
|
||||
n_expert_used(model.hparams.n_expert_used_max()), no_alloc(no_alloc), layers(model.layers.size()), cpu_share(std::clamp(cpu_share, 0.0f, 1.0f)) {
|
||||
for (size_t i = 0; i < backends.size(); ++i) {
|
||||
const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i]));
|
||||
if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
|
||||
@@ -861,6 +862,90 @@ struct llama_moe_cache::impl {
|
||||
return true;
|
||||
}
|
||||
|
||||
// on-demand VRAM: free the banks and forget the cached experts, the slot maps all read -1 again
|
||||
void release_device() {
|
||||
if (no_alloc) {
|
||||
return;
|
||||
}
|
||||
size_t freed = 0;
|
||||
for (device & d : devices) {
|
||||
if (!d.buf) {
|
||||
continue;
|
||||
}
|
||||
freed += ggml_backend_buffer_get_size(d.buf.get());
|
||||
d.buf.reset();
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) {
|
||||
t->buffer = nullptr;
|
||||
t->data = nullptr;
|
||||
}
|
||||
}
|
||||
invalidate();
|
||||
if (freed > 0) {
|
||||
LLAMA_LOG_INFO("%s: released %.2f MiB of MoE cache\n", __func__, freed/1024.0/1024.0);
|
||||
}
|
||||
}
|
||||
|
||||
// the banks come back empty, so the slot maps stay invalidated
|
||||
bool restore_device() {
|
||||
if (no_alloc) {
|
||||
return true;
|
||||
}
|
||||
size_t restored = 0;
|
||||
for (device & d : devices) {
|
||||
if (!d.ctx || d.buf) {
|
||||
continue;
|
||||
}
|
||||
// also re-inits the views of the banks
|
||||
d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft));
|
||||
if (!d.buf) {
|
||||
LLAMA_LOG_ERROR("%s: failed to reallocate the MoE cache buffer (out of VRAM?)\n", __func__);
|
||||
return false;
|
||||
}
|
||||
ggml_backend_buffer_clear(d.buf.get(), 0);
|
||||
ggml_backend_buffer_set_usage(d.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
d.buf_size = ggml_backend_buffer_get_size(d.buf.get());
|
||||
restored += d.buf_size;
|
||||
}
|
||||
if (restored > 0) {
|
||||
LLAMA_LOG_INFO("%s: restored %.2f MiB of MoE cache\n", __func__, restored/1024.0/1024.0);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool resident() const {
|
||||
for (const device & d : devices) {
|
||||
if (d.ctx && !d.buf) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// empty every LRU and mark all experts as not cached
|
||||
void invalidate() {
|
||||
if (buf_host) {
|
||||
ggml_backend_buffer_clear(buf_host.get(), 0xff); // the CPU maps and the unused slot maps are -1
|
||||
}
|
||||
for (group & g : groups) {
|
||||
if (g.n_slots == 0) {
|
||||
continue;
|
||||
}
|
||||
// start from a fresh LRU: init() does not reset all of its state (n_filled, the scratch vectors)
|
||||
g.lru = moe_cache_lru();
|
||||
g.lru.init((int32_t) layers.size(), (int32_t) g.ref[0]->ne[2], g.n_slots);
|
||||
for (int32_t il : g.layers) {
|
||||
layer & l = layers[il];
|
||||
if (l.slot_map != nullptr) {
|
||||
int32_t * slots = (int32_t *) l.slot_map->data;
|
||||
std::fill(slots, slots + l.slot_map->ne[1], g.lru.empty); // not cached
|
||||
g.lru.slot_map[il] = slots;
|
||||
}
|
||||
l.cpu_experts.clear();
|
||||
l.cpu_acc = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void log_stats() const {
|
||||
auto log = [](const char * name, const stats & st) {
|
||||
const size_t n = st.hits + st.misses;
|
||||
@@ -970,6 +1055,18 @@ int64_t llama_moe_cache::copy_experts(ggml_backend_t backend, const ggml_tensor
|
||||
return pimpl->copy_experts(backend, w, dst, e, last);
|
||||
}
|
||||
|
||||
void llama_moe_cache::release_device() {
|
||||
pimpl->release_device();
|
||||
}
|
||||
|
||||
bool llama_moe_cache::restore_device() {
|
||||
return pimpl->restore_device();
|
||||
}
|
||||
|
||||
bool llama_moe_cache::resident() const {
|
||||
return pimpl->resident();
|
||||
}
|
||||
|
||||
std::map<ggml_backend_buffer_type_t, size_t> llama_moe_cache::memory_breakdown() const {
|
||||
std::map<ggml_backend_buffer_type_t, size_t> res;
|
||||
for (const auto & d : pimpl->devices) {
|
||||
|
||||
@@ -60,6 +60,12 @@ public:
|
||||
// returns the number of experts copied, 0 if expert e is not in the cache
|
||||
int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last);
|
||||
|
||||
// on-demand VRAM: free the device banks and empty the cache, the caller must synchronize first
|
||||
void release_device();
|
||||
// reallocate the banks empty, returns false if out of memory
|
||||
bool restore_device();
|
||||
bool resident() const;
|
||||
|
||||
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
|
||||
|
||||
private:
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <stdexcept>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
int llama_model_inkling2_loop::layer_phys(int il) const {
|
||||
return il < n_phys ? il : core_start + (il - n_phys) % (core_end - core_start);
|
||||
}
|
||||
@@ -69,6 +76,101 @@ void llama_model_inkling2_loop::post_arch_tensors(llama_model_loader &) {
|
||||
}
|
||||
}
|
||||
|
||||
// the scheduler makes one device copy of a host weight per graph, alive from its first to its last use, and the expert copy
|
||||
// callback runs at the first use only: give each later pass its own view, so it gets its own copy with the experts of its own ids
|
||||
// knob: LLAMA_LOOP_HOST_VIEWS=0 keeps the shared tensors
|
||||
void llama_model_inkling2_loop::post_load_tensors(llama_model_loader & ml) {
|
||||
const char * env = getenv("LLAMA_LOOP_HOST_VIEWS");
|
||||
if (env != nullptr && strcmp(env, "0") == 0) {
|
||||
LLAMA_LOG_INFO("%s: LLAMA_LOOP_HOST_VIEWS=0, the loop passes share the host weights\n", __func__);
|
||||
return;
|
||||
}
|
||||
|
||||
// llama_layer holds only tensor pointers, see llama_context::opt_init
|
||||
static_assert(sizeof(llama_layer) % sizeof(ggml_tensor *) == 0, "llama_layer must hold only tensor pointers");
|
||||
const size_t n_ptr = sizeof(llama_layer)/sizeof(ggml_tensor *);
|
||||
|
||||
const int n_slot = (int) hparams.n_layer();
|
||||
const int n_core = (int) (core_end - core_start);
|
||||
|
||||
// weights on a device are used in place, only host weights are copied
|
||||
std::vector<std::vector<ggml_tensor *>> host(n_slot);
|
||||
size_t n_views = 0;
|
||||
for (int il = n_phys; il < n_slot; ++il) {
|
||||
ggml_tensor ** ts = reinterpret_cast<ggml_tensor **>(&layers[il]);
|
||||
for (size_t i = 0; i < n_ptr; ++i) {
|
||||
ggml_tensor * t = ts[i];
|
||||
if (t == nullptr || t->buffer == nullptr || !ggml_backend_buffer_is_host(t->buffer) || (!ml.no_alloc && t->data == nullptr)) {
|
||||
continue;
|
||||
}
|
||||
if (std::find(host[il].begin(), host[il].end(), t) == host[il].end()) {
|
||||
host[il].push_back(t);
|
||||
}
|
||||
}
|
||||
n_views += host[il].size();
|
||||
}
|
||||
if (n_views == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
ggml_init_params init_params = {
|
||||
/*.mem_size =*/ n_views*ggml_tensor_overhead(),
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ctx_visits.reset(ggml_init(init_params));
|
||||
if (!ctx_visits) {
|
||||
throw std::runtime_error("inkling2_loop: failed to create the context of the loop views");
|
||||
}
|
||||
|
||||
size_t n_bytes = 0;
|
||||
for (int il = n_phys; il < n_slot; ++il) {
|
||||
const int pass = 1 + (il - n_phys)/n_core;
|
||||
|
||||
std::unordered_map<const ggml_tensor *, ggml_tensor *> view_of;
|
||||
for (ggml_tensor * t : host[il]) {
|
||||
ggml_tensor * v = ggml_view_tensor(ctx_visits.get(), t);
|
||||
ggml_format_name(v, "%s.visit%d", t->name, pass);
|
||||
if (ml.no_alloc) {
|
||||
// the dummy buffer has no data, it only tells the scheduler that this is a weight
|
||||
v->buffer = t->buffer;
|
||||
} else if (ggml_backend_view_init(v) != GGML_STATUS_SUCCESS) {
|
||||
throw std::runtime_error(format("inkling2_loop: failed to initialize the view %s", v->name));
|
||||
}
|
||||
view_of[t] = v;
|
||||
n_bytes += ggml_nbytes(t);
|
||||
|
||||
// the per-weight maps are keyed by the tensor
|
||||
if (const auto it = w_scales.find(t); it != w_scales.end()) {
|
||||
ggml_tensor * s = it->second;
|
||||
w_scales.emplace(v, s);
|
||||
}
|
||||
if (const auto it = prec_policy.prec_acc.find(t); it != prec_policy.prec_acc.end()) {
|
||||
const ggml_prec prec = it->second;
|
||||
prec_policy.prec_acc.emplace(v, prec);
|
||||
}
|
||||
if (const auto it = prec_policy.prec_src1.find(t); it != prec_policy.prec_src1.end()) {
|
||||
const ggml_prec prec = it->second;
|
||||
prec_policy.prec_src1.emplace(v, prec);
|
||||
}
|
||||
if (const auto it = hdmd.rot.find(t); it != hdmd.rot.end()) {
|
||||
const llama_hadamard_transform tr = it->second;
|
||||
hdmd.rot.emplace(v, tr);
|
||||
}
|
||||
}
|
||||
|
||||
ggml_tensor ** ts = reinterpret_cast<ggml_tensor **>(&layers[il]);
|
||||
for (size_t i = 0; i < n_ptr; ++i) {
|
||||
if (const auto it = view_of.find(ts[i]); it != view_of.end()) {
|
||||
ts[i] = it->second;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LLAMA_LOG_INFO("%s: %zu views of host weights for the loop passes after the first (%.2f MiB, not copied)\n",
|
||||
__func__, n_views, n_bytes/1024.0/1024.0);
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_inkling2_loop::build_arch_graph(const llm_graph_params & params) const {
|
||||
return std::make_unique<graph>(params);
|
||||
}
|
||||
|
||||
+1
-14
@@ -250,7 +250,6 @@ ggml_tensor * llama_model_inkling2::graph::control(ggml_tensor * cur, ggml_tenso
|
||||
|
||||
ggml_tensor * llama_model_inkling2::graph::build_inputs() {
|
||||
inp = build_inp_mem_hybrid_iswa_rel(hparams.n_relative_extent, hparams.n_swa);
|
||||
attn = inp->get_attn();
|
||||
recr = inp->get_recr();
|
||||
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
@@ -347,30 +346,18 @@ ggml_tensor * llama_model_inkling2::graph::build_layer(int il, int ip, ggml_tens
|
||||
cb(Vcur, "Vcur", il);
|
||||
cb(Rcur, "Rcur", il);
|
||||
|
||||
ggml_tensor * rel_pos = is_swa ? inp->self_rel_pos_swa : inp->self_rel_pos;
|
||||
|
||||
const int64_t n_stream = (is_swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw)->ne[3];
|
||||
const int64_t n_kv = rel_pos->ne[0];
|
||||
|
||||
ggml_tensor * sinks = build_lora_mm(layer.attn_sink_proj, inputs[0]);
|
||||
|
||||
// project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x n_head
|
||||
ggml_tensor * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, layer.attn_rel_bias, 0, 1, 0, 0), Rcur);
|
||||
|
||||
// the sink logit t is per query: exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))), with a zero sink
|
||||
table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, n_head, n_tokens));
|
||||
table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
|
||||
|
||||
ggml_tensor * kq_b = ggml_get_rows(ctx0, table, rel_pos);
|
||||
kq_b = ggml_reshape_4d(ctx0, kq_b, n_head, n_kv, n_tokens/n_stream, n_stream);
|
||||
kq_b = ggml_cont(ctx0, ggml_permute(ctx0, kq_b, 2, 0, 1, 3));
|
||||
cb(kq_b, "kq_b", il);
|
||||
|
||||
sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head), 0.0f);
|
||||
|
||||
cur = build_attn(inp,
|
||||
nullptr, nullptr, nullptr,
|
||||
Qcur, Kcur, Vcur, kq_b, sinks, 1.0f/sqrtf(n_embd_head), il);
|
||||
Qcur, Kcur, Vcur, table, sinks, 1.0f/sqrtf(n_embd_head), il);
|
||||
|
||||
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, inputs[0])));
|
||||
cur = convolve(build_lora_mm(layer.wo, cur), layer.attn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa);
|
||||
|
||||
+4
-1
@@ -2544,7 +2544,6 @@ struct llama_model_inkling2 : public llama_model_base {
|
||||
void build_output(ggml_tensor * cur, const std::vector<ggml_tensor *> & blocks);
|
||||
|
||||
llm_graph_input_mem_hybrid_iswa_rel * inp = nullptr;
|
||||
llm_graph_input_attn_kv_iswa * attn = nullptr;
|
||||
llm_graph_input_rs * recr = nullptr;
|
||||
|
||||
ggml_tensor * inp_out_ids = nullptr;
|
||||
@@ -2560,6 +2559,7 @@ struct llama_model_inkling2_loop : public llama_model_inkling2 {
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
void load_arch_tensors(llama_model_loader & ml) override;
|
||||
void post_arch_tensors(llama_model_loader & ml) override;
|
||||
void post_load_tensors(llama_model_loader & ml) override;
|
||||
|
||||
int n_layer_phys() const override { return n_phys; }
|
||||
int layer_phys(int il) const;
|
||||
@@ -2574,6 +2574,9 @@ struct llama_model_inkling2_loop : public llama_model_inkling2 {
|
||||
// indexed by physical layer - core_start
|
||||
std::vector<loop_layer> loop_layers;
|
||||
|
||||
// views of the host weights for the cache slots of the later passes, see post_load_tensors
|
||||
ggml_context_ptr ctx_visits;
|
||||
|
||||
struct graph : public llama_model_inkling2::graph {
|
||||
graph(const llm_graph_params & params);
|
||||
};
|
||||
|
||||
@@ -367,6 +367,7 @@ endif()
|
||||
llama_build(test-backend-ops.cpp LINK ggml Threads::Threads)
|
||||
|
||||
llama_build_and_test(test-backend-tensor.cpp LINK ggml)
|
||||
llama_build_and_test(test-flash-attn-indexed-bias.cpp LINK ggml)
|
||||
llama_build_and_test(test-sleep.cpp LINK ggml)
|
||||
llama_build_and_test(test-backend-sched.cpp LINK ggml)
|
||||
|
||||
|
||||
@@ -11758,6 +11758,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test
|
||||
test_cases.emplace_back(new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f));
|
||||
test_cases.emplace_back(new test_silu_back());
|
||||
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <vector>
|
||||
|
||||
static void set_kv(ggml_tensor * t, const std::vector<float> & data) {
|
||||
std::vector<uint8_t> bytes(ggml_nbytes(t));
|
||||
ggml_quantize_chunk(t->type, data.data(), bytes.data(), 0, ggml_nrows(t), t->ne[0], nullptr);
|
||||
ggml_backend_tensor_set(t, bytes.data(), 0, bytes.size());
|
||||
}
|
||||
|
||||
static void test_indexed_bias(ggml_backend_t backend, int width, bool use_rows, int n_query, int cells = 768, ggml_type kv_type = GGML_TYPE_F16) {
|
||||
constexpr int heads = 6, kv_heads = 2, streams = 2, bins = 17;
|
||||
const int cols = use_rows ? std::min(512, cells) : cells;
|
||||
|
||||
ggml_context * ctx = ggml_init({32*ggml_tensor_overhead() + ggml_graph_overhead(), nullptr, true});
|
||||
auto * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, width, n_query, heads, streams);
|
||||
auto * k = ggml_new_tensor_4d(ctx, kv_type, width, cells, kv_heads, use_rows ? 1 : streams);
|
||||
auto * v = ggml_dup_tensor(ctx, k);
|
||||
auto * mask = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, cols, n_query, 1, streams);
|
||||
auto * rows = use_rows ? ggml_new_tensor_2d(ctx, GGML_TYPE_I32, cols, streams) : nullptr;
|
||||
auto * table = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, bins, heads, n_query*streams);
|
||||
auto * indices = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, cols, n_query*streams);
|
||||
auto * sinks = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, heads);
|
||||
auto * out = ggml_flash_attn_ext_rows(ctx, q, k, v, mask, rows, 1.0f/sqrtf(width), 0, 0);
|
||||
ggml_flash_attn_ext_add_sinks(out, sinks);
|
||||
const bool supports_plain = ggml_backend_supports_op(backend, out);
|
||||
ggml_flash_attn_ext_add_indexed_bias(out, table, indices);
|
||||
GGML_ASSERT(ggml_flash_attn_ext_has_indexed_bias(out));
|
||||
ggml_flash_attn_ext_add_indexed_bias(out, nullptr, nullptr);
|
||||
GGML_ASSERT(!ggml_flash_attn_ext_has_indexed_bias(out) && !out->src[6] && !out->src[7]);
|
||||
GGML_ASSERT(ggml_backend_supports_op(backend, out) == supports_plain);
|
||||
ggml_flash_attn_ext_add_indexed_bias(out, table, indices);
|
||||
ggml_prec_set_acc(out, GGML_PREC_F32);
|
||||
|
||||
if (!ggml_backend_supports_op(backend, out)) {
|
||||
ggml_free(ctx);
|
||||
return;
|
||||
}
|
||||
|
||||
auto * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, out);
|
||||
auto * buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
|
||||
GGML_ASSERT(buf);
|
||||
ggml_backend_buffer_clear(buf, 0);
|
||||
|
||||
std::vector<int32_t> row(cols*streams), idx(ggml_nelements(indices));
|
||||
std::vector<ggml_fp16_t> visibility(idx.size());
|
||||
std::vector<float> keys(ggml_nelements(k)), values(ggml_nelements(v));
|
||||
std::vector<float> queries(ggml_nelements(q));
|
||||
float sink_logits[heads];
|
||||
for (int h = 0; h < heads; ++h) {
|
||||
sink_logits[h] = float(2*h - 3);
|
||||
}
|
||||
for (size_t i = 0; i < queries.size(); i += width) {
|
||||
queries[i] = 4.0f;
|
||||
}
|
||||
std::vector<float> biases(ggml_nelements(table)), expected(n_query*streams*heads), output(ggml_nelements(out));
|
||||
for (int s = 0; s < v->ne[3]; ++s) {
|
||||
for (int h = 0; h < kv_heads; ++h) {
|
||||
for (int c = 0; c < cells; ++c) {
|
||||
const float x = float((c + 3*h + 5*s) % 23 - 11)/16;
|
||||
const int offset = ((s*kv_heads + h)*cells + c)*width;
|
||||
std::fill_n(values.data() + offset, width, x);
|
||||
keys[offset] = float((7*c + 2*h + 3*s) % 29 - 14)/4;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int s = 0; s < streams; ++s) {
|
||||
for (int c = 0; c < cols; ++c) {
|
||||
row[s*cols + c] = use_rows ? (c % 19 == 0 ? -1 : (13*c + 73*s) % cells) : c;
|
||||
}
|
||||
for (int j = 0; j < n_query; ++j) {
|
||||
const int query = s*n_query + j;
|
||||
for (int h = 0; h < heads; ++h) {
|
||||
for (int b = 0; b < bins; ++b) {
|
||||
biases[(query*heads + h)*bins + b] = float((b*(3 + h) + query) % 13 - 6)/2.9f;
|
||||
}
|
||||
double sum = exp(double(sink_logits[h])), weighted = 0.0;
|
||||
for (int c = 0; c < cols; ++c) {
|
||||
const int cell = row[s*cols + c];
|
||||
const int i = query*cols + c;
|
||||
idx[i] = (c + 3*query) % bins;
|
||||
const bool visible = (s != 1 || j != 0) && cell >= 0 && (c + j) % 7 != 0;
|
||||
visibility[i] = ggml_fp32_to_fp16(visible ? 0.25f : -INFINITY);
|
||||
if (visible) {
|
||||
const int vk = (use_rows ? 0 : s)*kv_heads + h/(heads/kv_heads);
|
||||
const double qk = 4.0*keys[(vk*cells + cell)*width]/sqrt(width);
|
||||
const float m = 0.25f + biases[(query*heads + h)*bins + idx[i]];
|
||||
const double weight = exp(qk + ggml_fp16_to_fp32(ggml_fp32_to_fp16(m)));
|
||||
weighted += weight*values[(vk*cells + cell)*width];
|
||||
sum += weight;
|
||||
}
|
||||
}
|
||||
expected[query*heads + h] = weighted/sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
ggml_backend_tensor_set(q, queries.data(), 0, ggml_nbytes(q));
|
||||
set_kv(k, keys);
|
||||
set_kv(v, values);
|
||||
ggml_backend_tensor_set(mask, visibility.data(), 0, ggml_nbytes(mask));
|
||||
ggml_backend_tensor_set(table, biases.data(), 0, ggml_nbytes(table));
|
||||
ggml_backend_tensor_set(sinks, sink_logits, 0, sizeof(sink_logits));
|
||||
ggml_backend_tensor_set(indices, idx.data(), 0, ggml_nbytes(indices));
|
||||
if (rows) {
|
||||
ggml_backend_tensor_set(rows, row.data(), 0, ggml_nbytes(rows));
|
||||
}
|
||||
GGML_ASSERT(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS);
|
||||
ggml_backend_tensor_get(out, output.data(), 0, ggml_nbytes(out));
|
||||
float max_error = 0.0f;
|
||||
for (size_t i = 0; i < output.size(); ++i) {
|
||||
GGML_ASSERT(std::isfinite(output[i]));
|
||||
max_error = std::max(max_error, fabsf(output[i] - expected[i/width]));
|
||||
}
|
||||
printf("%s: width=%d, kv=%s, rows=%d, queries=%d, cells=%d, max error=%g\n", ggml_backend_name(backend), width, ggml_type_name(kv_type), use_rows, n_query, cells, max_error);
|
||||
GGML_ASSERT(max_error < 0.001f);
|
||||
ggml_backend_buffer_free(buf);
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
int main() {
|
||||
ggml_backend_load_all();
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
||||
auto dev = ggml_backend_dev_get(i);
|
||||
auto backend = ggml_backend_dev_init(dev, nullptr);
|
||||
GGML_ASSERT(backend);
|
||||
for (int width : {64, 128}) {
|
||||
test_indexed_bias(backend, width, false, 7, 16);
|
||||
test_indexed_bias(backend, width, true, 7, 16);
|
||||
test_indexed_bias(backend, width, false, 1, 768, GGML_TYPE_Q4_0);
|
||||
test_indexed_bias(backend, width, true, 1, 768, GGML_TYPE_Q4_0);
|
||||
for (int n_query : {1, 7, 65}) {
|
||||
test_indexed_bias(backend, width, false, n_query);
|
||||
test_indexed_bias(backend, width, true, n_query);
|
||||
}
|
||||
}
|
||||
ggml_backend_free(backend);
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,11 @@
|
||||
#include <functional>
|
||||
#include <float.h>
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
//
|
||||
// mtmd_model_loader
|
||||
//
|
||||
@@ -4445,6 +4450,14 @@ struct mtmd_model_loader {
|
||||
}
|
||||
}
|
||||
LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
|
||||
#if defined(__linux__)
|
||||
// the weights are copied into our buffer: drop the file's page cache (only a hint)
|
||||
const int fd = ::open(fname.c_str(), O_RDONLY);
|
||||
if (fd != -1) {
|
||||
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
|
||||
::close(fd);
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
|
||||
}
|
||||
@@ -4617,6 +4630,7 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params
|
||||
auto model = std::make_unique<mtmd_model>();
|
||||
model->hparams.custom_image_min_tokens = params.image_min_tokens;
|
||||
model->hparams.custom_image_max_tokens = params.image_max_tokens;
|
||||
model->no_alloc = params.no_alloc;
|
||||
loader.load_hparams(*model, modality);
|
||||
loader.load_tensors(*model, buft, params.no_alloc);
|
||||
const std::string & def = model->hparams.gen_default_voice;
|
||||
@@ -4645,6 +4659,66 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params
|
||||
return res;
|
||||
}
|
||||
|
||||
void mtmd_model_release_device(mtmd_model & model) {
|
||||
if (model.dev_released || model.no_alloc) {
|
||||
return;
|
||||
}
|
||||
ggml_backend_buffer_t buf = model.buf.get();
|
||||
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
|
||||
return; // CPU-backed encoder: nothing in VRAM to free
|
||||
}
|
||||
model.dev_buft = ggml_backend_buffer_get_type(buf);
|
||||
|
||||
ggml_context * cd = model.ctx_data.get();
|
||||
// capture the host shadow once (weights are read-only): compact, stable iteration order,
|
||||
// skipping view tensors (which alias a base and are restored implicitly)
|
||||
if (model.dev_shadow.empty()) {
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||
}
|
||||
model.dev_shadow.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, model.dev_shadow.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
}
|
||||
|
||||
// free the device buffer and clear the now-dangling tensor pointers so restore reallocates cleanly
|
||||
model.buf.reset();
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||
t->buffer = nullptr;
|
||||
t->data = nullptr;
|
||||
}
|
||||
model.dev_released = true;
|
||||
}
|
||||
|
||||
bool mtmd_model_restore_device(mtmd_model & model) {
|
||||
if (!model.dev_released) {
|
||||
return true;
|
||||
}
|
||||
ggml_context * cd = model.ctx_data.get();
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(cd, model.dev_buft);
|
||||
if (buf == nullptr) {
|
||||
LOG_ERR("%s: failed to reallocate encoder device buffer (out of VRAM?)\n", __func__);
|
||||
return false;
|
||||
}
|
||||
ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_set(t, model.dev_shadow.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
model.buf.reset(buf);
|
||||
model.dev_released = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
mtmd_model_caps mtmd_model_get_caps(const char * fname) {
|
||||
mtmd_model_caps res;
|
||||
mtmd_model_loader loader(fname, /* skip_tensors= */ true);
|
||||
|
||||
@@ -1161,6 +1161,14 @@ struct mtmd_model {
|
||||
ggml_backend_buffer_ptr buf;
|
||||
std::map<ggml_backend_dev_t, size_t> mem_weights;
|
||||
|
||||
// on-demand device (VRAM) residency: the weights are read-only, so a host shadow is captured once and
|
||||
// the device buffer can be freed while the model is cold, then rebuilt on wake
|
||||
// (mirrors llama_model::release_device_weights), see mtmd_model_release_device()
|
||||
bool no_alloc = false;
|
||||
ggml_backend_buffer_type_t dev_buft = nullptr;
|
||||
std::vector<uint8_t> dev_shadow;
|
||||
bool dev_released = false;
|
||||
|
||||
// embeddings
|
||||
ggml_tensor * class_embedding = nullptr;
|
||||
ggml_tensor * class_position_embedding = nullptr;
|
||||
@@ -1614,6 +1622,11 @@ struct mtmd_models {
|
||||
// the weights are allocated in buft, throws on error
|
||||
mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft);
|
||||
|
||||
// free the device weight buffer to a host shadow and rebuild it on demand, a no-op for a CPU-backed model
|
||||
// the caller must make sure that no graph using the weights is in flight
|
||||
void mtmd_model_release_device(mtmd_model & model);
|
||||
bool mtmd_model_restore_device(mtmd_model & model); // false if the reallocation failed (out of VRAM)
|
||||
|
||||
struct mtmd_model_caps {
|
||||
bool has_vision;
|
||||
bool has_audio;
|
||||
|
||||
@@ -1671,6 +1671,32 @@ std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::
|
||||
return output;
|
||||
}
|
||||
|
||||
void mtmd_release_device(mtmd_context * ctx) {
|
||||
if (ctx == nullptr) {
|
||||
return;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(ctx->compute_mutex);
|
||||
// ensure no encode is in flight before freeing the weights
|
||||
if (ctx->backends && ctx->backends->sched) {
|
||||
ggml_backend_sched_synchronize(ctx->backends->sched.get());
|
||||
}
|
||||
for (auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) {
|
||||
if (m) { mtmd_model_release_device(*m); }
|
||||
}
|
||||
}
|
||||
|
||||
bool mtmd_restore_device(mtmd_context * ctx) {
|
||||
if (ctx == nullptr) {
|
||||
return true;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(ctx->compute_mutex);
|
||||
bool ok = true;
|
||||
for (auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) {
|
||||
if (m) { ok = mtmd_model_restore_device(*m) && ok; }
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
struct mtmd_tokenizer {
|
||||
const mtmd_context * ctx;
|
||||
|
||||
|
||||
@@ -147,6 +147,12 @@ MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname,
|
||||
|
||||
MTMD_API void mtmd_free(mtmd_context * ctx);
|
||||
|
||||
// on-demand device (VRAM) residency: free / rebuild the vision+audio encoder weight buffers so an
|
||||
// idle (cold) multimodal model releases its encoder VRAM and reclaims it on wake. No-op for a
|
||||
// CPU-backed encoder. Restore returns false if reallocation failed (out of VRAM).
|
||||
MTMD_API void mtmd_release_device(mtmd_context * ctx);
|
||||
MTMD_API bool mtmd_restore_device(mtmd_context * ctx);
|
||||
|
||||
// whether we need to set non-causal mask before llama_decode
|
||||
// if chunk is nullptr, we assume the default case where chunk is an image chunk
|
||||
MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);
|
||||
|
||||
@@ -32,8 +32,11 @@
|
||||
#include <memory>
|
||||
#include <filesystem>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <utility>
|
||||
#include <fstream>
|
||||
#include <thread>
|
||||
#include <atomic>
|
||||
|
||||
// fix problem with std::min and std::max
|
||||
#if defined(_WIN32)
|
||||
@@ -44,6 +47,16 @@
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
// POSIX file locking for the cross-process VRAM arbiter (see vram_share_* below)
|
||||
#if !defined(_WIN32)
|
||||
#include <sys/file.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <poll.h>
|
||||
#include <cerrno>
|
||||
#endif
|
||||
|
||||
constexpr int HTTP_POLLING_SECONDS = 1;
|
||||
|
||||
static common_speculative_output_limits server_output_limits(const common_params & params) {
|
||||
@@ -912,7 +925,36 @@ static int encode_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch,
|
||||
// TODO @ngxson : move this log line to debug when it become more stable
|
||||
SLT_TRC(slot, "encoding mtmd batch from idx = %zu, n_chunks = %d\n", idx, n_added);
|
||||
|
||||
// Bring the vision/audio encoder into VRAM just for this encode. In on-demand mode the
|
||||
// encoder is normally kept in RAM (its VRAM freed for KV / expert cache). If it does not
|
||||
// fit, evict the LLM backbone weights first: they are NOT needed while the encoder runs (the
|
||||
// decode that uses them happens afterwards) and their host shadow is read-only, so this is a
|
||||
// cheap free (no D2H) and leaves the KV cache / prompt cache untouched.
|
||||
static const bool mmproj_ondemand = getenv("LLAMA_MMPROJ_ONDEMAND") != nullptr;
|
||||
bool weights_evicted = false;
|
||||
if (mctx && !mtmd_restore_device(mctx)) {
|
||||
if (mmproj_ondemand) {
|
||||
llama_context_release_device(slot.ctx_tgt, /* evict_kv = */ false); // free backbone, keep KV
|
||||
weights_evicted = true;
|
||||
}
|
||||
if (!mtmd_restore_device(mctx)) {
|
||||
if (weights_evicted) { llama_context_restore_device(slot.ctx_tgt); }
|
||||
SLT_ERR(slot, "%s", "failed to bring the multimodal encoder into VRAM for encoding\n");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
res = mtmd_batch_encode(mbatch.get());
|
||||
|
||||
// release the encoder VRAM again (on-demand), then restore the backbone weights for decode.
|
||||
// Order matters: free the encoder BEFORE re-uploading the weights so the peak stays within VRAM.
|
||||
if (mctx && mmproj_ondemand) {
|
||||
mtmd_release_device(mctx);
|
||||
}
|
||||
if (weights_evicted) {
|
||||
llama_context_restore_device(slot.ctx_tgt);
|
||||
}
|
||||
|
||||
if (res != 0) {
|
||||
SLT_ERR(slot, "failed to encode mtmd batch for chunk idx = %zu, res = %d\n", idx, res);
|
||||
return -1;
|
||||
@@ -1028,6 +1070,15 @@ static bool add_mtmd_chunk(const server_slot & slot, const mtmd_input_chunk * ch
|
||||
return true;
|
||||
}
|
||||
|
||||
// smaller prompts are not saved, and a smaller gain is not worth a file read
|
||||
static constexpr size_t DISK_CACHE_MIN_TOKENS = 256;
|
||||
|
||||
// the token list of a text-only prompt, false if it holds media (get_tokens() asserts when an mmproj is loaded)
|
||||
static bool disk_cache_text(const server_tokens & tokens, llama_tokens & out) {
|
||||
out = tokens.get_text_tokens();
|
||||
return out.size() == tokens.size();
|
||||
}
|
||||
|
||||
//
|
||||
// server_context_impl (private implementation)
|
||||
//
|
||||
@@ -1078,6 +1129,8 @@ public:
|
||||
// we don't call it again here to avoid double free
|
||||
destroy();
|
||||
}
|
||||
// only once our VRAM is freed (normally vram_share_exit() has done both already)
|
||||
vram_share_shutdown();
|
||||
}
|
||||
|
||||
server_metrics get_metrics() const {
|
||||
@@ -1104,6 +1157,429 @@ private:
|
||||
llama_model * model_dft = nullptr;
|
||||
llama_context * ctx_dft = nullptr;
|
||||
|
||||
// Cross-process VRAM arbiter: when LLAMA_SLEEP_VRAM_ONLY is set, several always-loaded
|
||||
// llama-server processes time-share one GPU. A single flock() on <arena>/token.lock is the
|
||||
// baton: "resident (device buffers in VRAM) iff I hold the token". A model warms up (locks +
|
||||
// restores) when it wakes for a request, and goes cold (releases + unlocks) when it is idle and
|
||||
// another process waits for the token, so only one model holds VRAM at a time. The process, its
|
||||
// context and its prompt cache stay alive, so nothing is re-prefilled. Going cold releases the
|
||||
// weights; with LLAMA_SLEEP_EVICT_KV it also evicts the KV/recurrent state to a host shadow and
|
||||
// frees the scheduler and backend scratch. All of it is restored bit-exact on wake.
|
||||
bool vram_only = false; // release-mode sleep active (LLAMA_SLEEP_VRAM_ONLY)
|
||||
bool vram_evict_kv = false; // also evict the KV cache to host on release (LLAMA_SLEEP_EVICT_KV)
|
||||
bool vram_flock = false; // cross-process flock coordination available
|
||||
std::atomic<bool> vram_cold{true}; // weights currently released
|
||||
int vram_lock_fd = -1; // fd for <arena>/token.lock
|
||||
|
||||
// "want" lock: a process that waits for the token holds <arena>/want.lock shared, and drops it
|
||||
// as soon as it owns the token. The holder probes that lock with a non-blocking exclusive
|
||||
// flock, which fails only while somebody waits. This is level-triggered state owned by the
|
||||
// kernel, so there is no event to miss and nothing goes stale if a waiter dies.
|
||||
// The probe needs its own fd: flock() treats two open file descriptions of one file
|
||||
// independently, so probing on the waiter fd would convert our own lock instead of conflicting.
|
||||
int vram_want_fd = -1; // waiter side, LOCK_SH while waiting
|
||||
int vram_want_probe_fd = -1; // probe side, LOCK_EX | LOCK_NB only
|
||||
bool vram_want_held = false; // only touched by the waiting thread
|
||||
|
||||
// open the shared arena (token.lock + want.lock). Idempotent; sets vram_only/vram_flock.
|
||||
void vram_arena_open() {
|
||||
if (getenv("LLAMA_SLEEP_VRAM_ONLY") == nullptr) {
|
||||
return;
|
||||
}
|
||||
vram_only = true;
|
||||
vram_evict_kv = getenv("LLAMA_SLEEP_EVICT_KV") != nullptr;
|
||||
#if !defined(_WIN32)
|
||||
if (vram_lock_fd >= 0) {
|
||||
return; // already open
|
||||
}
|
||||
const char * arena_env = getenv("LLAMA_VRAM_ARENA");
|
||||
const std::string arena = arena_env ? arena_env : "/dev/shm/llama-vram";
|
||||
mkdir(arena.c_str(), 0777);
|
||||
const std::string lock_path = arena + "/token.lock";
|
||||
vram_lock_fd = open(lock_path.c_str(), O_RDWR | O_CREAT, 0666);
|
||||
if (vram_lock_fd >= 0) {
|
||||
vram_flock = true;
|
||||
const std::string want_path = arena + "/want.lock";
|
||||
vram_want_fd = open(want_path.c_str(), O_RDWR | O_CREAT, 0666);
|
||||
vram_want_probe_fd = open(want_path.c_str(), O_RDWR | O_CREAT, 0666);
|
||||
if (vram_want_fd < 0 || vram_want_probe_fd < 0) {
|
||||
SRV_WRN("VRAM arbiter: cannot open %s, the holder will only release on idle\n", want_path.c_str());
|
||||
}
|
||||
SRV_INF("VRAM arbiter: cross-process GPU sharing via %s (want %s)\n",
|
||||
lock_path.c_str(), want_path.c_str());
|
||||
} else {
|
||||
SRV_WRN("VRAM arbiter: cannot open %s, running VRAM-only sleep without cross-process lock\n", lock_path.c_str());
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// Acquire the VRAM token BEFORE uploading this model's weights, so any model currently resident
|
||||
// on the GPU releases first and the load uploads into free VRAM instead of racing it (which
|
||||
// could OOM, e.g. the 4B task model holding VRAM while a large model loads). Blocks until free.
|
||||
// Called from load_model() right before common_init_from_params().
|
||||
void vram_acquire_for_load() {
|
||||
vram_arena_open();
|
||||
if (!vram_only) {
|
||||
return;
|
||||
}
|
||||
if (!vram_take_token()) {
|
||||
SRV_WRN("%s", "VRAM arbiter: loading weights without the GPU token\n");
|
||||
}
|
||||
vram_cold = false; // we hold the token; weights will be resident after the upload
|
||||
}
|
||||
|
||||
// Called from init() after the (coordinated) load. The model stays warm (holding the token
|
||||
// acquired in vram_acquire_for_load) and serves its first request without a re-warm; the queue
|
||||
// releases it once another model registers on want.lock.
|
||||
void vram_share_init() {
|
||||
vram_arena_open();
|
||||
if (!vram_only) {
|
||||
return;
|
||||
}
|
||||
vram_cold = false; // resident and holding the token after the coordinated load
|
||||
#if !defined(_WIN32)
|
||||
if (vram_flock && vram_want_probe_fd >= 0) {
|
||||
queue_tasks.on_should_yield([this]{ return vram_should_yield(); });
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// true while another process waits for the token and we still hold it. Called by should_sleep()
|
||||
// on the start_loop() thread, under mutex_tasks, and only when the queue is idle - so a release
|
||||
// can never happen underneath an in-flight decode, and there is no event to latch: the answer
|
||||
// comes from live kernel state at the moment the loop asks.
|
||||
bool vram_should_yield() {
|
||||
#if !defined(_WIN32)
|
||||
if (!vram_flock || vram_cold || vram_want_probe_fd < 0) {
|
||||
return false; // we hold no token to give away
|
||||
}
|
||||
// never blocks: LOCK_NB only, and we drop the probe lock at once if we win it
|
||||
if (flock(vram_want_probe_fd, LOCK_EX | LOCK_NB) == 0) {
|
||||
flock(vram_want_probe_fd, LOCK_UN);
|
||||
return false; // nobody wants the GPU
|
||||
}
|
||||
return errno == EWOULDBLOCK || errno == EAGAIN;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
// register as a waiter, so the current token holder sees us and yields. Shared, so several
|
||||
// waiters coexist. The kernel drops it if we die, so it can never strand the holder.
|
||||
void vram_want_acquire() {
|
||||
#if !defined(_WIN32)
|
||||
if (vram_want_fd < 0 || vram_want_held) {
|
||||
return;
|
||||
}
|
||||
while (flock(vram_want_fd, LOCK_SH) != 0) {
|
||||
if (errno != EINTR) { // our signal handlers run without SA_RESTART
|
||||
SRV_WRN("VRAM arbiter: cannot take want.lock (errno %d)\n", errno);
|
||||
return;
|
||||
}
|
||||
}
|
||||
vram_want_held = true;
|
||||
#endif
|
||||
}
|
||||
|
||||
// stop asking for the token. Must run on every path out of vram_take_token(), or the holder
|
||||
// yields forever.
|
||||
void vram_want_release() {
|
||||
#if !defined(_WIN32)
|
||||
if (vram_want_fd < 0 || !vram_want_held) {
|
||||
return;
|
||||
}
|
||||
flock(vram_want_fd, LOCK_UN);
|
||||
vram_want_held = false;
|
||||
#endif
|
||||
}
|
||||
|
||||
// take the shared VRAM token: register on want.lock, then wait for token.lock. want.lock is
|
||||
// level-triggered state, so the holder still learns of us if it is loading or restoring when we
|
||||
// start to wait. No timeout: the holder can be mid-generation, and decoding without the token
|
||||
// risks an OOM that kills both processes. Returns false without the token if the server shuts
|
||||
// down meanwhile (the caller checks queue_tasks.is_stopping()).
|
||||
bool vram_take_token() {
|
||||
#if !defined(_WIN32)
|
||||
if (!vram_flock) {
|
||||
return true; // no cross-process lock: there is no token to take
|
||||
}
|
||||
vram_want_acquire();
|
||||
bool logged = false;
|
||||
while (flock(vram_lock_fd, LOCK_EX | LOCK_NB) != 0) {
|
||||
if (errno == EWOULDBLOCK || errno == EAGAIN) {
|
||||
if (!logged) {
|
||||
SRV_INF("%s", "VRAM arbiter: waiting for the GPU token\n");
|
||||
logged = true;
|
||||
}
|
||||
// poll, not a blocking flock: terminate() must be able to stop this wait, and it can
|
||||
// only signal us through an atomic (we hold mutex_tasks here)
|
||||
if (queue_tasks.is_stopping()) {
|
||||
SRV_INF("%s", "VRAM arbiter: shutting down, no longer waiting for the GPU token\n");
|
||||
vram_want_release();
|
||||
return false;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
||||
continue;
|
||||
}
|
||||
if (errno == EINTR) {
|
||||
continue; // our signal handlers run without SA_RESTART
|
||||
}
|
||||
SRV_WRN("VRAM arbiter: flock failed (errno %d)\n", errno);
|
||||
vram_want_release();
|
||||
return false;
|
||||
}
|
||||
vram_want_release();
|
||||
return true;
|
||||
#else
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
// acquire the VRAM token (blocking) and bring the device buffers back (weights; with
|
||||
// LLAMA_SLEEP_EVICT_KV also the KV/recurrent state and the scheduler). Called on the loop thread:
|
||||
// on wake from handle_sleeping_state(false), before update_slots() touches the KV, and again
|
||||
// right before each decode (a no-op when already warm). It never races compute. Returns false,
|
||||
// still cold, if the server shuts down first: the caller must not touch the device then.
|
||||
bool vram_ensure_warm() {
|
||||
if (!vram_only || !vram_cold) {
|
||||
return true;
|
||||
}
|
||||
if (queue_tasks.is_stopping()) {
|
||||
return false; // no restore just to exit
|
||||
}
|
||||
if (!vram_take_token()) {
|
||||
if (queue_tasks.is_stopping()) {
|
||||
return false;
|
||||
}
|
||||
SRV_WRN("%s", "VRAM arbiter: restoring weights without the GPU token\n");
|
||||
}
|
||||
llama_context_restore_device(ctx_tgt);
|
||||
if (ctx_dft != nullptr) {
|
||||
llama_context_restore_device(ctx_dft);
|
||||
}
|
||||
// NOTE: the multimodal (vision/audio) encoder is intentionally NOT restored here. It is
|
||||
// brought into VRAM just-in-time before an image/audio encode (encode_mtmd_chunk) and, in
|
||||
// on-demand mode, released again right after - so a warm text-only model holds no encoder
|
||||
// VRAM (that space is free for KV / expert cache). A cold->warm wake for a *text* request
|
||||
// therefore leaves the encoder in RAM; an image request restores it at encode time.
|
||||
// NOTE: keep this AFTER the restore: vram_cold == false means "holding the token AND
|
||||
// resident". Today the only reader that could give the token away, vram_should_yield(), runs
|
||||
// on this same loop thread and so cannot run before we return; the order is what kept the
|
||||
// retired cross-thread doorbell design from yielding mid-restore (FORK.md F6, attempt 1).
|
||||
// Do not move it without an A/B of ordinary handoff latency. We dropped want.lock when we
|
||||
// took the token, so the next probe reports only the waiters that are still there.
|
||||
vram_cold = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// the loop has stopped: free our VRAM and hand the token over now, without waiting for the HTTP
|
||||
// threads and the teardown. Discarded, not released: no host copy, and the context never decodes
|
||||
// again. Runs on the loop thread after the loop returned, so nothing else uses the device.
|
||||
void vram_share_exit() {
|
||||
if (vram_only && !vram_cold && ctx_tgt != nullptr) {
|
||||
llama_context_discard_device(ctx_tgt);
|
||||
if (ctx_dft != nullptr) {
|
||||
llama_context_discard_device(ctx_dft);
|
||||
}
|
||||
if (mctx != nullptr) {
|
||||
mtmd_release_device(mctx);
|
||||
}
|
||||
vram_cold = true;
|
||||
SRV_INF("%s", "VRAM arbiter: loop stopped, device memory freed, releasing the GPU token\n");
|
||||
}
|
||||
vram_share_shutdown();
|
||||
}
|
||||
|
||||
void vram_share_shutdown() {
|
||||
#if !defined(_WIN32)
|
||||
vram_want_release();
|
||||
if (vram_want_probe_fd >= 0) { close(vram_want_probe_fd); vram_want_probe_fd = -1; }
|
||||
if (vram_want_fd >= 0) { close(vram_want_fd); vram_want_fd = -1; }
|
||||
if (vram_lock_fd >= 0) { flock(vram_lock_fd, LOCK_UN); close(vram_lock_fd); vram_lock_fd = -1; }
|
||||
#endif
|
||||
}
|
||||
|
||||
// release the device weights (their host shadow is kept; with LLAMA_SLEEP_EVICT_KV also the
|
||||
// KV/recurrent state, scheduler and backend scratch) and drop the VRAM token so another model
|
||||
// can warm up. Called on the loop thread from handle_sleeping_state(true): the queue is idle and
|
||||
// another process waits for the token.
|
||||
void vram_go_cold() {
|
||||
if (!vram_only || vram_cold) {
|
||||
return;
|
||||
}
|
||||
llama_context_release_device(ctx_tgt, vram_evict_kv);
|
||||
if (ctx_dft != nullptr) {
|
||||
llama_context_release_device(ctx_dft, vram_evict_kv);
|
||||
}
|
||||
// also release the multimodal (vision/audio) encoder weights (dead weight while cold; a
|
||||
// no-op when LLAMA_MMPROJ_ONDEMAND already keeps them out). Their read-only host shadow is
|
||||
// captured once. vram_ensure_warm() does not bring them back: encode_mtmd_chunk() restores
|
||||
// the encoder only for the duration of an encode.
|
||||
if (mctx != nullptr) {
|
||||
mtmd_release_device(mctx);
|
||||
}
|
||||
#if !defined(_WIN32)
|
||||
if (vram_flock) {
|
||||
flock(vram_lock_fd, LOCK_UN);
|
||||
}
|
||||
#endif
|
||||
vram_cold = true;
|
||||
}
|
||||
|
||||
// the model file and every parameter that changes the state layout or the KV values
|
||||
std::string disk_cache_key() const {
|
||||
std::ostringstream key;
|
||||
const auto add_file = [&](const std::string & path) {
|
||||
std::error_code ec;
|
||||
const auto size = std::filesystem::file_size(path, ec);
|
||||
const auto mtime = std::filesystem::last_write_time(path, ec).time_since_epoch().count();
|
||||
key << path << " " << size << " " << (long long) mtime << "\n";
|
||||
};
|
||||
key << "commit " << llama_commit() << "\n";
|
||||
add_file(params_base.model.path);
|
||||
for (const auto & la : params_base.lora_adapters) {
|
||||
add_file(la.path);
|
||||
key << la.scale << "\n";
|
||||
}
|
||||
for (const auto & cv : params_base.control_vectors) {
|
||||
add_file(cv.fname);
|
||||
key << cv.strength << "\n";
|
||||
}
|
||||
key << llama_n_ctx(ctx_tgt) << " " << llama_n_ctx_seq(ctx_tgt) << " " << llama_n_seq_max(ctx_tgt) << " " << llama_n_ubatch(ctx_tgt) << "\n";
|
||||
key << params_base.kv_unified << " " << params_base.swa_full << " " << ggml_type_name(params_base.cache_type_k) << " "
|
||||
<< ggml_type_name(params_base.cache_type_v) << " " << llama_flash_attn_type_name(params_base.flash_attn_type) << "\n";
|
||||
key << std::hexfloat << (int) params_base.rope_scaling_type << " " << params_base.rope_freq_base << " " << params_base.rope_freq_scale << " "
|
||||
<< params_base.yarn_ext_factor << " " << params_base.yarn_attn_factor << " " << params_base.yarn_beta_fast << " "
|
||||
<< params_base.yarn_beta_slow << " " << params_base.yarn_orig_ctx << "\n";
|
||||
return key.str();
|
||||
}
|
||||
|
||||
// on shutdown, the loop has stopped: copy the state of each slot with a prompt to host memory
|
||||
// while it is still readable. disk_cache_flush() writes them once the GPU token is released.
|
||||
void disk_cache_collect() {
|
||||
if (!disk_cache || ctx_tgt == nullptr) {
|
||||
return;
|
||||
}
|
||||
std::vector<server_slot *> todo;
|
||||
for (auto & slot : slots) {
|
||||
llama_tokens tokens;
|
||||
if (!disk_cache_text(slot.prompt.tokens, tokens) || tokens.size() < DISK_CACHE_MIN_TOKENS) {
|
||||
continue;
|
||||
}
|
||||
// the lora config is not in the file, only the server one can be restored
|
||||
if (!are_lora_equal(slot.prompt.lora, params_base.lora_adapters)) {
|
||||
continue;
|
||||
}
|
||||
// the last batch is not decoded if the shutdown came during a token wait
|
||||
if (llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id) + 1 != (llama_pos) tokens.size()) {
|
||||
SLT_WRN(slot, "disk cache: the state does not match the %zu prompt tokens, not saved\n", tokens.size());
|
||||
continue;
|
||||
}
|
||||
if (disk_cache->has(tokens)) {
|
||||
SLT_INF(slot, "disk cache: the prompt of %zu tokens is already saved\n", tokens.size());
|
||||
continue;
|
||||
}
|
||||
todo.push_back(&slot);
|
||||
}
|
||||
if (todo.empty()) {
|
||||
return;
|
||||
}
|
||||
// cold: the state is in a host shadow (LLAMA_SLEEP_EVICT_KV) or still in VRAM, read it without the device
|
||||
if (vram_only && vram_cold && !llama_context_restore_memory_host(ctx_tgt)) {
|
||||
SRV_WRN("%s", "disk cache: cannot read the released state, not saved\n");
|
||||
return;
|
||||
}
|
||||
for (auto * pslot : todo) {
|
||||
auto & slot = *pslot;
|
||||
const int64_t t_start = ggml_time_us();
|
||||
server_prompt_cache_state st;
|
||||
try {
|
||||
st.data.main.resize(llama_state_seq_get_size_ext(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE));
|
||||
} catch (const std::bad_alloc &) {
|
||||
SLT_WRN(slot, "%s", "disk cache: out of memory, not saved\n");
|
||||
continue;
|
||||
}
|
||||
if (st.data.main.empty() || llama_state_seq_get_data_ext(ctx_tgt, st.data.main.data(), st.data.main.size(), slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) != st.data.main.size()) {
|
||||
SLT_WRN(slot, "%s", "disk cache: failed to copy the state, not saved\n");
|
||||
continue;
|
||||
}
|
||||
st.prompt = std::move(slot.prompt);
|
||||
SLT_INF(slot, "disk cache: copied the state of %d tokens (%.1f MiB, %zu checkpoints) in %.0f ms\n",
|
||||
st.prompt.n_tokens(), st.size() / (1024.0 * 1024.0), st.prompt.checkpoints.size(), (ggml_time_us() - t_start) / 1000.0);
|
||||
disk_pending.push_back(std::move(st));
|
||||
}
|
||||
}
|
||||
|
||||
void disk_cache_flush() {
|
||||
for (const auto & st : disk_pending) {
|
||||
const int64_t t_start = ggml_time_us();
|
||||
if (disk_cache->save(st.prompt, st.data.main)) {
|
||||
SRV_INF("disk cache: saved the prompt of %d tokens (%.1f MiB) in %.0f ms\n",
|
||||
st.prompt.n_tokens(), st.size() / (1024.0 * 1024.0), (ggml_time_us() - t_start) / 1000.0);
|
||||
}
|
||||
}
|
||||
disk_pending.clear();
|
||||
}
|
||||
|
||||
// replace the slot prompt with a disk cache entry that reuses at least DISK_CACHE_MIN_TOKENS more of the new prompt
|
||||
void disk_cache_load(server_slot & slot, const server_tokens & tokens_new) {
|
||||
if (vram_only && vram_cold) {
|
||||
return; // restore only while holding the GPU
|
||||
}
|
||||
llama_tokens tokens;
|
||||
if (!disk_cache_text(tokens_new, tokens)) {
|
||||
return;
|
||||
}
|
||||
// a recurrent or SWA state cannot be cut back to an arbitrary prefix (see update_slots)
|
||||
const bool can_trim = ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_PART && n_swa == 0;
|
||||
std::vector<int64_t> ckpt_n_tokens;
|
||||
for (const auto & c : slot.prompt.checkpoints) {
|
||||
ckpt_n_tokens.push_back(c->n_tokens);
|
||||
}
|
||||
llama_tokens tokens_slot;
|
||||
const size_t n_have = disk_cache_text(slot.prompt.tokens, tokens_slot) ?
|
||||
server_prompt_disk_cache::n_reuse(tokens_slot, ckpt_n_tokens, tokens, can_trim) : slot.prompt.tokens.get_common_prefix(tokens_new);
|
||||
const int i = disk_cache->find(tokens, n_have, DISK_CACHE_MIN_TOKENS, can_trim);
|
||||
if (i < 0) {
|
||||
return;
|
||||
}
|
||||
const std::string path = disk_cache->entries[i].path;
|
||||
const size_t n_reuse = server_prompt_disk_cache::n_reuse(disk_cache->entries[i].tokens, disk_cache->entries[i].ckpt_n_tokens, tokens, can_trim);
|
||||
|
||||
const int64_t t_start = ggml_time_us();
|
||||
server_prompt prompt;
|
||||
common_state_data data;
|
||||
if (!disk_cache->load(i, prompt, data, mctx != nullptr)) {
|
||||
return;
|
||||
}
|
||||
const int64_t t_read = ggml_time_us();
|
||||
|
||||
// keep the current slot prompt in the RAM cache, as update_prompt_cache() does
|
||||
if (prompt_cache && slot.prompt_save(*prompt_cache)) {
|
||||
prompt_cache->update();
|
||||
}
|
||||
slot.prompt_clear();
|
||||
if (llama_state_seq_set_data_ext(ctx_tgt, data.data(), data.size(), slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) != data.size()) {
|
||||
SLT_WRN(slot, "disk cache: failed to restore the state from '%s', removing it\n", path.c_str());
|
||||
slot.prompt_clear();
|
||||
disk_cache->remove(i);
|
||||
return;
|
||||
}
|
||||
while (prompt.checkpoints.size() > (size_t) std::max(0, params_base.n_ctx_checkpoints)) {
|
||||
prompt.checkpoints.pop_front();
|
||||
}
|
||||
size_t size = data.size();
|
||||
for (const auto & c : prompt.checkpoints) {
|
||||
size += c->size();
|
||||
}
|
||||
// only prompts made with the server lora config are saved, launch_slot_with_task() compares it with the task
|
||||
prompt.lora = params_base.lora_adapters;
|
||||
slot.prompt = std::move(prompt);
|
||||
SLT_INF(slot, "disk cache: restored %d tokens (%.1f MiB, %zu checkpoints) from '%s', read %.0f ms, restore %.0f ms, %zu of %zu prompt tokens reusable (had %zu)\n",
|
||||
slot.prompt.n_tokens(), size / (1024.0 * 1024.0), slot.prompt.checkpoints.size(), path.c_str(),
|
||||
(t_read - t_start) / 1000.0, (ggml_time_us() - t_read) / 1000.0, n_reuse, tokens.size(), n_have);
|
||||
}
|
||||
|
||||
common_speculative_init_result_ptr spec_init;
|
||||
|
||||
common_context_seq_rm_type ctx_tgt_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
|
||||
@@ -1133,6 +1609,10 @@ private:
|
||||
|
||||
std::unique_ptr<server_prompt_cache> prompt_cache;
|
||||
|
||||
// prompt states saved to disk on shutdown and restored after a restart (--cache-disk-path)
|
||||
std::unique_ptr<server_prompt_disk_cache> disk_cache;
|
||||
std::vector<server_prompt_cache_state> disk_pending; // copied before the device buffers are freed, written after
|
||||
|
||||
server_metrics metrics;
|
||||
|
||||
// queued prompt stats - llama_decode() is async, so the timing is only valid after a sync
|
||||
@@ -1171,6 +1651,32 @@ private:
|
||||
|
||||
void handle_sleeping_state(bool new_state) {
|
||||
GGML_ASSERT(sleeping != new_state);
|
||||
|
||||
// Lightweight VRAM-only sleep: instead of a full unload/reload (which frees RAM and the
|
||||
// KV cache and pays a full reload on wake), just release the model's device (VRAM) buffers
|
||||
// to a host shadow (keeping the context, the prompt cache and the host weights; the KV stays
|
||||
// on the device unless LLAMA_SLEEP_EVICT_KV is set) and drop the shared VRAM token. Waking
|
||||
// restores eagerly, on the wake transition below: vram_ensure_warm() re-acquires the token
|
||||
// and restores before update_slots() runs. A single GPU is so time-shared between models
|
||||
// with no re-prefill.
|
||||
if (vram_only && ctx_tgt != nullptr) {
|
||||
if (new_state) {
|
||||
SRV_INF("server entering sleeping state (VRAM-only: releasing device weights, %s)\n",
|
||||
vram_evict_kv ? "evicting KV cache to host" : "keeping KV cache in VRAM");
|
||||
vram_go_cold();
|
||||
} else {
|
||||
SRV_INF("%s", "server exiting sleeping state (VRAM-only: restoring device weights/KV)\n");
|
||||
// Restore NOW, on wake, before update_slots runs. update_slots touches the KV cache
|
||||
// (e.g. SWA checkpoint creation reads it via ggml_backend_tensor_get) before the
|
||||
// decode-time vram_ensure_warm(), so with KV eviction the KV must already be resident
|
||||
// here or those reads hit a freed (null) buffer.
|
||||
// false (still cold) only on shutdown: the loop then exits before update_slots
|
||||
vram_ensure_warm();
|
||||
}
|
||||
sleeping = new_state;
|
||||
return;
|
||||
}
|
||||
|
||||
if (new_state) {
|
||||
if (callback_state) {
|
||||
callback_state(SERVER_STATE_SLEEPING, {});
|
||||
@@ -1323,6 +1829,11 @@ private:
|
||||
params_base.load_progress_callback_user_data = &load_progress_text;
|
||||
}
|
||||
|
||||
// VRAM arbiter: acquire the GPU token before uploading weights, so any resident model (e.g.
|
||||
// the warm 4B task model) releases first and this load uploads into free VRAM instead of
|
||||
// racing it and OOM-ing. No-op unless LLAMA_SLEEP_VRAM_ONLY is set.
|
||||
vram_acquire_for_load();
|
||||
|
||||
llama_init = common_init_from_params(params_base);
|
||||
|
||||
model_tgt = llama_init->model();
|
||||
@@ -1422,6 +1933,13 @@ private:
|
||||
init_opt.video_params.ffmpeg_bin_dir = params_base.video_ffmpeg_bin_dir.empty()
|
||||
? nullptr : params_base.video_ffmpeg_bin_dir.c_str();
|
||||
|
||||
// on-demand encoder: keep the vision/audio encoder weights in RAM (out of VRAM) until an
|
||||
// image/audio actually needs encoding, freeing that VRAM for KV / expert cache on the
|
||||
// common text-only path. encode_mtmd_chunk() brings the encoder in just-in-time.
|
||||
if (getenv("LLAMA_MMPROJ_ONDEMAND") != nullptr) {
|
||||
mtmd_release_device(mctx);
|
||||
}
|
||||
|
||||
if (params_base.ctx_shift) {
|
||||
params_base.ctx_shift = false;
|
||||
SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");
|
||||
@@ -1633,8 +2151,18 @@ private:
|
||||
} else {
|
||||
SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
|
||||
}
|
||||
|
||||
SRV_TRC("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n");
|
||||
|
||||
disk_cache.reset();
|
||||
if (!params_base.cache_disk_path.empty()) {
|
||||
if (spec) {
|
||||
SRV_WRN("%s", "disk cache: not supported with speculative decoding, disabled\n");
|
||||
} else {
|
||||
disk_cache = std::make_unique<server_prompt_disk_cache>(params_base.cache_disk_path, params_base.cache_disk_mib, disk_cache_key());
|
||||
}
|
||||
}
|
||||
|
||||
if (params_base.n_ctx_checkpoints > 0) {
|
||||
SRV_TRC("context checkpoints enabled, max = %d, min spacing = %d\n",
|
||||
params_base.n_ctx_checkpoints, params_base.checkpoint_min_step);
|
||||
@@ -1688,6 +2216,11 @@ private:
|
||||
handle_sleeping_state(sleeping);
|
||||
});
|
||||
|
||||
// VRAM arbiter: arm the yield probe. The load was coordinated (vram_acquire_for_load
|
||||
// grabbed the token before uploading), so we stay warm holding the token and serve the
|
||||
// first request without a re-warm; the loop releases it when another model waits for it.
|
||||
vram_share_init();
|
||||
|
||||
metrics.init();
|
||||
|
||||
if (params_base.cache_idle_slots) {
|
||||
@@ -3021,6 +3554,10 @@ private:
|
||||
update_prompt_cache(*slot, task);
|
||||
}
|
||||
|
||||
if (disk_cache && task.params.cache_prompt && !task.is_stateless()) {
|
||||
disk_cache_load(*slot, task.tokens);
|
||||
}
|
||||
|
||||
if (task.is_parent()) {
|
||||
if (!launch_slots_with_parent_task(*slot, child_slots, std::move(task))) {
|
||||
SRV_ERR("failed to launch slot with parent task, id_task = %d\n", id_task);
|
||||
@@ -4405,6 +4942,19 @@ private:
|
||||
|
||||
const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1;
|
||||
|
||||
// [fork F14] do not break at user-message starts when the rest of the prompt
|
||||
// from here fits in one ubatch: the break turns one ubatch into two, and with
|
||||
// MoE experts on the host each one of >= 32 tokens uploads the experts again.
|
||||
// A checkpoint at this batch start (the near-end rule below) and at the end
|
||||
// minus 4 still exist, so a later prompt that diverges inside the merged range
|
||||
// re-processes less than one ubatch more. LLAMA_USERSTART_MERGE=0 disables.
|
||||
static const bool userstart_merge = [] {
|
||||
const char * env = getenv("LLAMA_USERSTART_MERGE");
|
||||
return env == nullptr || std::string(env) != "0";
|
||||
}();
|
||||
const bool merge_user_starts = userstart_merge && !has_mtmd &&
|
||||
slot.task->n_tokens() - slot.prompt.n_tokens() <= n_ubatch;
|
||||
|
||||
// add prompt tokens for processing in the current batch
|
||||
while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) {
|
||||
// get next token to process
|
||||
@@ -4448,7 +4998,7 @@ private:
|
||||
slot.prompt.tokens.push_back(cur_tok);
|
||||
|
||||
// break at the last user message, or at user messages at least min step past the last checkpoint
|
||||
if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
|
||||
if (do_checkpoint && !merge_user_starts && spans.is_user_start(slot.prompt.n_tokens())) {
|
||||
const auto pos = slot.prompt.n_tokens();
|
||||
const auto & checkpoints = slot.prompt.checkpoints;
|
||||
|
||||
@@ -4642,6 +5192,13 @@ private:
|
||||
has_output |= batch.tokens[i].output;
|
||||
}
|
||||
|
||||
// VRAM arbiter: make sure our weights are resident (acquiring the shared GPU token first)
|
||||
// before we decode. Runs on the loop thread before entering yield_to_queue(), so it never
|
||||
// races an in-flight decode.
|
||||
if (!vram_ensure_warm()) {
|
||||
throw std::runtime_error("server is shutting down");
|
||||
}
|
||||
|
||||
// yield to the queue, so we can still handle metrics tasks while decoding
|
||||
// note: the sync is done here too, so that the wait is also covered by the yield
|
||||
int ret = 0;
|
||||
@@ -5159,9 +5716,15 @@ bool server_context::load_model(common_params & params) {
|
||||
void server_context::start_loop() {
|
||||
auto & params = impl->params_base;
|
||||
impl->queue_tasks.start_loop(params.sleep_idle_seconds * 1000);
|
||||
// copy the slot states while the device buffers exist, write them after the GPU token is released
|
||||
impl->disk_cache_collect();
|
||||
impl->vram_share_exit();
|
||||
impl->disk_cache_flush();
|
||||
}
|
||||
|
||||
void server_context::terminate() {
|
||||
// wake the HTTP threads waiting for results first: the loop will not produce them any more
|
||||
impl->queue_results.terminate();
|
||||
impl->queue_tasks.terminate();
|
||||
}
|
||||
|
||||
@@ -5238,8 +5801,12 @@ struct server_res_generator : server_res_spipe {
|
||||
server_queue * queue_awake = nullptr; // set while this response keeps the server awake
|
||||
server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false)
|
||||
: rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) {
|
||||
// fast path in case sleeping is disabled
|
||||
bypass_sleep |= sleep_idle_seconds < 0;
|
||||
// fast path in case sleeping is disabled. Note: the VRAM arbiter (LLAMA_SLEEP_VRAM_ONLY)
|
||||
// can put the server to sleep to hand over the GPU token even when idle-sleep is disabled
|
||||
// (sleep_idle_seconds < 0), so in that case requests must still wake it - otherwise an
|
||||
// arbiter-slept server would hang, never returning from a request.
|
||||
static const bool vram_arbiter = getenv("LLAMA_SLEEP_VRAM_ONLY") != nullptr;
|
||||
bypass_sleep |= (sleep_idle_seconds < 0 && !vram_arbiter);
|
||||
if (!bypass_sleep) {
|
||||
queue_tasks.wait_until_no_sleep();
|
||||
queue_awake = &queue_tasks;
|
||||
|
||||
@@ -114,16 +114,17 @@ void server_queue::pop_deferred_task(int id_slot) {
|
||||
|
||||
void server_queue::wait_until_no_sleep() {
|
||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||
if (sleeping) {
|
||||
// re-ask on every wake: the loop clears req_stop_sleeping on the way into sleep, so a loop that
|
||||
// goes back to sleep before we run would strand us here forever
|
||||
// a loop that terminates while asleep never wakes: give up then, the reader reports the shutdown
|
||||
while (sleeping && !stopping) {
|
||||
if (!req_stop_sleeping) {
|
||||
QUE_DBG("%s", "requesting to stop sleeping\n");
|
||||
req_stop_sleeping = true;
|
||||
condition_tasks.notify_one(); // only main thread is waiting on this
|
||||
condition_tasks.notify_all(); // other threads may wait on this too
|
||||
}
|
||||
QUE_DBG("%s", "waiting until no sleep\n");
|
||||
condition_tasks.wait(lock, [&]{
|
||||
return !sleeping;
|
||||
});
|
||||
condition_tasks.wait(lock);
|
||||
}
|
||||
n_inflight++;
|
||||
}
|
||||
@@ -135,6 +136,7 @@ void server_queue::release_inflight() {
|
||||
}
|
||||
|
||||
void server_queue::terminate() {
|
||||
stopping = true; // before the lock: the loop may hold it while it waits for the VRAM token
|
||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||
running = false;
|
||||
condition_tasks.notify_all();
|
||||
@@ -143,7 +145,7 @@ void server_queue::terminate() {
|
||||
bool server_queue::process_new_tasks(bool is_yielding) {
|
||||
while (true) {
|
||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||
if (!running) {
|
||||
if (!running || stopping) {
|
||||
QUE_DBG("%s", "terminate\n");
|
||||
return true;
|
||||
}
|
||||
@@ -291,10 +293,22 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
|
||||
worker.yielding = false;
|
||||
worker.thread = std::thread([this]() { worker_loop(); });
|
||||
|
||||
constexpr auto max_wait_time = std::chrono::seconds(1);
|
||||
// the arbiter predicate is read from live state, so this timeout bounds how fast we notice a
|
||||
// process waiting for the VRAM token
|
||||
const auto max_wait_time = should_yield_cb ? std::chrono::milliseconds(10) : std::chrono::milliseconds(1000);
|
||||
// after a wake, keep the VRAM for at least this long: the request that woke us is not in the
|
||||
// queue yet, so yielding at once would only send it through another wake
|
||||
constexpr int64_t yield_grace_ms = 100;
|
||||
int64_t time_last_wake = 0;
|
||||
auto should_sleep = [&]() -> bool {
|
||||
// caller must hold mutex_tasks
|
||||
if (idle_sleep_ms < 0 || n_inflight > 0) {
|
||||
if (n_inflight > 0) {
|
||||
return false;
|
||||
}
|
||||
if (should_yield_cb && ggml_time_ms() - time_last_wake >= yield_grace_ms && should_yield_cb()) {
|
||||
return true; // another process waits for the VRAM token - release now
|
||||
}
|
||||
if (idle_sleep_ms < 0) {
|
||||
return false;
|
||||
}
|
||||
int64_t now = ggml_time_ms();
|
||||
@@ -337,9 +351,11 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
|
||||
cb(true);
|
||||
}
|
||||
req_stop_sleeping = false;
|
||||
// wait until we are requested to exit sleeping state
|
||||
// wait until we are requested to exit sleeping state, or a task arrives: post() only
|
||||
// notifies, so a task queued right after wait_until_no_sleep() saw us awake must be
|
||||
// able to wake us by itself, else it waits here for an unrelated request
|
||||
condition_tasks.wait(lock, [&]{
|
||||
return (!running || req_stop_sleeping);
|
||||
return (!running || req_stop_sleeping || !queue_tasks.empty());
|
||||
});
|
||||
if (!running) { // may changed during sleep
|
||||
break; // terminate
|
||||
@@ -352,17 +368,21 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
|
||||
}
|
||||
sleeping = false;
|
||||
time_last_task = ggml_time_ms();
|
||||
time_last_wake = time_last_task;
|
||||
condition_tasks.notify_all(); // notify wait_until_no_sleep()
|
||||
break; // process new tasks
|
||||
} else {
|
||||
// wait for new tasks or timeout for checking sleeping condition
|
||||
// wait for new tasks, or timeout for checking sleeping condition
|
||||
bool res = condition_tasks.wait_for(lock, max_wait_time, [&]{
|
||||
return (!queue_tasks.empty() || !running);
|
||||
});
|
||||
if (res) {
|
||||
if (res && !queue_tasks.empty()) {
|
||||
break; // new task arrived or terminate
|
||||
}
|
||||
// otherwise, loop again to check sleeping condition
|
||||
if (!running) {
|
||||
break;
|
||||
}
|
||||
// otherwise (timeout), loop again to re-check should_sleep
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -465,11 +485,12 @@ server_task_result_ptr server_response::recv_with_timeout(const std::unordered_s
|
||||
}
|
||||
}
|
||||
|
||||
std::cv_status cr_res = condition_results.wait_until(lock, deadline);
|
||||
if (!running) {
|
||||
RES_DBG("%s : queue result stop\n", __func__);
|
||||
std::terminate(); // we cannot return here since the caller is HTTP code
|
||||
return nullptr; // the caller checks is_running()
|
||||
}
|
||||
|
||||
std::cv_status cr_res = condition_results.wait_until(lock, deadline);
|
||||
if (cr_res == std::cv_status::timeout) {
|
||||
return nullptr;
|
||||
}
|
||||
@@ -510,6 +531,7 @@ void server_response::broadcast(server_task_result_ptr && result) {
|
||||
}
|
||||
|
||||
void server_response::terminate() {
|
||||
std::unique_lock<std::mutex> lock(mutex_results);
|
||||
running = false;
|
||||
condition_results.notify_all();
|
||||
}
|
||||
@@ -557,6 +579,14 @@ server_task_result_ptr server_response_reader::next(const std::function<bool()>
|
||||
while (true) {
|
||||
server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
|
||||
if (result == nullptr) {
|
||||
if (!queue_results.is_running()) {
|
||||
// the server is shutting down: no result will ever come
|
||||
stop();
|
||||
auto res = std::make_unique<server_task_result_error>();
|
||||
res->err_type = ERROR_TYPE_UNAVAILABLE;
|
||||
res->err_msg = "server is shutting down";
|
||||
return res;
|
||||
}
|
||||
// timeout, check stop condition
|
||||
if (should_stop()) {
|
||||
return nullptr;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "server-task.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <deque>
|
||||
#include <exception>
|
||||
@@ -16,6 +17,9 @@ struct server_queue {
|
||||
private:
|
||||
int id = 0;
|
||||
bool running = false;
|
||||
// set by terminate() before it takes mutex_tasks, so a loop blocked under that mutex (e.g. in a
|
||||
// VRAM arbiter token wait) can see the shutdown and bail out
|
||||
std::atomic<bool> stopping{false};
|
||||
bool sleeping = false;
|
||||
bool req_stop_sleeping = false;
|
||||
int64_t time_last_task = 0;
|
||||
@@ -46,6 +50,7 @@ private:
|
||||
std::function<bool(server_task &&, bool)> callback_new_task;
|
||||
std::function<void(void)> callback_update_slots;
|
||||
std::vector<std::function<void(bool)>> callback_sleeping_state;
|
||||
std::function<bool()> should_yield_cb;
|
||||
|
||||
public:
|
||||
~server_queue() { worker_stop(); }
|
||||
@@ -82,6 +87,11 @@ public:
|
||||
// end the start_loop routine
|
||||
void terminate();
|
||||
|
||||
// true once terminate() was called; safe from any thread, including under mutex_tasks
|
||||
bool is_stopping() const {
|
||||
return stopping.load();
|
||||
}
|
||||
|
||||
/**
|
||||
* Main loop consists of these steps:
|
||||
* - Wait until a new task arrives
|
||||
@@ -132,6 +142,14 @@ public:
|
||||
callback_update_slots = std::move(callback);
|
||||
}
|
||||
|
||||
// Register a predicate asking the loop to sleep (release VRAM) as soon as it is idle - used by
|
||||
// the VRAM arbiter to check whether another process waits for the GPU token.
|
||||
// Called on the start_loop() thread while holding mutex_tasks, so it must not block or post
|
||||
// tasks. While it is set, the idle wait polls faster, so the loop reacts without a notify.
|
||||
void on_should_yield(std::function<bool()> callback) {
|
||||
should_yield_cb = std::move(callback);
|
||||
}
|
||||
|
||||
// Register callback for sleeping state change; multiple callbacks are allowed
|
||||
// for example: register order cb0, cb1, cb2
|
||||
// entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true)
|
||||
@@ -198,8 +216,13 @@ public:
|
||||
// (used by router mode)
|
||||
void broadcast(server_task_result_ptr && result);
|
||||
|
||||
// terminate the waiting loop
|
||||
// terminate the waiting loop: pending and later readers get an error instead of a result
|
||||
void terminate();
|
||||
|
||||
bool is_running() {
|
||||
std::unique_lock<std::mutex> lock(mutex_results);
|
||||
return running;
|
||||
}
|
||||
};
|
||||
|
||||
// RAII wrapper to make working with server_queue and server_response easier
|
||||
|
||||
@@ -5,6 +5,16 @@
|
||||
#include "log.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
//
|
||||
// server_prompt
|
||||
@@ -434,3 +444,442 @@ server_slot_plan server_slot_sched::select(const std::vector<server_slot_info> &
|
||||
|
||||
return { ret->id, update_cache };
|
||||
}
|
||||
|
||||
//
|
||||
// server_prompt_disk_cache
|
||||
//
|
||||
|
||||
// file = header + payload; the payload is a list of sections, each padded to 8 bytes:
|
||||
// tokens, checkpoint table, state size, state, the state of each checkpoint
|
||||
// hash_payload is FNV-1a over the 64-bit words of the payload, so a truncated or corrupt file is never used
|
||||
static constexpr uint32_t DISK_CACHE_MAGIC = 0x4443504c; // "LPCD"
|
||||
static constexpr uint32_t DISK_CACHE_VERSION = 1;
|
||||
static constexpr const char * DISK_CACHE_EXT = ".lpcd";
|
||||
|
||||
// keep this much space free on the file system
|
||||
static constexpr uint64_t DISK_CACHE_MIN_FREE = 1024ull*1024ull*1024ull;
|
||||
|
||||
struct disk_cache_header {
|
||||
uint32_t magic;
|
||||
uint32_t version;
|
||||
uint64_t key_hash;
|
||||
uint64_t n_tokens;
|
||||
uint64_t n_ckpt;
|
||||
uint64_t size_payload;
|
||||
uint64_t hash_payload;
|
||||
};
|
||||
|
||||
struct disk_cache_ckpt {
|
||||
int64_t n_tokens;
|
||||
llama_pos pos_min;
|
||||
llama_pos pos_max;
|
||||
uint64_t size;
|
||||
};
|
||||
|
||||
static_assert(sizeof(disk_cache_header) == 48 && sizeof(disk_cache_ckpt) == 24, "disk cache layout");
|
||||
|
||||
static uint64_t disk_cache_pad(uint64_t n) {
|
||||
return (8 - n % 8) % 8;
|
||||
}
|
||||
|
||||
static uint64_t disk_cache_hash_bytes(const void * data, size_t n) {
|
||||
uint64_t h = 0xcbf29ce484222325ull;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
h = (h ^ ((const uint8_t *) data)[i]) * 0x100000001b3ull;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
static uint64_t disk_cache_payload_size(uint64_t n_tokens, const std::vector<disk_cache_ckpt> & ckpts, uint64_t n_state) {
|
||||
const auto padded = [](uint64_t n) { return n + disk_cache_pad(n); };
|
||||
uint64_t res = padded(n_tokens*sizeof(llama_token)) + padded(ckpts.size()*sizeof(disk_cache_ckpt)) + sizeof(uint64_t) + padded(n_state);
|
||||
for (const auto & c : ckpts) {
|
||||
res += padded(c.size);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// sequential section IO with the running payload hash
|
||||
struct disk_cache_io {
|
||||
FILE * f = nullptr;
|
||||
uint64_t hash = 0xcbf29ce484222325ull;
|
||||
bool ok = true;
|
||||
|
||||
~disk_cache_io() {
|
||||
if (f) {
|
||||
fclose(f);
|
||||
}
|
||||
}
|
||||
|
||||
void add_hash(const uint8_t * p, size_t size, const uint8_t * pad) {
|
||||
const size_t n_full = size - size % 8;
|
||||
for (size_t i = 0; i < n_full; i += 8) {
|
||||
uint64_t w;
|
||||
memcpy(&w, p + i, 8);
|
||||
hash = (hash ^ w) * 0x100000001b3ull;
|
||||
}
|
||||
if (size % 8) {
|
||||
uint64_t w;
|
||||
memcpy(&w, p + n_full, size % 8);
|
||||
memcpy((uint8_t *) &w + size % 8, pad, disk_cache_pad(size));
|
||||
hash = (hash ^ w) * 0x100000001b3ull;
|
||||
}
|
||||
}
|
||||
|
||||
void write(const void * src, size_t size) {
|
||||
static const uint8_t zero[8] = {};
|
||||
const size_t n_pad = disk_cache_pad(size);
|
||||
ok = ok && fwrite(src, 1, size, f) == size && fwrite(zero, 1, n_pad, f) == n_pad;
|
||||
add_hash((const uint8_t *) src, size, zero);
|
||||
}
|
||||
|
||||
void read(void * dst, size_t size) {
|
||||
uint8_t pad[8];
|
||||
const size_t n_pad = disk_cache_pad(size);
|
||||
ok = ok && fread(dst, 1, size, f) == size && fread(pad, 1, n_pad, f) == n_pad;
|
||||
if (ok) {
|
||||
add_hash((const uint8_t *) dst, size, pad);
|
||||
}
|
||||
}
|
||||
|
||||
// the data is in our buffers, the cached pages are waste (see llama_file::drop_cache)
|
||||
void drop_cache() {
|
||||
#if defined(__linux__)
|
||||
posix_fadvise(fileno(f), 0, 0, POSIX_FADV_DONTNEED);
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
// open a file and read everything up to the state data: 1 = entry of this key, 0 = other key or version, -1 = bad file
|
||||
static int disk_cache_open(const std::string & path, uint64_t key_hash, disk_cache_io & io, disk_cache_header & hdr,
|
||||
llama_tokens & tokens, std::vector<disk_cache_ckpt> & ckpts, uint64_t & n_state) {
|
||||
std::error_code ec;
|
||||
const uint64_t size = std::filesystem::file_size(path, ec);
|
||||
io.f = ec ? nullptr : fopen(path.c_str(), "rb");
|
||||
if (io.f == nullptr || fread(&hdr, 1, sizeof(hdr), io.f) != sizeof(hdr) || hdr.magic != DISK_CACHE_MAGIC) {
|
||||
return -1;
|
||||
}
|
||||
if (hdr.version != DISK_CACHE_VERSION) {
|
||||
return 0;
|
||||
}
|
||||
if (size != sizeof(hdr) + hdr.size_payload || hdr.n_tokens > size/sizeof(llama_token) || hdr.n_ckpt > size/sizeof(disk_cache_ckpt)) {
|
||||
return -1;
|
||||
}
|
||||
if (hdr.key_hash != key_hash) {
|
||||
return 0;
|
||||
}
|
||||
tokens.resize(hdr.n_tokens);
|
||||
ckpts.resize(hdr.n_ckpt);
|
||||
io.read(tokens.data(), tokens.size()*sizeof(llama_token));
|
||||
io.read(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt));
|
||||
io.read(&n_state, sizeof(n_state));
|
||||
if (!io.ok || n_state > size) {
|
||||
return -1;
|
||||
}
|
||||
for (const auto & c : ckpts) {
|
||||
if (c.size > size) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return disk_cache_payload_size(hdr.n_tokens, ckpts, n_state) == hdr.size_payload ? 1 : -1;
|
||||
}
|
||||
|
||||
static void disk_cache_touch(const std::string & path) {
|
||||
std::error_code ec;
|
||||
std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now(), ec);
|
||||
}
|
||||
|
||||
server_prompt_disk_cache::server_prompt_disk_cache(const std::string & dir, int32_t limit_size_mib, const std::string & key) : dir(dir) {
|
||||
key_hash = disk_cache_hash_bytes(key.data(), key.size());
|
||||
limit_size = 1024ull*1024ull*std::max(0, limit_size_mib);
|
||||
|
||||
size_t n_other = 0;
|
||||
size_t n_bad = 0;
|
||||
try {
|
||||
std::filesystem::create_directories(dir);
|
||||
const auto now = std::filesystem::file_time_type::clock::now();
|
||||
for (const auto & de : std::filesystem::directory_iterator(dir)) {
|
||||
const std::string path = de.path().string();
|
||||
if (!de.is_regular_file()) {
|
||||
continue;
|
||||
}
|
||||
// left by a writer that was killed
|
||||
if (path.find(std::string(DISK_CACHE_EXT) + ".tmp") != std::string::npos) {
|
||||
if (now - de.last_write_time() > std::chrono::hours(1)) {
|
||||
std::filesystem::remove(de.path());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (de.path().extension() != DISK_CACHE_EXT) {
|
||||
continue;
|
||||
}
|
||||
disk_cache_io io;
|
||||
disk_cache_header hdr;
|
||||
server_prompt_disk_entry e;
|
||||
std::vector<disk_cache_ckpt> ckpts;
|
||||
uint64_t n_state = 0;
|
||||
const int res = disk_cache_open(path, key_hash, io, hdr, e.tokens, ckpts, n_state);
|
||||
if (res < 0) {
|
||||
SRV_WRN("disk cache: '%s' is truncated or corrupt, removing it\n", path.c_str());
|
||||
std::filesystem::remove(de.path());
|
||||
n_bad++;
|
||||
continue;
|
||||
}
|
||||
if (res == 0) {
|
||||
n_other++;
|
||||
continue;
|
||||
}
|
||||
e.path = path;
|
||||
for (const auto & c : ckpts) {
|
||||
e.ckpt_n_tokens.push_back(c.n_tokens);
|
||||
}
|
||||
entries.push_back(std::move(e));
|
||||
}
|
||||
} catch (const std::exception & err) {
|
||||
SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what());
|
||||
}
|
||||
|
||||
SRV_INF("disk cache: '%s', %zu prompts of this model, %zu other files, %zu bad files removed, limit %.0f MiB\n",
|
||||
dir.c_str(), entries.size(), n_other, n_bad, limit_size / (1024.0 * 1024.0));
|
||||
}
|
||||
|
||||
size_t server_prompt_disk_cache::n_reuse(const llama_tokens & tokens, const std::vector<int64_t> & ckpt_n_tokens, const llama_tokens & tokens_new, bool can_trim) {
|
||||
size_t lcp = 0;
|
||||
while (lcp < tokens.size() && lcp < tokens_new.size() && tokens[lcp] == tokens_new[lcp]) {
|
||||
lcp++;
|
||||
}
|
||||
if (lcp == tokens.size() || can_trim) {
|
||||
return lcp;
|
||||
}
|
||||
size_t res = 0;
|
||||
for (const int64_t n : ckpt_n_tokens) {
|
||||
if (n <= (int64_t) lcp) {
|
||||
res = std::max(res, (size_t) n);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
int server_prompt_disk_cache::find(const llama_tokens & tokens_new, size_t n_have, size_t n_min, bool can_trim) const {
|
||||
int best = -1;
|
||||
size_t n_best = 0;
|
||||
for (size_t i = 0; i < entries.size(); i++) {
|
||||
const size_t n = n_reuse(entries[i].tokens, entries[i].ckpt_n_tokens, tokens_new, can_trim);
|
||||
if (n >= n_have + n_min && n > n_best) {
|
||||
best = (int) i;
|
||||
n_best = n;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
bool server_prompt_disk_cache::has(const llama_tokens & tokens) {
|
||||
for (const auto & e : entries) {
|
||||
std::error_code ec;
|
||||
if (e.tokens == tokens && std::filesystem::exists(e.path, ec)) {
|
||||
disk_cache_touch(e.path);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void server_prompt_disk_cache::remove(int i) {
|
||||
std::error_code ec;
|
||||
std::filesystem::remove(entries[i].path, ec);
|
||||
entries.erase(entries.begin() + i);
|
||||
}
|
||||
|
||||
bool server_prompt_disk_cache::load(int i, server_prompt & prompt, common_state_data & data, bool has_mtmd) {
|
||||
const std::string path = entries[i].path;
|
||||
|
||||
disk_cache_io io;
|
||||
disk_cache_header hdr;
|
||||
llama_tokens tokens;
|
||||
std::vector<disk_cache_ckpt> ckpts;
|
||||
uint64_t n_state = 0;
|
||||
std::list<std::shared_ptr<const common_prompt_checkpoint>> checkpoints;
|
||||
|
||||
bool ok = disk_cache_open(path, key_hash, io, hdr, tokens, ckpts, n_state) == 1 && tokens == entries[i].tokens;
|
||||
if (ok) {
|
||||
try {
|
||||
data.resize(n_state);
|
||||
io.read(data.data(), data.size());
|
||||
for (const auto & c : ckpts) {
|
||||
auto cur = std::make_shared<common_prompt_checkpoint>();
|
||||
cur->id_task = -1; // not created by a task: may fail to load, then the prompt is re-processed
|
||||
cur->n_tokens = c.n_tokens;
|
||||
cur->pos_min = c.pos_min;
|
||||
cur->pos_max = c.pos_max;
|
||||
cur->data_tgt.resize(c.size);
|
||||
io.read(cur->data_tgt.data(), cur->data_tgt.size());
|
||||
checkpoints.push_back(std::move(cur));
|
||||
}
|
||||
} catch (const std::bad_alloc &) {
|
||||
SRV_WRN("disk cache: out of memory reading '%s'\n", path.c_str());
|
||||
data.clear();
|
||||
return false;
|
||||
}
|
||||
ok = io.ok && io.hash == hdr.hash_payload;
|
||||
io.drop_cache();
|
||||
}
|
||||
if (!ok) {
|
||||
SRV_WRN("disk cache: '%s' is missing, truncated or corrupt, removing it\n", path.c_str());
|
||||
data.clear();
|
||||
remove(i);
|
||||
return false;
|
||||
}
|
||||
|
||||
disk_cache_touch(path);
|
||||
|
||||
prompt.tokens = server_tokens(tokens, has_mtmd);
|
||||
prompt.checkpoints = std::move(checkpoints);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool server_prompt_disk_cache::save(const server_prompt & prompt, const common_state_data & data) {
|
||||
// text only, see disk_cache_collect
|
||||
const llama_tokens tokens = prompt.tokens.get_text_tokens();
|
||||
|
||||
std::vector<disk_cache_ckpt> ckpts;
|
||||
for (const auto & c : prompt.checkpoints) {
|
||||
ckpts.push_back({ c->n_tokens, c->pos_min, c->pos_max, c->data_tgt.size() });
|
||||
}
|
||||
|
||||
disk_cache_header hdr = {
|
||||
/*.magic =*/ DISK_CACHE_MAGIC,
|
||||
/*.version =*/ DISK_CACHE_VERSION,
|
||||
/*.key_hash =*/ key_hash,
|
||||
/*.n_tokens =*/ tokens.size(),
|
||||
/*.n_ckpt =*/ ckpts.size(),
|
||||
/*.size_payload =*/ disk_cache_payload_size(tokens.size(), ckpts, data.size()),
|
||||
/*.hash_payload =*/ 0,
|
||||
};
|
||||
const uint64_t size_file = sizeof(hdr) + hdr.size_payload;
|
||||
|
||||
char name[64];
|
||||
snprintf(name, sizeof(name), "%016llx-%016llx", (unsigned long long) key_hash,
|
||||
(unsigned long long) disk_cache_hash_bytes(tokens.data(), tokens.size()*sizeof(llama_token)));
|
||||
const std::string path = (std::filesystem::path(dir) / name).string() + DISK_CACHE_EXT;
|
||||
const std::string path_tmp = path + ".tmp";
|
||||
|
||||
// entries contained in the new one are obsolete: removed once it is written, not counted for the limit
|
||||
const auto is_obsolete = [&](const server_prompt_disk_entry & e) {
|
||||
return e.path != path && e.tokens.size() <= tokens.size() && std::equal(e.tokens.begin(), e.tokens.end(), tokens.begin());
|
||||
};
|
||||
std::vector<std::string> obsolete;
|
||||
for (const auto & e : entries) {
|
||||
if (is_obsolete(e)) {
|
||||
obsolete.push_back(e.path);
|
||||
}
|
||||
}
|
||||
|
||||
if (limit_size > 0 && size_file > limit_size) {
|
||||
SRV_WRN("disk cache: prompt state of %.1f MiB exceeds the limit of %.1f MiB, not saved\n", size_file / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0));
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// evict the least recently used files of every model in dir
|
||||
struct file_info {
|
||||
std::filesystem::path path;
|
||||
uint64_t size;
|
||||
std::filesystem::file_time_type time;
|
||||
};
|
||||
std::vector<file_info> files;
|
||||
uint64_t size_total = 0;
|
||||
for (const auto & de : std::filesystem::directory_iterator(dir)) {
|
||||
if (de.is_regular_file() && de.path().extension() == DISK_CACHE_EXT && de.path().string() != path &&
|
||||
std::find(obsolete.begin(), obsolete.end(), de.path().string()) == obsolete.end()) {
|
||||
files.push_back({ de.path(), de.file_size(), de.last_write_time() });
|
||||
size_total += files.back().size;
|
||||
}
|
||||
}
|
||||
std::sort(files.begin(), files.end(), [](const file_info & a, const file_info & b) { return a.time < b.time; });
|
||||
for (size_t i = 0; limit_size > 0 && size_total + size_file > limit_size && i < files.size(); i++) {
|
||||
SRV_INF("disk cache: size limit, removing '%s' (%.1f MiB)\n", files[i].path.string().c_str(), files[i].size / (1024.0 * 1024.0));
|
||||
std::filesystem::remove(files[i].path);
|
||||
size_total -= files[i].size;
|
||||
for (size_t j = 0; j < entries.size(); j++) {
|
||||
if (entries[j].path == files[i].path.string()) {
|
||||
entries.erase(entries.begin() + j);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
const uint64_t avail = std::filesystem::space(dir).available;
|
||||
if (avail < size_file + DISK_CACHE_MIN_FREE) {
|
||||
SRV_WRN("disk cache: only %.1f MiB free in '%s', prompt state of %.1f MiB not saved\n", avail / (1024.0 * 1024.0), dir.c_str(), size_file / (1024.0 * 1024.0));
|
||||
return false;
|
||||
}
|
||||
} catch (const std::exception & err) {
|
||||
SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what());
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ok;
|
||||
{
|
||||
disk_cache_io io;
|
||||
io.f = fopen(path_tmp.c_str(), "wb");
|
||||
ok = io.f != nullptr && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr);
|
||||
if (ok) {
|
||||
const uint64_t n_state = data.size();
|
||||
io.write(tokens.data(), tokens.size()*sizeof(llama_token));
|
||||
io.write(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt));
|
||||
io.write(&n_state, sizeof(n_state));
|
||||
io.write(data.data(), data.size());
|
||||
for (const auto & c : prompt.checkpoints) {
|
||||
io.write(c->data_tgt.data(), c->data_tgt.size());
|
||||
}
|
||||
hdr.hash_payload = io.hash;
|
||||
ok = io.ok && fseek(io.f, 0, SEEK_SET) == 0 && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr) && fflush(io.f) == 0;
|
||||
#if defined(__linux__)
|
||||
// write back now, so the pages can be dropped and the rename never exposes missing data
|
||||
ok = ok && fdatasync(fileno(io.f)) == 0;
|
||||
io.drop_cache();
|
||||
#endif
|
||||
}
|
||||
if (io.f != nullptr && fclose(io.f) != 0) {
|
||||
ok = false;
|
||||
}
|
||||
io.f = nullptr;
|
||||
if (!ok) {
|
||||
SRV_WRN("disk cache: failed to write '%s': %s, not saved\n", path_tmp.c_str(), strerror(errno));
|
||||
}
|
||||
}
|
||||
std::error_code ec;
|
||||
if (ok) {
|
||||
std::filesystem::rename(path_tmp, path, ec);
|
||||
ok = !ec;
|
||||
if (!ok) {
|
||||
SRV_WRN("disk cache: failed to rename '%s': %s, not saved\n", path_tmp.c_str(), ec.message().c_str());
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
std::filesystem::remove(path_tmp, ec);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < entries.size(); ) {
|
||||
const auto & e = entries[i];
|
||||
if (e.path == path) {
|
||||
entries.erase(entries.begin() + i);
|
||||
} else if (is_obsolete(e)) {
|
||||
SRV_INF("disk cache: removing '%s', a prefix of the new prompt\n", e.path.c_str());
|
||||
remove((int) i);
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
server_prompt_disk_entry e;
|
||||
e.path = path;
|
||||
e.tokens = tokens;
|
||||
for (const auto & c : ckpts) {
|
||||
e.ckpt_n_tokens.push_back(c.n_tokens);
|
||||
}
|
||||
entries.push_back(std::move(e));
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -103,6 +103,42 @@ struct server_prompt_cache {
|
||||
void update();
|
||||
};
|
||||
|
||||
// prompt states saved as files, to reuse a prompt after a server restart
|
||||
struct server_prompt_disk_entry {
|
||||
std::string path;
|
||||
llama_tokens tokens;
|
||||
std::vector<int64_t> ckpt_n_tokens;
|
||||
};
|
||||
|
||||
struct server_prompt_disk_cache {
|
||||
// key: model file and every parameter that changes the state layout, see the server context
|
||||
server_prompt_disk_cache(const std::string & dir, int32_t limit_size_mib, const std::string & key);
|
||||
|
||||
std::string dir;
|
||||
uint64_t key_hash;
|
||||
size_t limit_size;
|
||||
|
||||
// entries of this key only, read from the file headers at startup
|
||||
std::vector<server_prompt_disk_entry> entries;
|
||||
|
||||
// tokens of tokens_new that a state of `tokens` with checkpoints at `ckpt_n_tokens` can reuse
|
||||
// a recurrent or SWA state is reusable only up to an exact prefix or a checkpoint
|
||||
static size_t n_reuse(const llama_tokens & tokens, const std::vector<int64_t> & ckpt_n_tokens, const llama_tokens & tokens_new, bool can_trim);
|
||||
|
||||
// index of the entry that reuses the most of tokens_new, and at least n_min more than n_have; -1 if none
|
||||
int find(const llama_tokens & tokens_new, size_t n_have, size_t n_min, bool can_trim) const;
|
||||
|
||||
bool has(const llama_tokens & tokens);
|
||||
|
||||
// read and verify entry i; a bad file is removed
|
||||
bool load(int i, server_prompt & prompt, common_state_data & data, bool has_mtmd);
|
||||
|
||||
// write atomically, after evicting the oldest files in dir over the size limit
|
||||
bool save(const server_prompt & prompt, const common_state_data & data);
|
||||
|
||||
void remove(int i);
|
||||
};
|
||||
|
||||
// what the scheduler needs to know about a slot
|
||||
struct server_slot_info {
|
||||
int id = -1;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cerrno>
|
||||
#include <clocale>
|
||||
#include <exception>
|
||||
#include <mutex>
|
||||
@@ -24,11 +25,22 @@
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
static std::function<void(int)> shutdown_handler;
|
||||
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
|
||||
|
||||
#if !defined(_WIN32)
|
||||
// The handler can interrupt the start_loop() thread while it holds server_queue::mutex_tasks (e.g.
|
||||
// in the sleeping callbacks, during a VRAM arbiter release or token wait), so it must not run
|
||||
// shutdown_handler itself: terminate() takes that mutex and the thread would deadlock on itself.
|
||||
// It only writes the signal number here; shutdown_handler runs on a dedicated thread.
|
||||
static int shutdown_pipe[2] = { -1, -1 };
|
||||
#endif
|
||||
|
||||
static inline void signal_handler(int signal) {
|
||||
if (is_terminating.test_and_set()) {
|
||||
// in case it hangs, we can force terminate the server by hitting Ctrl+C twice
|
||||
@@ -37,9 +49,48 @@ static inline void signal_handler(int signal) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
if (shutdown_pipe[1] >= 0) {
|
||||
const char sig = (char) signal;
|
||||
const ssize_t res = write(shutdown_pipe[1], &sig, 1); // async-signal-safe
|
||||
(void) res;
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
shutdown_handler(signal);
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
// runs shutdown_handler for signal_handler; joined (after a 0 byte) when the instance returns
|
||||
struct shutdown_thread {
|
||||
std::thread thread;
|
||||
|
||||
void start() {
|
||||
if (pipe(shutdown_pipe) != 0) {
|
||||
return; // fall back to calling shutdown_handler from the signal handler
|
||||
}
|
||||
fcntl(shutdown_pipe[0], F_SETFD, FD_CLOEXEC); // not inherited by spawned tools / MCP servers
|
||||
fcntl(shutdown_pipe[1], F_SETFD, FD_CLOEXEC);
|
||||
thread = std::thread([]() {
|
||||
char sig = 0;
|
||||
while (read(shutdown_pipe[0], &sig, 1) < 0 && errno == EINTR) {}
|
||||
if (sig != 0) {
|
||||
shutdown_handler(sig);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
~shutdown_thread() {
|
||||
if (thread.joinable()) {
|
||||
const char sig = 0;
|
||||
const ssize_t res = write(shutdown_pipe[1], &sig, 1);
|
||||
(void) res;
|
||||
thread.join();
|
||||
}
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
// how the server is run; everything is on when it is the llama-server binary
|
||||
struct server_run_opts {
|
||||
bool install_signal_handlers = true; // off when embedded in llama-cli, which handles them itself
|
||||
@@ -715,7 +766,13 @@ static int server_run_instance(common_params & params, server_role & role, commo
|
||||
ctx_server.terminate();
|
||||
};
|
||||
|
||||
#if !defined(_WIN32)
|
||||
shutdown_thread sig_thread; // declared after ctx_server & co, so joined before they are destroyed
|
||||
#endif
|
||||
if (opts.install_signal_handlers) {
|
||||
#if !defined(_WIN32)
|
||||
sig_thread.start();
|
||||
#endif
|
||||
server_install_signal_handlers();
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user