From 312c29c2b1655f8c6ca16fdaee194e73b28919c9 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 22 Nov 2026 03:01:46 +0100 Subject: [PATCH] llama: inkling2_loop - per-pass views of host weights for the later loop passes inkling2_loop runs the core layers (6..17 on Inkling-2-Mini) three times, and post_arch_tensors makes every cache slot share the tensors of its physical layer. With the experts in host memory (--n-cpu-moe, -ot exps=CPU) the scheduler makes one device copy per (host tensor, backend) per graph, at the first split that reads it, and keeps it until the last reader. So: - the expert copies of all 12 loop layers are alive at once (about 6.5 GiB on UD-Q4_K_XL), which is most of the ~7 GiB Vulkan0 compute buffer; - sched_copy_experts runs only for the first reader. When it reads the ids back (ubatches of 32-63 tokens: MUL_MAT_ID is offloaded from 32 tokens, the F1 skip starts at 4*n_expert ids), it uploads only the experts that pass 0 selected, and passes 1 and 2 read stale VRAM for the others (upstream issue #32755, "GGGG" output). Add a post_load_tensors hook to llama_model (called after the weights are loaded, and on the no_alloc path after the dummy buffers are set). In inkling2_loop it replaces, for each slot of a later pass, every tensor in a host buffer with a view of it (name ".visit", buffer and data of the original). The scheduler hashes tensors by pointer, so each pass gets its own copy, freed after that pass, and the copy callback sees the ids of that pass. Weights in device buffers and CPU extra buffers (CPU_REPACK, not host) stay shared. The pointer-keyed weight maps (w_scales, prec_policy, hdmd.rot) get entries for the views. The views live in a model-owned context outside ctxs_bufs, so the F4 release/restore, tensor counts and the memory breakdown do not see them; they point only at host buffers, which F4 never releases. LLAMA_LOOP_HOST_VIEWS=0 keeps the shared tensors (A/B knob). Not built or run yet. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ --- src/llama-model.cpp | 3 ++ src/llama-model.h | 3 ++ src/models/inkling2-loop.cpp | 102 +++++++++++++++++++++++++++++++++++ src/models/models.h | 4 ++ 4 files changed, 112 insertions(+) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 45610c765..cfea16a77 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2253,6 +2253,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } if (ml.no_alloc) { + post_load_tensors(ml); return true; } @@ -2283,6 +2284,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { load_tensors_hadamard(); + post_load_tensors(ml); + return true; } diff --git a/src/llama-model.h b/src/llama-model.h index 87023c1f7..3a0202e69 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -1091,6 +1091,9 @@ struct llama_model { // optional, runs after the scale pass virtual void post_arch_tensors(llama_model_loader &) {} + // optional, runs after the weight buffers are allocated and loaded (with no_alloc: after the dummy buffers are set) + virtual void post_load_tensors(llama_model_loader &) {} + protected: llama_model_params params; diff --git a/src/models/inkling2-loop.cpp b/src/models/inkling2-loop.cpp index 8366a1191..81e3adeb5 100644 --- a/src/models/inkling2-loop.cpp +++ b/src/models/inkling2-loop.cpp @@ -1,5 +1,12 @@ #include "models.h" +#include +#include +#include +#include +#include +#include + int llama_model_inkling2_loop::layer_phys(int il) const { return il < n_phys ? il : core_start + (il - n_phys) % (core_end - core_start); } @@ -69,6 +76,101 @@ void llama_model_inkling2_loop::post_arch_tensors(llama_model_loader &) { } } +// the scheduler makes one device copy of a host weight per graph, alive from its first to its last use, and the expert copy +// callback runs at the first use only: give each later pass its own view, so it gets its own copy with the experts of its own ids +// knob: LLAMA_LOOP_HOST_VIEWS=0 keeps the shared tensors +void llama_model_inkling2_loop::post_load_tensors(llama_model_loader & ml) { + const char * env = getenv("LLAMA_LOOP_HOST_VIEWS"); + if (env != nullptr && strcmp(env, "0") == 0) { + LLAMA_LOG_INFO("%s: LLAMA_LOOP_HOST_VIEWS=0, the loop passes share the host weights\n", __func__); + return; + } + + // llama_layer holds only tensor pointers, see llama_context::opt_init + static_assert(sizeof(llama_layer) % sizeof(ggml_tensor *) == 0, "llama_layer must hold only tensor pointers"); + const size_t n_ptr = sizeof(llama_layer)/sizeof(ggml_tensor *); + + const int n_slot = (int) hparams.n_layer(); + const int n_core = (int) (core_end - core_start); + + // weights on a device are used in place, only host weights are copied + std::vector> host(n_slot); + size_t n_views = 0; + for (int il = n_phys; il < n_slot; ++il) { + ggml_tensor ** ts = reinterpret_cast(&layers[il]); + for (size_t i = 0; i < n_ptr; ++i) { + ggml_tensor * t = ts[i]; + if (t == nullptr || t->buffer == nullptr || !ggml_backend_buffer_is_host(t->buffer) || (!ml.no_alloc && t->data == nullptr)) { + continue; + } + if (std::find(host[il].begin(), host[il].end(), t) == host[il].end()) { + host[il].push_back(t); + } + } + n_views += host[il].size(); + } + if (n_views == 0) { + return; + } + + ggml_init_params init_params = { + /*.mem_size =*/ n_views*ggml_tensor_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx_visits.reset(ggml_init(init_params)); + if (!ctx_visits) { + throw std::runtime_error("inkling2_loop: failed to create the context of the loop views"); + } + + size_t n_bytes = 0; + for (int il = n_phys; il < n_slot; ++il) { + const int pass = 1 + (il - n_phys)/n_core; + + std::unordered_map view_of; + for (ggml_tensor * t : host[il]) { + ggml_tensor * v = ggml_view_tensor(ctx_visits.get(), t); + ggml_format_name(v, "%s.visit%d", t->name, pass); + if (ml.no_alloc) { + // the dummy buffer has no data, it only tells the scheduler that this is a weight + v->buffer = t->buffer; + } else if (ggml_backend_view_init(v) != GGML_STATUS_SUCCESS) { + throw std::runtime_error(format("inkling2_loop: failed to initialize the view %s", v->name)); + } + view_of[t] = v; + n_bytes += ggml_nbytes(t); + + // the per-weight maps are keyed by the tensor + if (const auto it = w_scales.find(t); it != w_scales.end()) { + ggml_tensor * s = it->second; + w_scales.emplace(v, s); + } + if (const auto it = prec_policy.prec_acc.find(t); it != prec_policy.prec_acc.end()) { + const ggml_prec prec = it->second; + prec_policy.prec_acc.emplace(v, prec); + } + if (const auto it = prec_policy.prec_src1.find(t); it != prec_policy.prec_src1.end()) { + const ggml_prec prec = it->second; + prec_policy.prec_src1.emplace(v, prec); + } + if (const auto it = hdmd.rot.find(t); it != hdmd.rot.end()) { + const llama_hadamard_transform tr = it->second; + hdmd.rot.emplace(v, tr); + } + } + + ggml_tensor ** ts = reinterpret_cast(&layers[il]); + for (size_t i = 0; i < n_ptr; ++i) { + if (const auto it = view_of.find(ts[i]); it != view_of.end()) { + ts[i] = it->second; + } + } + } + + LLAMA_LOG_INFO("%s: %zu views of host weights for the loop passes after the first (%.2f MiB, not copied)\n", + __func__, n_views, n_bytes/1024.0/1024.0); +} + std::unique_ptr llama_model_inkling2_loop::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } diff --git a/src/models/models.h b/src/models/models.h index 195b40328..423c94ce5 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2559,6 +2559,7 @@ struct llama_model_inkling2_loop : public llama_model_inkling2 { void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; void post_arch_tensors(llama_model_loader & ml) override; + void post_load_tensors(llama_model_loader & ml) override; int n_layer_phys() const override { return n_phys; } int layer_phys(int il) const; @@ -2573,6 +2574,9 @@ struct llama_model_inkling2_loop : public llama_model_inkling2 { // indexed by physical layer - core_start std::vector loop_layers; + // views of the host weights for the cache slots of the later passes, see post_load_tensors + ggml_context_ptr ctx_visits; + struct graph : public llama_model_inkling2::graph { graph(const llm_graph_params & params); };