diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 45610c765..cfea16a77 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2253,6 +2253,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } if (ml.no_alloc) { + post_load_tensors(ml); return true; } @@ -2283,6 +2284,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { load_tensors_hadamard(); + post_load_tensors(ml); + return true; } diff --git a/src/llama-model.h b/src/llama-model.h index 87023c1f7..3a0202e69 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -1091,6 +1091,9 @@ struct llama_model { // optional, runs after the scale pass virtual void post_arch_tensors(llama_model_loader &) {} + // optional, runs after the weight buffers are allocated and loaded (with no_alloc: after the dummy buffers are set) + virtual void post_load_tensors(llama_model_loader &) {} + protected: llama_model_params params; diff --git a/src/models/inkling2-loop.cpp b/src/models/inkling2-loop.cpp index 8366a1191..81e3adeb5 100644 --- a/src/models/inkling2-loop.cpp +++ b/src/models/inkling2-loop.cpp @@ -1,5 +1,12 @@ #include "models.h" +#include +#include +#include +#include +#include +#include + int llama_model_inkling2_loop::layer_phys(int il) const { return il < n_phys ? il : core_start + (il - n_phys) % (core_end - core_start); } @@ -69,6 +76,101 @@ void llama_model_inkling2_loop::post_arch_tensors(llama_model_loader &) { } } +// the scheduler makes one device copy of a host weight per graph, alive from its first to its last use, and the expert copy +// callback runs at the first use only: give each later pass its own view, so it gets its own copy with the experts of its own ids +// knob: LLAMA_LOOP_HOST_VIEWS=0 keeps the shared tensors +void llama_model_inkling2_loop::post_load_tensors(llama_model_loader & ml) { + const char * env = getenv("LLAMA_LOOP_HOST_VIEWS"); + if (env != nullptr && strcmp(env, "0") == 0) { + LLAMA_LOG_INFO("%s: LLAMA_LOOP_HOST_VIEWS=0, the loop passes share the host weights\n", __func__); + return; + } + + // llama_layer holds only tensor pointers, see llama_context::opt_init + static_assert(sizeof(llama_layer) % sizeof(ggml_tensor *) == 0, "llama_layer must hold only tensor pointers"); + const size_t n_ptr = sizeof(llama_layer)/sizeof(ggml_tensor *); + + const int n_slot = (int) hparams.n_layer(); + const int n_core = (int) (core_end - core_start); + + // weights on a device are used in place, only host weights are copied + std::vector> host(n_slot); + size_t n_views = 0; + for (int il = n_phys; il < n_slot; ++il) { + ggml_tensor ** ts = reinterpret_cast(&layers[il]); + for (size_t i = 0; i < n_ptr; ++i) { + ggml_tensor * t = ts[i]; + if (t == nullptr || t->buffer == nullptr || !ggml_backend_buffer_is_host(t->buffer) || (!ml.no_alloc && t->data == nullptr)) { + continue; + } + if (std::find(host[il].begin(), host[il].end(), t) == host[il].end()) { + host[il].push_back(t); + } + } + n_views += host[il].size(); + } + if (n_views == 0) { + return; + } + + ggml_init_params init_params = { + /*.mem_size =*/ n_views*ggml_tensor_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx_visits.reset(ggml_init(init_params)); + if (!ctx_visits) { + throw std::runtime_error("inkling2_loop: failed to create the context of the loop views"); + } + + size_t n_bytes = 0; + for (int il = n_phys; il < n_slot; ++il) { + const int pass = 1 + (il - n_phys)/n_core; + + std::unordered_map view_of; + for (ggml_tensor * t : host[il]) { + ggml_tensor * v = ggml_view_tensor(ctx_visits.get(), t); + ggml_format_name(v, "%s.visit%d", t->name, pass); + if (ml.no_alloc) { + // the dummy buffer has no data, it only tells the scheduler that this is a weight + v->buffer = t->buffer; + } else if (ggml_backend_view_init(v) != GGML_STATUS_SUCCESS) { + throw std::runtime_error(format("inkling2_loop: failed to initialize the view %s", v->name)); + } + view_of[t] = v; + n_bytes += ggml_nbytes(t); + + // the per-weight maps are keyed by the tensor + if (const auto it = w_scales.find(t); it != w_scales.end()) { + ggml_tensor * s = it->second; + w_scales.emplace(v, s); + } + if (const auto it = prec_policy.prec_acc.find(t); it != prec_policy.prec_acc.end()) { + const ggml_prec prec = it->second; + prec_policy.prec_acc.emplace(v, prec); + } + if (const auto it = prec_policy.prec_src1.find(t); it != prec_policy.prec_src1.end()) { + const ggml_prec prec = it->second; + prec_policy.prec_src1.emplace(v, prec); + } + if (const auto it = hdmd.rot.find(t); it != hdmd.rot.end()) { + const llama_hadamard_transform tr = it->second; + hdmd.rot.emplace(v, tr); + } + } + + ggml_tensor ** ts = reinterpret_cast(&layers[il]); + for (size_t i = 0; i < n_ptr; ++i) { + if (const auto it = view_of.find(ts[i]); it != view_of.end()) { + ts[i] = it->second; + } + } + } + + LLAMA_LOG_INFO("%s: %zu views of host weights for the loop passes after the first (%.2f MiB, not copied)\n", + __func__, n_views, n_bytes/1024.0/1024.0); +} + std::unique_ptr llama_model_inkling2_loop::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } diff --git a/src/models/models.h b/src/models/models.h index 195b40328..423c94ce5 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2559,6 +2559,7 @@ struct llama_model_inkling2_loop : public llama_model_inkling2 { void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; void post_arch_tensors(llama_model_loader & ml) override; + void post_load_tensors(llama_model_loader & ml) override; int n_layer_phys() const override { return n_phys; } int layer_phys(int il) const; @@ -2573,6 +2574,9 @@ struct llama_model_inkling2_loop : public llama_model_inkling2 { // indexed by physical layer - core_start std::vector loop_layers; + // views of the host weights for the cache slots of the later passes, see post_load_tensors + ggml_context_ptr ctx_visits; + struct graph : public llama_model_inkling2::graph { graph(const llm_graph_params & params); };