diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 045489df7..80a719670 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -835,7 +835,7 @@ const bool llama_mlock::SUPPORTED = false; #endif void llama_prefetch(llama_memory_ranges mr) { -#if defined(__linux__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602) +#if defined(__linux__) || defined(__APPLE__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602) if (mr.empty()) { return; } @@ -874,6 +874,7 @@ void llama_prefetch(llama_memory_ranges mr) { }; #else auto prefetch = [&]() { + // TODO: in a follow-up, use posix_madvise(.., POSIX_MADV_WILLNEED) to match the rest of this file if (madvise((void *) begin, end - begin, MADV_WILLNEED) != 0) { LLAMA_LOG_WARN("llama_prefetch: madvise(MADV_WILLNEED) failed: %s\n", strerror(errno)); return false; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 6cc0b2028..bc93c6f9e 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -13,6 +13,7 @@ #include #include #include +#include static const size_t kiB = 1024; static const size_t MiB = 1024*kiB; @@ -1446,21 +1447,80 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps } } -void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const { - GGML_ASSERT(!mappings.empty()); - const auto & mapping = mappings.at(idx); +using mapping_ranges = std::vector; + +// merge a list of [first, last) ranges into a sorted, non-overlapping list +static mapping_ranges merge_ranges(mapping_ranges ranges) { + std::sort(ranges.begin(), ranges.end(), [](const auto & a, const auto & b) { return a.first < b.first; }); + + mapping_ranges res; + for (const auto & range : ranges) { + if (!res.empty() && range.first <= res.back().last) { + res.back().last = std::max(res.back().last, range.last); + } else { + res.push_back(range); + } + } + + return res; +} + +// subtract the sorted, non-overlapping ranges 'sub' from 'a' +static mapping_ranges sub_ranges(const mapping_ranges & a, const mapping_ranges & sub) { + mapping_ranges res; + + for (const auto & x : a) { + size_t cur = x.first; + for (const auto & y : sub) { + if (y.last <= cur || y.first >= x.last) { + continue; + } + if (y.first > cur) { + res.push_back({ cur, y.first }); + } + cur = std::max(cur, y.last); + } + if (cur < x.last) { + res.push_back({ cur, x.last }); + } + } + + return res; +} + +std::vector llama_model_loader::get_mapping_ranges(uint32_t idx, ggml_context * ctx) const { + GGML_ASSERT(idx < mappings.size()); + + // data of this context, and data of the other contexts that share the same file + // a weight that is mapped in several contexts (e.g. the token embedding) is not foreign data + std::unordered_set own; + mapping_ranges own_ranges; + mapping_ranges other; - *first = mapping->size(); - *last = 0; - *addr = mapping->addr(); for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) { const auto * weight = get_weight(ggml_get_name(tensor)); if (!weight || weight->idx != idx) { continue; } - *first = std::min(*first, weight->offs); - *last = std::max(*last, weight->offs + ggml_nbytes(tensor)); + own.insert(weight); + own_ranges.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) }); } + + for (const auto & it : ctx_map) { + if (!it.second || it.second.get() == ctx) { + continue; + } + + for (ggml_tensor * tensor = ggml_get_first_tensor(it.second.get()); tensor; tensor = ggml_get_next_tensor(it.second.get(), tensor)) { + const auto * weight = get_weight(ggml_get_name(tensor)); + if (!weight || weight->idx != idx || own.count(weight)) { + continue; + } + other.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) }); + } + } + + return sub_ranges(merge_ranges(std::move(own_ranges)), merge_ranges(std::move(other))); } void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const { @@ -1492,7 +1552,7 @@ const void * llama_model_loader::load_data_range(const llama_tensor_weight & w, bool llama_model_loader::load_all_data( struct ggml_context * ctx, - llama_buf_map & bufs, + std::vector & bufs, llama_mlocks * lmlocks, llama_progress_callback progress_callback, void * progress_callback_user_data) { @@ -1528,7 +1588,7 @@ bool llama_model_loader::load_all_data( } // When not using mmaped io use async uploads from pinned memory to GPU memory. // First determine if the backend supports the necessary features for async uploads. - auto * buf = bufs.count(0) ? bufs.at(0) : nullptr; + auto * buf = bufs.empty() ? nullptr : bufs[0]; if (!buf) { LLAMA_LOG_DEBUG("%s: no buffer found for async uploads\n", func); return nullptr; @@ -1599,7 +1659,7 @@ bool llama_model_loader::load_all_data( if (upload_backend) { LLAMA_LOG_DEBUG("%s: using async uploads for device %s, buffer type %s, backend %s\n", __func__, ggml_backend_dev_name(ggml_backend_get_device(upload_backend)), - ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))), + ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs[0])), ggml_backend_name(upload_backend)); } @@ -1640,12 +1700,18 @@ bool llama_model_loader::load_all_data( if (from_mapping) { const auto & mapping = mappings.at(weight->idx); - ggml_backend_buffer_t buf_mmap = nullptr; - if (bufs.count(weight->idx)) { - buf_mmap = bufs.at(weight->idx); - } uint8_t * data = (uint8_t *) mapping->addr() + weight->offs; + // the tensors of a file can be spread across several buffers - find the one containing this tensor's data + ggml_backend_buffer_t buf_mmap = nullptr; + for (const auto & buf : bufs) { + const uint8_t * base = (const uint8_t *) ggml_backend_buffer_get_base(buf); + if (base && base <= data && data + n_size <= base + ggml_backend_buffer_get_size(buf)) { + buf_mmap = buf; + break; + } + } + if (check_tensors) { validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] { return std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size)); diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 9e51d0ce7..ed59079a4 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -16,8 +16,6 @@ #include #include -using llama_buf_map = std::unordered_map; - // lists of buffer types used for each layer using buft_list_t = std::vector>; @@ -242,7 +240,16 @@ struct llama_model_loader { void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr); - void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const; + // byte range within a mapped file + struct mapping_range { + size_t first; // inclusive + size_t last; // exclusive + }; + + // sorted, non-overlapping ranges of file 'idx' that back the tensors of 'ctx' + // ranges owned by other contexts are excluded, so that a buffer wrapping them won't pin + // data that this context never reads (lazy tensors, -cmoe tensors, ...) + std::vector get_mapping_ranges(uint32_t idx, ggml_context * ctx) const; // release a weight's mmap pages void unmap_weight(const llama_tensor_weight & w) const; @@ -254,7 +261,7 @@ struct llama_model_loader { // Returns false if cancelled by progress_callback bool load_all_data( struct ggml_context * ctx, - llama_buf_map & bufs, + std::vector & bufs, llama_mlocks * lmlocks, llama_progress_callback progress_callback, void * progress_callback_user_data); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 7072d029c..aeab7b75b 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1875,12 +1875,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { pimpl->mappings.reserve(ml.mappings.size()); // create the backend buffers - std::vector> ctx_buf_maps; - ctx_buf_maps.reserve(ml.ctx_map.size()); + // note: the buffers are owned by pimpl->ctxs_bufs, this is a raw-pointer view used to load the data + std::vector>> ctx_bufs; + ctx_bufs.reserve(ml.ctx_map.size()); - // Ensure we have enough capacity for the maximum backend buffer we will potentially create - const size_t n_max_backend_buffer = ml.ctx_map.size() * ml.files.size(); - pimpl->ctxs_bufs.reserve(n_max_backend_buffer); + // reserve hint: each context creates one buffer per file, or more when its tensors are spread + // across disjoint ranges of a file (see get_mapping_ranges) + pimpl->ctxs_bufs.reserve(ml.ctx_map.size() * ml.files.size()); for (auto & [ctx_key, ctx_ptr] : ml.ctx_map) { ggml_backend_buffer_type_t buft = ctx_key.buft; @@ -1891,9 +1892,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { continue; } - llama_buf_map buf_map; - buf_map.reserve(n_max_backend_buffer); - // check if it is possible to use buffer_from_host_ptr with this buffer type ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft); if (!dev) { @@ -1915,24 +1913,24 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) { GGML_ASSERT(!ml.no_alloc); + const size_t max_size = ggml_get_max_tensor_size(ctx); + for (uint32_t idx = 0; idx < ml.files.size(); idx++) { - // only the mmap region containing the tensors in the model is mapped to the backend buffer - // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, - // then we could just use metal for all layers - // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size - void * addr = nullptr; - size_t first, last; // NOLINT - ml.get_mapping_range(&first, &last, &addr, idx, ctx); - if (first >= last) { - continue; + // map the mmap regions that contain the tensors of this context + // this is important for metal with apple silicon: a backend buffer pins everything it + // wraps, so the regions owned by the other contexts (lazy tensors read on demand + // from disk, MoE tensors kept on the CPU, etc) are left out and stay out of RAM + // this also allows using partial offloading when the model size exceeds the metal buffer + // size, but not the RAM size + void * addr = ml.mappings.at(idx)->addr(); + + for (const auto & range : ml.get_mapping_ranges(idx, ctx)) { + ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + range.first, range.last - range.first, max_size); + if (buf == nullptr) { + throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft))); + } + bufs.emplace_back(buf); } - const size_t max_size = ggml_get_max_tensor_size(ctx); - ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + first, last - first, max_size); - if (buf == nullptr) { - throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft))); - } - bufs.emplace_back(buf); - buf_map.emplace(idx, buf); } } else { ggml_backend_buffer_t buf; @@ -1954,20 +1952,22 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { mlock_buf->grow_to(ggml_backend_buffer_get_size(buf)); } bufs.emplace_back(buf); - for (uint32_t idx = 0; idx < ml.files.size(); idx++) { - buf_map.emplace(idx, buf); - } } + std::vector buf_ptrs; + buf_ptrs.reserve(bufs.size()); + for (auto & buf : bufs) { // indicate that this buffer contains weights // this is used by ggml_backend_sched to improve op scheduling: ops that use a weight are preferably scheduled to the backend that contains the weight ggml_backend_buffer_set_usage(buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + buf_ptrs.emplace_back(buf.get()); } pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs)); - ctx_buf_maps.emplace_back(ctx, buf_map); + ctx_bufs.emplace_back(ctx, std::move(buf_ptrs)); } if (llama_supports_gpu_offload()) { @@ -2000,15 +2000,15 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { // without mmap, load non-host buffers first: their tensors go through a staging buffer, which is cheapest while the fewest weights are resident if (!ml.use_mmap) { - std::stable_partition(ctx_buf_maps.begin(), ctx_buf_maps.end(), [](const auto & ctx_buf_map) { - const auto & buf_map = ctx_buf_map.second; - return !buf_map.empty() && !ggml_backend_buffer_is_host(buf_map.begin()->second); + std::stable_partition(ctx_bufs.begin(), ctx_bufs.end(), [](const auto & x) { + const auto & bufs = x.second; + return !bufs.empty() && !ggml_backend_buffer_is_host(bufs[0]); }); } // load tensor data - for (auto & [ctx, buf_map] : ctx_buf_maps) { - if (!ml.load_all_data(ctx, buf_map, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) { + for (auto & [ctx, bufs] : ctx_bufs) { + if (!ml.load_all_data(ctx, bufs, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) { return false; } }