diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 03da443c8..89c9d9a61 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -316,6 +316,7 @@ struct ggml_hexagon_session { ggml_hexagon_opbatch* op_batch; ggml_hexagon_opqueue* op_queue; + std::unordered_set buffers; std::unordered_map> cloned_buffers; std::unordered_set virt_peers; std::unordered_set phys_peers; @@ -337,6 +338,7 @@ struct ggml_hexagon_session { ggml_hexagon_mdev_group mdev; ggml_backend_dev_t dev = nullptr; ggml_backend_hexagon_device_context * dev_ctx = nullptr; + ggml_hexagon_shared_buffer * shm_buf = nullptr; ggml_hexagon_fence_buffer * fence_buf = nullptr; ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr, uint32_t mdev_idx = 0, uint32_t mdev_count = 0) noexcept(false); @@ -373,8 +375,8 @@ struct ggml_hexagon_session { void flush_peers(); void flush_pending(bool all = true); - ggml_hexagon_shared_buffer * mmap_tensor(const ggml_tensor * t); - bool clone_buffer(const ggml_hexagon_shared_buffer*); + void mmap_buffers(); + void clone_buffer(const ggml_hexagon_shared_buffer*); void release_buffer(const ggml_hexagon_shared_buffer*); void unclone_buffer(const ggml_hexagon_shared_buffer*); @@ -434,7 +436,7 @@ struct ggml_hexagon_rpcmem_block { int fd = -1; size_t size = 0; - std::unordered_set mapped_clones; + std::unordered_set clones; ggml_hexagon_rpcmem_block(size_t size) { base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size); @@ -468,16 +470,24 @@ struct ggml_hexagon_shared_buffer { size_t size() const { return mem ? mem->size : 0; } int fd() const { return mem ? mem->fd : -1; } - void mmap(bool extended = false) { + void mmap() { if (!this->mem) return; if (this->mapped) return; - GGML_ASSERT(!this->pinned || !extended); + GGML_ASSERT(!this->pinned || !this->extended); - this->extended = extended; - fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED); + fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (this->extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED); int err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags); + if (err != 0) { + if (this->extended) { + flags = FASTRPC_MAP_FD_EXTENDED; + } + HEX_VERBOSE("ggml-hex: %s mmap failed (err 0x%08x), retrying in 100ms...\n", sess->c_name(), (unsigned) err); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags); + } + if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(), sess->domain_id, size(), fd(), (unsigned) err); @@ -550,10 +560,10 @@ struct ggml_hexagon_shared_buffer { // Clone constructor for cross-session mapping ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) { - this->sess = sess; - this->mem = other.mem; - this->mapped = false; - this->pinned = other.pinned; + this->sess = sess; + this->mem = other.mem; + this->mapped = false; + this->pinned = false; this->extended = other.extended; } @@ -563,19 +573,30 @@ struct ggml_hexagon_shared_buffer { }; struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { + uint8_t * data = nullptr; uint32_t slot_count = 0; uint32_t slot_head = 0; std::vector free_slots; ggml_backend_buffer backend_buffer{}; - ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size) - : ggml_hexagon_shared_buffer(sess, size, false /* pinned */), - slot_count(size / HTP_FENCE_SLOT_SIZE), + static size_t size_of(size_t n_slots) { + return n_slots * HTP_FENCE_SLOT_SIZE; + } + + ggml_hexagon_fence_buffer(ggml_hexagon_shared_buffer * shm_buf, ggml_backend_buffer_type_t buft, size_t offset, size_t n_slots) + : ggml_hexagon_shared_buffer(shm_buf->sess, *shm_buf), + data(shm_buf->base() + offset), + slot_count(n_slots), slot_head(0) { + GGML_ASSERT((uintptr_t) data % HEX_L2_LINE_SIZE == 0); + this->mapped = shm_buf->mapped; backend_buffer.buft = buft; backend_buffer.context = static_cast(this); - backend_buffer.size = size; - mmap(false); + backend_buffer.size = size_of(n_slots); + } + + ~ggml_hexagon_fence_buffer() { + this->mapped = false; } uint8_t * alloc_slot(uint32_t n_slots = 1) { @@ -583,11 +604,11 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { if (n_slots == 1 && !free_slots.empty()) { uint32_t slot = free_slots.back(); free_slots.pop_back(); - ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE; + ptr = data + (size_t) slot * HTP_FENCE_SLOT_SIZE; } else if (slot_head + n_slots <= slot_count) { uint32_t slot = slot_head; slot_head += n_slots; - ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE; + ptr = data + (size_t) slot * HTP_FENCE_SLOT_SIZE; } if (ptr) { memset(ptr, 0, (size_t) n_slots * HTP_FENCE_SLOT_SIZE); @@ -597,7 +618,7 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { void free_slot(void * ptr, uint32_t n_slots = 1) { if (!ptr) return; - uint32_t slot = ((uint8_t *) ptr - base()) / HTP_FENCE_SLOT_SIZE; + uint32_t slot = ((uint8_t *) ptr - data) / HTP_FENCE_SLOT_SIZE; for (uint32_t i = 0; i < n_slots; i++) { free_slots.push_back(slot + i); } @@ -636,6 +657,7 @@ inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) { auto sbuf = static_cast(buffer->context); + sbuf->sess->buffers.erase(sbuf); sbuf->sess->unclone_buffer(sbuf); delete sbuf; } @@ -975,31 +997,44 @@ static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_ty GGML_UNUSED(buft); } -struct ggml_backend_hexagon_alloc_buffer_n_plan_item { - size_t size; - int first; - int last; -}; - -using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector; - -static const char * ggml_hexagon_kv_layer_suffix(const struct ggml_tensor * t) { - if (strncmp(t->name, "cache_", 6) != 0) { - return NULL; - } - const char * p = strstr(t->name, "_l"); - if (!p || !isdigit((unsigned char)p[2])) { - return NULL; - } - return p; -} - struct ggml_backend_hexagon_alloc_unit { size_t size; int first; int last; }; +struct ggml_backend_hexagon_alloc_buffer_n_plan_item { + size_t size; + std::vector units; +}; + +using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector; + +static bool ggml_hexagon_tensor_layer(const struct ggml_tensor * t, int & kind, int & layer) { + if (strncmp(t->name, "blk.", 4) == 0 && isdigit((unsigned char) t->name[4])) { + char * end = NULL; + long l = strtol(t->name + 4, &end, 10); + if (end && *end == '.') { + kind = 0; + layer = (int) l; + return true; + } + } + if (strncmp(t->name, "cache_", 6) == 0) { + const char * p = strstr(t->name, "_l"); + if (p && isdigit((unsigned char) p[2])) { + char * end = NULL; + long l = strtol(p + 2, &end, 10); + if (end && *end == '\0') { + kind = 1; + layer = (int) l; + return true; + } + } + } + return false; +} + static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buffer_n_plan( ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { ggml_backend_hexagon_alloc_buffer_n_plan_t plan; @@ -1012,61 +1047,72 @@ static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buf int i = 0; while (i < n_tensors) { struct ggml_tensor * t = tensors[i]; - size_t unit_size = 0; - int unit_first = i; - int unit_last = i + 1; - - if (t->data == NULL && t->view_src == NULL) { - unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment); + if (t->data != NULL || t->view_src != NULL) { + i++; + continue; } - const char * layer_suffix = ggml_hexagon_kv_layer_suffix(t); + size_t unit_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment); + int unit_first = i; + int unit_last = i + 1; - while (unit_last < n_tensors) { + int kind = -1, layer = -1; + const bool has_layer = ggml_hexagon_tensor_layer(t, kind, layer); + + while (has_layer && unit_last < n_tensors) { struct ggml_tensor * next = tensors[unit_last]; - - if (next->view_src != NULL) { + if (next->data != NULL || next->view_src != NULL) { unit_last++; continue; } - if (layer_suffix != NULL) { - const char * next_suffix = ggml_hexagon_kv_layer_suffix(next); - if (next_suffix != NULL && strcmp(layer_suffix, next_suffix) == 0) { - if (next->data == NULL) { - unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment); - } - unit_last++; - continue; - } + int next_kind = -1, next_layer = -1; + if (!ggml_hexagon_tensor_layer(next, next_kind, next_layer) || + next_kind != kind || next_layer != layer) { + break; } - break; + size_t next_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment); + if (unit_size + next_size > max_size) { + break; + } + + unit_size += next_size; + unit_last++; } units.push_back({ unit_size, unit_first, unit_last }); i = unit_last; } - size_t cur_buf_size = 0; - int cur_buf_first = 0; - for (const auto & unit : units) { if (unit.size == 0) { continue; } - - if (cur_buf_size > 0 && (cur_buf_size + unit.size) > max_size) { - plan.push_back({ cur_buf_size, cur_buf_first, unit.first }); - cur_buf_size = 0; - cur_buf_first = unit.first; + bool placed = false; + for (auto & item : plan) { + if (item.size + unit.size <= max_size) { + item.size += unit.size; + item.units.push_back(unit); + placed = true; + break; + } + } + if (!placed) { + plan.push_back({ unit.size, { unit } }); } - - cur_buf_size += unit.size; } - if (cur_buf_size > 0) { - plan.push_back({ cur_buf_size, cur_buf_first, n_tensors }); + std::stable_sort(plan.begin(), plan.end(), [](const auto & a, const auto & b) { + return a.size > b.size; + }); + + if (!plan.empty()) { + for (const auto & unit : units) { + if (unit.size == 0) { + plan[0].units.push_back(unit); + } + } } return plan; @@ -1092,28 +1138,19 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n( struct ggml_tallocr tallocr = ggml_tallocr_new(buffer); struct ggml_tensor * t_failed = NULL; - for (int j = item.first; j < item.last; j++) { - struct ggml_tensor * t = tensors[j]; - if (t->data == NULL) { - if (t->view_src == NULL) { + for (const auto & unit : item.units) { + for (int j = unit.first; j < unit.last; j++) { + struct ggml_tensor * t = tensors[j]; + if (t->data == NULL && t->view_src == NULL) { if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) { t_failed = t; break; } - } else if (t->buffer == NULL) { - if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { - t_failed = t; - break; - } - } - } else { - if (t->view_src != NULL && t->buffer == NULL) { - if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { - t_failed = t; - break; - } } } + if (t_failed != NULL) { + break; + } } if (t_failed != NULL) { GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name); @@ -1131,6 +1168,19 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n( return NULL; } + for (int j = 0; j < n_tensors; j++) { + struct ggml_tensor * t = tensors[j]; + if (t->view_src != NULL && t->buffer == NULL) { + if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { + GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t->name); + for (ggml_backend_buffer_t b : buffers) { + ggml_backend_buffer_free(b); + } + return NULL; + } + } + } + if (buffers.size() == 1) { return buffers[0]; } @@ -1200,7 +1250,6 @@ struct ggml_hexagon_opbatch { ggml_hexagon_session* sess; std::vector ops; // htp_opnode of ops - std::vector h_bufs; // htp buffer descriptors std::vector h_tens; // htp tensor descriptors std::vector h_ops; // htp op descriptors @@ -1529,45 +1578,45 @@ struct ggml_hexagon_registry { struct ggml_hexagon_opqueue { // Shared buffer for storing batches ggml_hexagon_shared_buffer *shm_buf; - size_t shm_blk_size; + uint8_t * data; + size_t slot_size; size_t depth; using opvec = std::vector; std::vector op_cache; // per batch op cache - ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) { + static size_t size_of(size_t batch_size, size_t n_threads, size_t depth = 1) { size_t n_bufs = HTP_OP_MAX_BUFS; size_t n_ops = batch_size; size_t n_tensors = std::min(n_ops * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS); size_t tr_size = 0; if (opt_profile == 3) { - tr_size = sess->n_threads * opt_optrace * sizeof(htp_trace_desc); + tr_size = n_threads * opt_optrace * sizeof(htp_trace_desc); } - shm_blk_size = sizeof(htp_buf_desc) * n_bufs + - sizeof(htp_tensor) * n_tensors + - sizeof(htp_op_desc) * n_ops + - sizeof(htp_prof_desc) * n_ops + - tr_size; - - shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */); + size_t slot_size = sizeof(htp_buf_desc) * n_bufs + + sizeof(htp_tensor) * n_tensors + + sizeof(htp_op_desc) * n_ops + + sizeof(htp_prof_desc) * n_ops + + tr_size; + return slot_size * depth; + } + ggml_hexagon_opqueue(ggml_hexagon_shared_buffer *shm_buf, size_t offset, size_t batch_size, size_t depth) + : shm_buf(shm_buf), + data(shm_buf->base() + offset), + slot_size(size_of(batch_size, shm_buf->sess->n_threads)), + depth(depth) { op_cache.resize(depth); if (opt_verbose) { - GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", - sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size); + GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu slot-size %zu\n", + shm_buf->c_name(), batch_size, depth, size_of(batch_size, shm_buf->sess->n_threads, depth), slot_size); } } - ~ggml_hexagon_opqueue() { - delete shm_buf; - } - - size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; } - // push new batch bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) { static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8"); @@ -1603,13 +1652,13 @@ struct ggml_hexagon_opqueue { req.n_traces = 0; } - dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size); + dbuf.ptr = data + ((size_t) slot * slot_size); dbuf.fd = shm_buf->fd(); dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base(); dbuf.size = b_size + t_size + o_size + p_size + tr_size; - GGML_ASSERT(dbuf.size <= shm_blk_size); + GGML_ASSERT(dbuf.size <= slot_size); uint8_t * m_ptr = (uint8_t*) dbuf.ptr; uint8_t * b_ptr = m_ptr; m_ptr += b_size; @@ -1657,7 +1706,7 @@ struct ggml_hexagon_opqueue { } const size_t m_size = b_size + t_size + o_size + p_size + tr_size; - GGML_ASSERT(m_size <= shm_blk_size); + GGML_ASSERT(m_size <= slot_size); HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, @@ -1775,6 +1824,7 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) { if (op_batch->n_ops < min_ops) { return; } op_batch->sort_buffers(); + this->mmap_buffers(); htp_opbatch_req req {}; dspqueue_buffer dbuf{}; @@ -1819,10 +1869,17 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) { } void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { - auto clone_tensor_buffer = [this](const ggml_tensor * t) { - auto sbuf = this->mmap_tensor(t); - if (!sbuf) return; - if (sbuf->sess != this) { + auto register_tensor = [&](const ggml_tensor * t) { + if (!t) return; + auto * sbuf = static_cast(t->buffer->context); + if (ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + sbuf->extended = opt_dma64; + } + if (sbuf->sess == this) { + if (!sbuf->mapped) { + this->buffers.insert(sbuf); + } + } else { this->clone_buffer(sbuf); } for (auto & sub : this->mdev.sessions) { @@ -1830,11 +1887,11 @@ void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { } }; - for (auto t : node.get_inputs()) { - clone_tensor_buffer(t); + for (const auto * src : node.get_inputs()) { + register_tensor(src); } - for (auto t : node.get_outputs()) { - clone_tensor_buffer(t); + for (const auto * dst : node.get_outputs()) { + register_tensor(dst); } const bool fits_tensors = op_batch->fit_tensors(node); @@ -1945,46 +2002,61 @@ void ggml_hexagon_session::enqueue_allreduce( this->enqueue_op(ar_node); } -ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) { - if (!t) return nullptr; +void ggml_hexagon_session::mmap_buffers() { + auto collect_session = [](ggml_hexagon_session * sess, std::unordered_set & unmapped) { + for (auto * sbuf : sess->buffers) { + if (!sbuf->mapped) { + unmapped.insert(sbuf); + } + } + for (auto & [fd, clone] : sess->cloned_buffers) { + if (!clone->mapped) { + unmapped.insert(clone.get()); + } + } + }; - auto sbuf = static_cast(t->buffer->context); - if (!sbuf->mapped) { - const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; - const bool extended = opt_dma64 && is_weight; - sbuf->mmap(extended); + std::unordered_set unmapped; + collect_session(this, unmapped); + for (auto & sub : this->mdev.sessions) { + collect_session(sub.get(), unmapped); + } + + if (!unmapped.empty()) { + std::vector sorted(unmapped.begin(), unmapped.end()); + std::sort(sorted.begin(), sorted.end(), [](const auto * a, const auto * b) { + if (a->size() != b->size()) { + return a->size() > b->size(); + } + if (a->fd() != b->fd()) { + return a->fd() < b->fd(); + } + return a->sess->domain_id < b->sess->domain_id; + }); + for (auto * sbuf : sorted) { + sbuf->mmap(); + } } - return sbuf; } -bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf) +void ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf) { GGML_ASSERT(sbuf && sbuf->mem); - if (sbuf->sess == this) return true; + if (sbuf->sess == this) return; auto mem = sbuf->mem; int fd = mem->fd; GGML_ASSERT(fd >= 0); - if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true; - - GGML_ASSERT(sbuf->mapped); + if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return; HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(), sbuf->c_name(), sbuf->base(), sbuf->size(), fd); auto clone = std::make_unique(this, *sbuf); - try { - clone->mmap(sbuf->extended); - } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what()); - return false; - } - this->cloned_buffers[fd] = std::move(clone); - mem->mapped_clones.insert(this); - return true; + mem->clones.insert(this); } void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) { @@ -2000,14 +2072,14 @@ void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbu auto clone = std::move(it->second); this->cloned_buffers.erase(it); } - mem->mapped_clones.erase(this); + mem->clones.erase(this); } void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) { GGML_ASSERT(sbuf && sbuf->mem); auto mem = sbuf->mem; - std::vector sessions(mem->mapped_clones.begin(), mem->mapped_clones.end()); + std::vector sessions(mem->clones.begin(), mem->clones.end()); for (auto * sess : sessions) { sess->release_buffer(sbuf); @@ -2276,9 +2348,14 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n } // Allocate buffers and state for op batching - this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); + const size_t n_fences = 512; + const size_t opq_size = ggml_hexagon_opqueue::size_of(opt_opbatch, this->n_threads, opt_opqueue); + const size_t fence_offset = GGML_PAD(opq_size, HEX_L2_LINE_SIZE); + const size_t fence_size = ggml_hexagon_fence_buffer::size_of(n_fences); - this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024); + this->shm_buf = new ggml_hexagon_shared_buffer(this, fence_offset + fence_size, true /* pinned */); + this->op_queue = new ggml_hexagon_opqueue(this->shm_buf, 0, opt_opbatch, opt_opqueue); + this->fence_buf = new ggml_hexagon_fence_buffer(this->shm_buf, &dev_ctx->fence_buffer_type, fence_offset, n_fences); if (this->mdev.count > 1) { this->mdev_fence_slot = this->alloc_fence(this->mdev.count); } @@ -2287,7 +2364,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n opt_vmem = ggml_hexagon_measure_max_vmem(this); GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem); } - const size_t shm_size = this->op_queue->shm_size(); + const size_t shm_size = this->shm_buf->size(); this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem; this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); @@ -2328,8 +2405,6 @@ void ggml_hexagon_session::release() noexcept(true) { } } - delete this->op_batch; - delete this->op_queue; for (auto & it : this->cpy_fence_slots) { free_fence((void *) it.second, 1); } @@ -2340,6 +2415,14 @@ void ggml_hexagon_session::release() noexcept(true) { delete this->fence_buf; this->fence_buf = nullptr; } + + delete this->op_batch; + delete this->op_queue; + delete this->shm_buf; + this->op_batch = nullptr; + this->op_queue = nullptr; + this->shm_buf = nullptr; + while (!this->cloned_buffers.empty()) { release_buffer(this->cloned_buffers.begin()->second.get()); } @@ -2734,19 +2817,12 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_ auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); - sess_src->mmap_tensor(src); - auto sbuf_dst = sess_dst->mmap_tensor(dst); - - if (!sess_src->clone_buffer(sbuf_dst)) { return false; } - const uint64_t src_key = ggml_hexagon_session_key(sess_src); auto & fence_slot = sess_dst->cpy_fence_slots[src_key]; if (!fence_slot) { fence_slot = (volatile uint32_t *) sess_dst->alloc_fence(1); } - if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; } - uint32_t fence_seq = sess_dst->next_fence_seq(); HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n", @@ -2766,11 +2842,6 @@ static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_ auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); - auto sbuf_src = sess_src->mmap_tensor(src); - sess_dst->mmap_tensor(dst); - - if (!sess_dst->clone_buffer(sbuf_src)) { return false; } - HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); @@ -2893,7 +2964,6 @@ static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend return; } - sess->clone_buffer(hex_event->fence_sess->fence_buf); sess->add_peer(hex_event->sess); sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ true); } diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index 721fa2162..b6b608631 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -82,6 +82,7 @@ struct htp_ops_context { struct htp_context { dspqueue_t dsp_queue; + struct htp_mmap shm; struct htp_mmap mmap[HTP_MAX_MMAPS]; dma_queue_t dma[HTP_MAX_NTHREADS]; struct htp_thread_trace trace[HTP_MAX_NTHREADS]; diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index ad457d1a6..c1df5a164 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -1089,7 +1089,12 @@ static int execute_op(struct htp_ops_context * octx) { } static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) { - b->base = NULL; + if (b->fd == ctx->shm.fd) { + b->base = ctx->shm.base; + return true; + } + + b->base = 0; for (uint32_t i=0; immap + i; @@ -1203,7 +1208,7 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin for (uint32_t i=0; i < n_bufs; i++) { struct htp_buf_desc *b = bufs + i; - if (!(b->flags & HTP_BUF_EXTENDED)) { + if (!(b->flags & HTP_BUF_EXTENDED) && b->fd != ctx->shm.fd) { b->base = 0; } if (!mmap_buf(ctx, b)) { @@ -1316,6 +1321,9 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size); // Setup descriptor pointers + ctx->shm.fd = dbuf->fd; + ctx->shm.base = (uintptr_t) dbuf->ptr - dbuf->offset; + uint8_t * m_ptr = dbuf->ptr; struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size; struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size;