hexagon: improve buffer allocations and mappings (mmap stacking) (#32144)

* hex-bufs: combine queue and fence buffers

* hex-bufs: use n_threads for queue sizing

* hex-mmap: optimize buffer stacking at the host to avoid iova fragmentation

* hex-mmap: add simply retry for spurios mmap failures

* hex-mmap: use eager mode for extended mmap retries
This commit is contained in:
Max Krasnyansky authored and GitHub committed 2026-11-13 07:56:20 -08:00
1 parent fe14cad51b
commit 9daa0aa22b
3 files changed
+239 -160

No files matched your search

+228 -158
View File
@@ -316,6 +316,7 @@ struct ggml_hexagon_session {
ggml_hexagon_opbatch* op_batch;
ggml_hexagon_opqueue* op_queue;
std::unordered_set<ggml_hexagon_shared_buffer *> buffers;
std::unordered_map<int, std::unique_ptr<ggml_hexagon_shared_buffer>> cloned_buffers;
std::unordered_set<ggml_hexagon_session *> virt_peers;
std::unordered_set<ggml_hexagon_session *> phys_peers;
@@ -337,6 +338,7 @@ struct ggml_hexagon_session {
ggml_hexagon_mdev_group mdev;
ggml_backend_dev_t dev = nullptr;
ggml_backend_hexagon_device_context * dev_ctx = nullptr;
ggml_hexagon_shared_buffer * shm_buf = nullptr;
ggml_hexagon_fence_buffer * fence_buf = nullptr;
ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr, uint32_t mdev_idx = 0, uint32_t mdev_count = 0) noexcept(false);
@@ -373,8 +375,8 @@ struct ggml_hexagon_session {
void flush_peers();
void flush_pending(bool all = true);
ggml_hexagon_shared_buffer * mmap_tensor(const ggml_tensor * t);
bool clone_buffer(const ggml_hexagon_shared_buffer*);
void mmap_buffers();
void clone_buffer(const ggml_hexagon_shared_buffer*);
void release_buffer(const ggml_hexagon_shared_buffer*);
void unclone_buffer(const ggml_hexagon_shared_buffer*);
@@ -434,7 +436,7 @@ struct ggml_hexagon_rpcmem_block {
int fd = -1;
size_t size = 0;
std::unordered_set<ggml_hexagon_session *> mapped_clones;
std::unordered_set<ggml_hexagon_session *> clones;
ggml_hexagon_rpcmem_block(size_t size) {
base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size);
@@ -468,16 +470,24 @@ struct ggml_hexagon_shared_buffer {
size_t size() const { return mem ? mem->size : 0; }
int fd() const { return mem ? mem->fd : -1; }
void mmap(bool extended = false) {
void mmap() {
if (!this->mem) return;
if (this->mapped) return;
GGML_ASSERT(!this->pinned || !extended);
GGML_ASSERT(!this->pinned || !this->extended);
this->extended = extended;
fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED);
fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (this->extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED);
int err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags);
if (err != 0) {
if (this->extended) {
flags = FASTRPC_MAP_FD_EXTENDED;
}
HEX_VERBOSE("ggml-hex: %s mmap failed (err 0x%08x), retrying in 100ms...\n", sess->c_name(), (unsigned) err);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags);
}
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(),
sess->domain_id, size(), fd(), (unsigned) err);
@@ -550,10 +560,10 @@ struct ggml_hexagon_shared_buffer {
// Clone constructor for cross-session mapping
ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) {
this->sess = sess;
this->mem = other.mem;
this->mapped = false;
this->pinned = other.pinned;
this->sess = sess;
this->mem = other.mem;
this->mapped = false;
this->pinned = false;
this->extended = other.extended;
}
@@ -563,19 +573,30 @@ struct ggml_hexagon_shared_buffer {
};
struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
uint8_t * data = nullptr;
uint32_t slot_count = 0;
uint32_t slot_head = 0;
std::vector<uint32_t> free_slots;
ggml_backend_buffer backend_buffer{};
ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size)
: ggml_hexagon_shared_buffer(sess, size, false /* pinned */),
slot_count(size / HTP_FENCE_SLOT_SIZE),
static size_t size_of(size_t n_slots) {
return n_slots * HTP_FENCE_SLOT_SIZE;
}
ggml_hexagon_fence_buffer(ggml_hexagon_shared_buffer * shm_buf, ggml_backend_buffer_type_t buft, size_t offset, size_t n_slots)
: ggml_hexagon_shared_buffer(shm_buf->sess, *shm_buf),
data(shm_buf->base() + offset),
slot_count(n_slots),
slot_head(0) {
GGML_ASSERT((uintptr_t) data % HEX_L2_LINE_SIZE == 0);
this->mapped = shm_buf->mapped;
backend_buffer.buft = buft;
backend_buffer.context = static_cast<ggml_hexagon_shared_buffer *>(this);
backend_buffer.size = size;
mmap(false);
backend_buffer.size = size_of(n_slots);
}
~ggml_hexagon_fence_buffer() {
this->mapped = false;
}
uint8_t * alloc_slot(uint32_t n_slots = 1) {
@@ -583,11 +604,11 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
if (n_slots == 1 && !free_slots.empty()) {
uint32_t slot = free_slots.back();
free_slots.pop_back();
ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;
ptr = data + (size_t) slot * HTP_FENCE_SLOT_SIZE;
} else if (slot_head + n_slots <= slot_count) {
uint32_t slot = slot_head;
slot_head += n_slots;
ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;
ptr = data + (size_t) slot * HTP_FENCE_SLOT_SIZE;
}
if (ptr) {
memset(ptr, 0, (size_t) n_slots * HTP_FENCE_SLOT_SIZE);
@@ -597,7 +618,7 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
void free_slot(void * ptr, uint32_t n_slots = 1) {
if (!ptr) return;
uint32_t slot = ((uint8_t *) ptr - base()) / HTP_FENCE_SLOT_SIZE;
uint32_t slot = ((uint8_t *) ptr - data) / HTP_FENCE_SLOT_SIZE;
for (uint32_t i = 0; i < n_slots; i++) {
free_slots.push_back(slot + i);
}
@@ -636,6 +657,7 @@ inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void *
static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) {
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context);
sbuf->sess->buffers.erase(sbuf);
sbuf->sess->unclone_buffer(sbuf);
delete sbuf;
}
@@ -975,31 +997,44 @@ static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_ty
GGML_UNUSED(buft);
}
struct ggml_backend_hexagon_alloc_buffer_n_plan_item {
size_t size;
int first;
int last;
};
using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector<ggml_backend_hexagon_alloc_buffer_n_plan_item>;
static const char * ggml_hexagon_kv_layer_suffix(const struct ggml_tensor * t) {
if (strncmp(t->name, "cache_", 6) != 0) {
return NULL;
}
const char * p = strstr(t->name, "_l");
if (!p || !isdigit((unsigned char)p[2])) {
return NULL;
}
return p;
}
struct ggml_backend_hexagon_alloc_unit {
size_t size;
int first;
int last;
};
struct ggml_backend_hexagon_alloc_buffer_n_plan_item {
size_t size;
std::vector<ggml_backend_hexagon_alloc_unit> units;
};
using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector<ggml_backend_hexagon_alloc_buffer_n_plan_item>;
static bool ggml_hexagon_tensor_layer(const struct ggml_tensor * t, int & kind, int & layer) {
if (strncmp(t->name, "blk.", 4) == 0 && isdigit((unsigned char) t->name[4])) {
char * end = NULL;
long l = strtol(t->name + 4, &end, 10);
if (end && *end == '.') {
kind = 0;
layer = (int) l;
return true;
}
}
if (strncmp(t->name, "cache_", 6) == 0) {
const char * p = strstr(t->name, "_l");
if (p && isdigit((unsigned char) p[2])) {
char * end = NULL;
long l = strtol(p + 2, &end, 10);
if (end && *end == '\0') {
kind = 1;
layer = (int) l;
return true;
}
}
}
return false;
}
static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buffer_n_plan(
ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {
ggml_backend_hexagon_alloc_buffer_n_plan_t plan;
@@ -1012,61 +1047,72 @@ static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buf
int i = 0;
while (i < n_tensors) {
struct ggml_tensor * t = tensors[i];
size_t unit_size = 0;
int unit_first = i;
int unit_last = i + 1;
if (t->data == NULL && t->view_src == NULL) {
unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
if (t->data != NULL || t->view_src != NULL) {
i++;
continue;
}
const char * layer_suffix = ggml_hexagon_kv_layer_suffix(t);
size_t unit_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
int unit_first = i;
int unit_last = i + 1;
while (unit_last < n_tensors) {
int kind = -1, layer = -1;
const bool has_layer = ggml_hexagon_tensor_layer(t, kind, layer);
while (has_layer && unit_last < n_tensors) {
struct ggml_tensor * next = tensors[unit_last];
if (next->view_src != NULL) {
if (next->data != NULL || next->view_src != NULL) {
unit_last++;
continue;
}
if (layer_suffix != NULL) {
const char * next_suffix = ggml_hexagon_kv_layer_suffix(next);
if (next_suffix != NULL && strcmp(layer_suffix, next_suffix) == 0) {
if (next->data == NULL) {
unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment);
}
unit_last++;
continue;
}
int next_kind = -1, next_layer = -1;
if (!ggml_hexagon_tensor_layer(next, next_kind, next_layer) ||
next_kind != kind || next_layer != layer) {
break;
}
break;
size_t next_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment);
if (unit_size + next_size > max_size) {
break;
}
unit_size += next_size;
unit_last++;
}
units.push_back({ unit_size, unit_first, unit_last });
i = unit_last;
}
size_t cur_buf_size = 0;
int cur_buf_first = 0;
for (const auto & unit : units) {
if (unit.size == 0) {
continue;
}
if (cur_buf_size > 0 && (cur_buf_size + unit.size) > max_size) {
plan.push_back({ cur_buf_size, cur_buf_first, unit.first });
cur_buf_size = 0;
cur_buf_first = unit.first;
bool placed = false;
for (auto & item : plan) {
if (item.size + unit.size <= max_size) {
item.size += unit.size;
item.units.push_back(unit);
placed = true;
break;
}
}
if (!placed) {
plan.push_back({ unit.size, { unit } });
}
cur_buf_size += unit.size;
}
if (cur_buf_size > 0) {
plan.push_back({ cur_buf_size, cur_buf_first, n_tensors });
std::stable_sort(plan.begin(), plan.end(), [](const auto & a, const auto & b) {
return a.size > b.size;
});
if (!plan.empty()) {
for (const auto & unit : units) {
if (unit.size == 0) {
plan[0].units.push_back(unit);
}
}
}
return plan;
@@ -1092,28 +1138,19 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n(
struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
struct ggml_tensor * t_failed = NULL;
for (int j = item.first; j < item.last; j++) {
struct ggml_tensor * t = tensors[j];
if (t->data == NULL) {
if (t->view_src == NULL) {
for (const auto & unit : item.units) {
for (int j = unit.first; j < unit.last; j++) {
struct ggml_tensor * t = tensors[j];
if (t->data == NULL && t->view_src == NULL) {
if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
t_failed = t;
break;
}
} else if (t->buffer == NULL) {
if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
t_failed = t;
break;
}
}
} else {
if (t->view_src != NULL && t->buffer == NULL) {
if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
t_failed = t;
break;
}
}
}
if (t_failed != NULL) {
break;
}
}
if (t_failed != NULL) {
GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);
@@ -1131,6 +1168,19 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n(
return NULL;
}
for (int j = 0; j < n_tensors; j++) {
struct ggml_tensor * t = tensors[j];
if (t->view_src != NULL && t->buffer == NULL) {
if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t->name);
for (ggml_backend_buffer_t b : buffers) {
ggml_backend_buffer_free(b);
}
return NULL;
}
}
}
if (buffers.size() == 1) {
return buffers[0];
}
@@ -1200,7 +1250,6 @@ struct ggml_hexagon_opbatch {
ggml_hexagon_session* sess;
std::vector<htp_opnode> ops; // htp_opnode of ops
std::vector<htp_buf_desc> h_bufs; // htp buffer descriptors
std::vector<htp_tensor> h_tens; // htp tensor descriptors
std::vector<htp_op_desc> h_ops; // htp op descriptors
@@ -1529,45 +1578,45 @@ struct ggml_hexagon_registry {
struct ggml_hexagon_opqueue {
// Shared buffer for storing batches
ggml_hexagon_shared_buffer *shm_buf;
size_t shm_blk_size;
uint8_t * data;
size_t slot_size;
size_t depth;
using opvec = std::vector<htp_opnode>;
std::vector<opvec> op_cache; // per batch op cache
ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) {
static size_t size_of(size_t batch_size, size_t n_threads, size_t depth = 1) {
size_t n_bufs = HTP_OP_MAX_BUFS;
size_t n_ops = batch_size;
size_t n_tensors = std::min<size_t>(n_ops * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS);
size_t tr_size = 0;
if (opt_profile == 3) {
tr_size = sess->n_threads * opt_optrace * sizeof(htp_trace_desc);
tr_size = n_threads * opt_optrace * sizeof(htp_trace_desc);
}
shm_blk_size = sizeof(htp_buf_desc) * n_bufs +
sizeof(htp_tensor) * n_tensors +
sizeof(htp_op_desc) * n_ops +
sizeof(htp_prof_desc) * n_ops +
tr_size;
shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */);
size_t slot_size = sizeof(htp_buf_desc) * n_bufs +
sizeof(htp_tensor) * n_tensors +
sizeof(htp_op_desc) * n_ops +
sizeof(htp_prof_desc) * n_ops +
tr_size;
return slot_size * depth;
}
ggml_hexagon_opqueue(ggml_hexagon_shared_buffer *shm_buf, size_t offset, size_t batch_size, size_t depth)
: shm_buf(shm_buf),
data(shm_buf->base() + offset),
slot_size(size_of(batch_size, shm_buf->sess->n_threads)),
depth(depth) {
op_cache.resize(depth);
if (opt_verbose) {
GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",
sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size);
GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu slot-size %zu\n",
shm_buf->c_name(), batch_size, depth, size_of(batch_size, shm_buf->sess->n_threads, depth), slot_size);
}
}
~ggml_hexagon_opqueue() {
delete shm_buf;
}
size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; }
// push new batch
bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) {
static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8");
@@ -1603,13 +1652,13 @@ struct ggml_hexagon_opqueue {
req.n_traces = 0;
}
dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size);
dbuf.ptr = data + ((size_t) slot * slot_size);
dbuf.fd = shm_buf->fd();
dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base();
dbuf.size = b_size + t_size + o_size + p_size + tr_size;
GGML_ASSERT(dbuf.size <= shm_blk_size);
GGML_ASSERT(dbuf.size <= slot_size);
uint8_t * m_ptr = (uint8_t*) dbuf.ptr;
uint8_t * b_ptr = m_ptr; m_ptr += b_size;
@@ -1657,7 +1706,7 @@ struct ggml_hexagon_opqueue {
}
const size_t m_size = b_size + t_size + o_size + p_size + tr_size;
GGML_ASSERT(m_size <= shm_blk_size);
GGML_ASSERT(m_size <= slot_size);
HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",
shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops,
@@ -1775,6 +1824,7 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) {
if (op_batch->n_ops < min_ops) { return; }
op_batch->sort_buffers();
this->mmap_buffers();
htp_opbatch_req req {};
dspqueue_buffer dbuf{};
@@ -1819,10 +1869,17 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) {
}
void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
auto clone_tensor_buffer = [this](const ggml_tensor * t) {
auto sbuf = this->mmap_tensor(t);
if (!sbuf) return;
if (sbuf->sess != this) {
auto register_tensor = [&](const ggml_tensor * t) {
if (!t) return;
auto * sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
if (ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
sbuf->extended = opt_dma64;
}
if (sbuf->sess == this) {
if (!sbuf->mapped) {
this->buffers.insert(sbuf);
}
} else {
this->clone_buffer(sbuf);
}
for (auto & sub : this->mdev.sessions) {
@@ -1830,11 +1887,11 @@ void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
}
};
for (auto t : node.get_inputs()) {
clone_tensor_buffer(t);
for (const auto * src : node.get_inputs()) {
register_tensor(src);
}
for (auto t : node.get_outputs()) {
clone_tensor_buffer(t);
for (const auto * dst : node.get_outputs()) {
register_tensor(dst);
}
const bool fits_tensors = op_batch->fit_tensors(node);
@@ -1945,46 +2002,61 @@ void ggml_hexagon_session::enqueue_allreduce(
this->enqueue_op(ar_node);
}
ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) {
if (!t) return nullptr;
void ggml_hexagon_session::mmap_buffers() {
auto collect_session = [](ggml_hexagon_session * sess, std::unordered_set<ggml_hexagon_shared_buffer *> & unmapped) {
for (auto * sbuf : sess->buffers) {
if (!sbuf->mapped) {
unmapped.insert(sbuf);
}
}
for (auto & [fd, clone] : sess->cloned_buffers) {
if (!clone->mapped) {
unmapped.insert(clone.get());
}
}
};
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
if (!sbuf->mapped) {
const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
const bool extended = opt_dma64 && is_weight;
sbuf->mmap(extended);
std::unordered_set<ggml_hexagon_shared_buffer *> unmapped;
collect_session(this, unmapped);
for (auto & sub : this->mdev.sessions) {
collect_session(sub.get(), unmapped);
}
if (!unmapped.empty()) {
std::vector<ggml_hexagon_shared_buffer *> sorted(unmapped.begin(), unmapped.end());
std::sort(sorted.begin(), sorted.end(), [](const auto * a, const auto * b) {
if (a->size() != b->size()) {
return a->size() > b->size();
}
if (a->fd() != b->fd()) {
return a->fd() < b->fd();
}
return a->sess->domain_id < b->sess->domain_id;
});
for (auto * sbuf : sorted) {
sbuf->mmap();
}
}
return sbuf;
}
bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)
void ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)
{
GGML_ASSERT(sbuf && sbuf->mem);
if (sbuf->sess == this) return true;
if (sbuf->sess == this) return;
auto mem = sbuf->mem;
int fd = mem->fd;
GGML_ASSERT(fd >= 0);
if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true;
GGML_ASSERT(sbuf->mapped);
if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return;
HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(),
sbuf->c_name(), sbuf->base(), sbuf->size(), fd);
auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf);
try {
clone->mmap(sbuf->extended);
} catch (const std::exception & exc) {
GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what());
return false;
}
this->cloned_buffers[fd] = std::move(clone);
mem->mapped_clones.insert(this);
return true;
mem->clones.insert(this);
}
void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) {
@@ -2000,14 +2072,14 @@ void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbu
auto clone = std::move(it->second);
this->cloned_buffers.erase(it);
}
mem->mapped_clones.erase(this);
mem->clones.erase(this);
}
void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) {
GGML_ASSERT(sbuf && sbuf->mem);
auto mem = sbuf->mem;
std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end());
std::vector<ggml_hexagon_session *> sessions(mem->clones.begin(), mem->clones.end());
for (auto * sess : sessions) {
sess->release_buffer(sbuf);
@@ -2276,9 +2348,14 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
}
// Allocate buffers and state for op batching
this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);
const size_t n_fences = 512;
const size_t opq_size = ggml_hexagon_opqueue::size_of(opt_opbatch, this->n_threads, opt_opqueue);
const size_t fence_offset = GGML_PAD(opq_size, HEX_L2_LINE_SIZE);
const size_t fence_size = ggml_hexagon_fence_buffer::size_of(n_fences);
this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024);
this->shm_buf = new ggml_hexagon_shared_buffer(this, fence_offset + fence_size, true /* pinned */);
this->op_queue = new ggml_hexagon_opqueue(this->shm_buf, 0, opt_opbatch, opt_opqueue);
this->fence_buf = new ggml_hexagon_fence_buffer(this->shm_buf, &dev_ctx->fence_buffer_type, fence_offset, n_fences);
if (this->mdev.count > 1) {
this->mdev_fence_slot = this->alloc_fence(this->mdev.count);
}
@@ -2287,7 +2364,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
opt_vmem = ggml_hexagon_measure_max_vmem(this);
GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem);
}
const size_t shm_size = this->op_queue->shm_size();
const size_t shm_size = this->shm_buf->size();
this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem;
this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
@@ -2328,8 +2405,6 @@ void ggml_hexagon_session::release() noexcept(true) {
}
}
delete this->op_batch;
delete this->op_queue;
for (auto & it : this->cpy_fence_slots) {
free_fence((void *) it.second, 1);
}
@@ -2340,6 +2415,14 @@ void ggml_hexagon_session::release() noexcept(true) {
delete this->fence_buf;
this->fence_buf = nullptr;
}
delete this->op_batch;
delete this->op_queue;
delete this->shm_buf;
this->op_batch = nullptr;
this->op_queue = nullptr;
this->shm_buf = nullptr;
while (!this->cloned_buffers.empty()) {
release_buffer(this->cloned_buffers.begin()->second.get());
}
@@ -2734,19 +2817,12 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
sess_src->mmap_tensor(src);
auto sbuf_dst = sess_dst->mmap_tensor(dst);
if (!sess_src->clone_buffer(sbuf_dst)) { return false; }
const uint64_t src_key = ggml_hexagon_session_key(sess_src);
auto & fence_slot = sess_dst->cpy_fence_slots[src_key];
if (!fence_slot) {
fence_slot = (volatile uint32_t *) sess_dst->alloc_fence(1);
}
if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; }
uint32_t fence_seq = sess_dst->next_fence_seq();
HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n",
@@ -2766,11 +2842,6 @@ static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
auto sbuf_src = sess_src->mmap_tensor(src);
sess_dst->mmap_tensor(dst);
if (!sess_dst->clone_buffer(sbuf_src)) { return false; }
HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n",
sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src));
@@ -2893,7 +2964,6 @@ static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend
return;
}
sess->clone_buffer(hex_event->fence_sess->fence_buf);
sess->add_peer(hex_event->sess);
sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ true);
}
+1
View File
@@ -82,6 +82,7 @@ struct htp_ops_context {
struct htp_context {
dspqueue_t dsp_queue;
struct htp_mmap shm;
struct htp_mmap mmap[HTP_MAX_MMAPS];
dma_queue_t dma[HTP_MAX_NTHREADS];
struct htp_thread_trace trace[HTP_MAX_NTHREADS];
+10 -2
View File
@@ -1089,7 +1089,12 @@ static int execute_op(struct htp_ops_context * octx) {
}
static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) {
b->base = NULL;
if (b->fd == ctx->shm.fd) {
b->base = ctx->shm.base;
return true;
}
b->base = 0;
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
struct htp_mmap *m = ctx->mmap + i;
@@ -1203,7 +1208,7 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin
for (uint32_t i=0; i < n_bufs; i++) {
struct htp_buf_desc *b = bufs + i;
if (!(b->flags & HTP_BUF_EXTENDED)) {
if (!(b->flags & HTP_BUF_EXTENDED) && b->fd != ctx->shm.fd) {
b->base = 0;
}
if (!mmap_buf(ctx, b)) {
@@ -1316,6 +1321,9 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size);
// Setup descriptor pointers
ctx->shm.fd = dbuf->fd;
ctx->shm.base = (uintptr_t) dbuf->ptr - dbuf->offset;
uint8_t * m_ptr = dbuf->ptr;
struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size;
struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size;