hexagon: improve buffer allocations and mappings (mmap stacking) (#32144)
* hex-bufs: combine queue and fence buffers * hex-bufs: use n_threads for queue sizing * hex-mmap: optimize buffer stacking at the host to avoid iova fragmentation * hex-mmap: add simply retry for spurios mmap failures * hex-mmap: use eager mode for extended mmap retries
This commit is contained in:
1 parent
fe14cad51b
commit
9daa0aa22b
3 files changed
+239
-160
No files matched your search
@@ -316,6 +316,7 @@ struct ggml_hexagon_session {
|
||||
ggml_hexagon_opbatch* op_batch;
|
||||
ggml_hexagon_opqueue* op_queue;
|
||||
|
||||
std::unordered_set<ggml_hexagon_shared_buffer *> buffers;
|
||||
std::unordered_map<int, std::unique_ptr<ggml_hexagon_shared_buffer>> cloned_buffers;
|
||||
std::unordered_set<ggml_hexagon_session *> virt_peers;
|
||||
std::unordered_set<ggml_hexagon_session *> phys_peers;
|
||||
@@ -337,6 +338,7 @@ struct ggml_hexagon_session {
|
||||
ggml_hexagon_mdev_group mdev;
|
||||
ggml_backend_dev_t dev = nullptr;
|
||||
ggml_backend_hexagon_device_context * dev_ctx = nullptr;
|
||||
ggml_hexagon_shared_buffer * shm_buf = nullptr;
|
||||
ggml_hexagon_fence_buffer * fence_buf = nullptr;
|
||||
|
||||
ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr, uint32_t mdev_idx = 0, uint32_t mdev_count = 0) noexcept(false);
|
||||
@@ -373,8 +375,8 @@ struct ggml_hexagon_session {
|
||||
void flush_peers();
|
||||
void flush_pending(bool all = true);
|
||||
|
||||
ggml_hexagon_shared_buffer * mmap_tensor(const ggml_tensor * t);
|
||||
bool clone_buffer(const ggml_hexagon_shared_buffer*);
|
||||
void mmap_buffers();
|
||||
void clone_buffer(const ggml_hexagon_shared_buffer*);
|
||||
void release_buffer(const ggml_hexagon_shared_buffer*);
|
||||
void unclone_buffer(const ggml_hexagon_shared_buffer*);
|
||||
|
||||
@@ -434,7 +436,7 @@ struct ggml_hexagon_rpcmem_block {
|
||||
int fd = -1;
|
||||
size_t size = 0;
|
||||
|
||||
std::unordered_set<ggml_hexagon_session *> mapped_clones;
|
||||
std::unordered_set<ggml_hexagon_session *> clones;
|
||||
|
||||
ggml_hexagon_rpcmem_block(size_t size) {
|
||||
base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size);
|
||||
@@ -468,16 +470,24 @@ struct ggml_hexagon_shared_buffer {
|
||||
size_t size() const { return mem ? mem->size : 0; }
|
||||
int fd() const { return mem ? mem->fd : -1; }
|
||||
|
||||
void mmap(bool extended = false) {
|
||||
void mmap() {
|
||||
if (!this->mem) return;
|
||||
if (this->mapped) return;
|
||||
|
||||
GGML_ASSERT(!this->pinned || !extended);
|
||||
GGML_ASSERT(!this->pinned || !this->extended);
|
||||
|
||||
this->extended = extended;
|
||||
fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED);
|
||||
fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (this->extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED);
|
||||
|
||||
int err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags);
|
||||
if (err != 0) {
|
||||
if (this->extended) {
|
||||
flags = FASTRPC_MAP_FD_EXTENDED;
|
||||
}
|
||||
HEX_VERBOSE("ggml-hex: %s mmap failed (err 0x%08x), retrying in 100ms...\n", sess->c_name(), (unsigned) err);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags);
|
||||
}
|
||||
|
||||
if (err != 0) {
|
||||
GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(),
|
||||
sess->domain_id, size(), fd(), (unsigned) err);
|
||||
@@ -550,10 +560,10 @@ struct ggml_hexagon_shared_buffer {
|
||||
|
||||
// Clone constructor for cross-session mapping
|
||||
ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) {
|
||||
this->sess = sess;
|
||||
this->mem = other.mem;
|
||||
this->mapped = false;
|
||||
this->pinned = other.pinned;
|
||||
this->sess = sess;
|
||||
this->mem = other.mem;
|
||||
this->mapped = false;
|
||||
this->pinned = false;
|
||||
this->extended = other.extended;
|
||||
}
|
||||
|
||||
@@ -563,19 +573,30 @@ struct ggml_hexagon_shared_buffer {
|
||||
};
|
||||
|
||||
struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
|
||||
uint8_t * data = nullptr;
|
||||
uint32_t slot_count = 0;
|
||||
uint32_t slot_head = 0;
|
||||
std::vector<uint32_t> free_slots;
|
||||
ggml_backend_buffer backend_buffer{};
|
||||
|
||||
ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size)
|
||||
: ggml_hexagon_shared_buffer(sess, size, false /* pinned */),
|
||||
slot_count(size / HTP_FENCE_SLOT_SIZE),
|
||||
static size_t size_of(size_t n_slots) {
|
||||
return n_slots * HTP_FENCE_SLOT_SIZE;
|
||||
}
|
||||
|
||||
ggml_hexagon_fence_buffer(ggml_hexagon_shared_buffer * shm_buf, ggml_backend_buffer_type_t buft, size_t offset, size_t n_slots)
|
||||
: ggml_hexagon_shared_buffer(shm_buf->sess, *shm_buf),
|
||||
data(shm_buf->base() + offset),
|
||||
slot_count(n_slots),
|
||||
slot_head(0) {
|
||||
GGML_ASSERT((uintptr_t) data % HEX_L2_LINE_SIZE == 0);
|
||||
this->mapped = shm_buf->mapped;
|
||||
backend_buffer.buft = buft;
|
||||
backend_buffer.context = static_cast<ggml_hexagon_shared_buffer *>(this);
|
||||
backend_buffer.size = size;
|
||||
mmap(false);
|
||||
backend_buffer.size = size_of(n_slots);
|
||||
}
|
||||
|
||||
~ggml_hexagon_fence_buffer() {
|
||||
this->mapped = false;
|
||||
}
|
||||
|
||||
uint8_t * alloc_slot(uint32_t n_slots = 1) {
|
||||
@@ -583,11 +604,11 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
|
||||
if (n_slots == 1 && !free_slots.empty()) {
|
||||
uint32_t slot = free_slots.back();
|
||||
free_slots.pop_back();
|
||||
ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;
|
||||
ptr = data + (size_t) slot * HTP_FENCE_SLOT_SIZE;
|
||||
} else if (slot_head + n_slots <= slot_count) {
|
||||
uint32_t slot = slot_head;
|
||||
slot_head += n_slots;
|
||||
ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;
|
||||
ptr = data + (size_t) slot * HTP_FENCE_SLOT_SIZE;
|
||||
}
|
||||
if (ptr) {
|
||||
memset(ptr, 0, (size_t) n_slots * HTP_FENCE_SLOT_SIZE);
|
||||
@@ -597,7 +618,7 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {
|
||||
|
||||
void free_slot(void * ptr, uint32_t n_slots = 1) {
|
||||
if (!ptr) return;
|
||||
uint32_t slot = ((uint8_t *) ptr - base()) / HTP_FENCE_SLOT_SIZE;
|
||||
uint32_t slot = ((uint8_t *) ptr - data) / HTP_FENCE_SLOT_SIZE;
|
||||
for (uint32_t i = 0; i < n_slots; i++) {
|
||||
free_slots.push_back(slot + i);
|
||||
}
|
||||
@@ -636,6 +657,7 @@ inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void *
|
||||
|
||||
static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
||||
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context);
|
||||
sbuf->sess->buffers.erase(sbuf);
|
||||
sbuf->sess->unclone_buffer(sbuf);
|
||||
delete sbuf;
|
||||
}
|
||||
@@ -975,31 +997,44 @@ static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_ty
|
||||
GGML_UNUSED(buft);
|
||||
}
|
||||
|
||||
struct ggml_backend_hexagon_alloc_buffer_n_plan_item {
|
||||
size_t size;
|
||||
int first;
|
||||
int last;
|
||||
};
|
||||
|
||||
using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector<ggml_backend_hexagon_alloc_buffer_n_plan_item>;
|
||||
|
||||
static const char * ggml_hexagon_kv_layer_suffix(const struct ggml_tensor * t) {
|
||||
if (strncmp(t->name, "cache_", 6) != 0) {
|
||||
return NULL;
|
||||
}
|
||||
const char * p = strstr(t->name, "_l");
|
||||
if (!p || !isdigit((unsigned char)p[2])) {
|
||||
return NULL;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
struct ggml_backend_hexagon_alloc_unit {
|
||||
size_t size;
|
||||
int first;
|
||||
int last;
|
||||
};
|
||||
|
||||
struct ggml_backend_hexagon_alloc_buffer_n_plan_item {
|
||||
size_t size;
|
||||
std::vector<ggml_backend_hexagon_alloc_unit> units;
|
||||
};
|
||||
|
||||
using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector<ggml_backend_hexagon_alloc_buffer_n_plan_item>;
|
||||
|
||||
static bool ggml_hexagon_tensor_layer(const struct ggml_tensor * t, int & kind, int & layer) {
|
||||
if (strncmp(t->name, "blk.", 4) == 0 && isdigit((unsigned char) t->name[4])) {
|
||||
char * end = NULL;
|
||||
long l = strtol(t->name + 4, &end, 10);
|
||||
if (end && *end == '.') {
|
||||
kind = 0;
|
||||
layer = (int) l;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (strncmp(t->name, "cache_", 6) == 0) {
|
||||
const char * p = strstr(t->name, "_l");
|
||||
if (p && isdigit((unsigned char) p[2])) {
|
||||
char * end = NULL;
|
||||
long l = strtol(p + 2, &end, 10);
|
||||
if (end && *end == '\0') {
|
||||
kind = 1;
|
||||
layer = (int) l;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buffer_n_plan(
|
||||
ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {
|
||||
ggml_backend_hexagon_alloc_buffer_n_plan_t plan;
|
||||
@@ -1012,61 +1047,72 @@ static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buf
|
||||
int i = 0;
|
||||
while (i < n_tensors) {
|
||||
struct ggml_tensor * t = tensors[i];
|
||||
size_t unit_size = 0;
|
||||
int unit_first = i;
|
||||
int unit_last = i + 1;
|
||||
|
||||
if (t->data == NULL && t->view_src == NULL) {
|
||||
unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
|
||||
if (t->data != NULL || t->view_src != NULL) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const char * layer_suffix = ggml_hexagon_kv_layer_suffix(t);
|
||||
size_t unit_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
|
||||
int unit_first = i;
|
||||
int unit_last = i + 1;
|
||||
|
||||
while (unit_last < n_tensors) {
|
||||
int kind = -1, layer = -1;
|
||||
const bool has_layer = ggml_hexagon_tensor_layer(t, kind, layer);
|
||||
|
||||
while (has_layer && unit_last < n_tensors) {
|
||||
struct ggml_tensor * next = tensors[unit_last];
|
||||
|
||||
if (next->view_src != NULL) {
|
||||
if (next->data != NULL || next->view_src != NULL) {
|
||||
unit_last++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (layer_suffix != NULL) {
|
||||
const char * next_suffix = ggml_hexagon_kv_layer_suffix(next);
|
||||
if (next_suffix != NULL && strcmp(layer_suffix, next_suffix) == 0) {
|
||||
if (next->data == NULL) {
|
||||
unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment);
|
||||
}
|
||||
unit_last++;
|
||||
continue;
|
||||
}
|
||||
int next_kind = -1, next_layer = -1;
|
||||
if (!ggml_hexagon_tensor_layer(next, next_kind, next_layer) ||
|
||||
next_kind != kind || next_layer != layer) {
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
size_t next_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment);
|
||||
if (unit_size + next_size > max_size) {
|
||||
break;
|
||||
}
|
||||
|
||||
unit_size += next_size;
|
||||
unit_last++;
|
||||
}
|
||||
|
||||
units.push_back({ unit_size, unit_first, unit_last });
|
||||
i = unit_last;
|
||||
}
|
||||
|
||||
size_t cur_buf_size = 0;
|
||||
int cur_buf_first = 0;
|
||||
|
||||
for (const auto & unit : units) {
|
||||
if (unit.size == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cur_buf_size > 0 && (cur_buf_size + unit.size) > max_size) {
|
||||
plan.push_back({ cur_buf_size, cur_buf_first, unit.first });
|
||||
cur_buf_size = 0;
|
||||
cur_buf_first = unit.first;
|
||||
bool placed = false;
|
||||
for (auto & item : plan) {
|
||||
if (item.size + unit.size <= max_size) {
|
||||
item.size += unit.size;
|
||||
item.units.push_back(unit);
|
||||
placed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!placed) {
|
||||
plan.push_back({ unit.size, { unit } });
|
||||
}
|
||||
|
||||
cur_buf_size += unit.size;
|
||||
}
|
||||
|
||||
if (cur_buf_size > 0) {
|
||||
plan.push_back({ cur_buf_size, cur_buf_first, n_tensors });
|
||||
std::stable_sort(plan.begin(), plan.end(), [](const auto & a, const auto & b) {
|
||||
return a.size > b.size;
|
||||
});
|
||||
|
||||
if (!plan.empty()) {
|
||||
for (const auto & unit : units) {
|
||||
if (unit.size == 0) {
|
||||
plan[0].units.push_back(unit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return plan;
|
||||
@@ -1092,28 +1138,19 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n(
|
||||
struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
|
||||
|
||||
struct ggml_tensor * t_failed = NULL;
|
||||
for (int j = item.first; j < item.last; j++) {
|
||||
struct ggml_tensor * t = tensors[j];
|
||||
if (t->data == NULL) {
|
||||
if (t->view_src == NULL) {
|
||||
for (const auto & unit : item.units) {
|
||||
for (int j = unit.first; j < unit.last; j++) {
|
||||
struct ggml_tensor * t = tensors[j];
|
||||
if (t->data == NULL && t->view_src == NULL) {
|
||||
if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
|
||||
t_failed = t;
|
||||
break;
|
||||
}
|
||||
} else if (t->buffer == NULL) {
|
||||
if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
|
||||
t_failed = t;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (t->view_src != NULL && t->buffer == NULL) {
|
||||
if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
|
||||
t_failed = t;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (t_failed != NULL) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (t_failed != NULL) {
|
||||
GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);
|
||||
@@ -1131,6 +1168,19 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n(
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (int j = 0; j < n_tensors; j++) {
|
||||
struct ggml_tensor * t = tensors[j];
|
||||
if (t->view_src != NULL && t->buffer == NULL) {
|
||||
if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
|
||||
GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t->name);
|
||||
for (ggml_backend_buffer_t b : buffers) {
|
||||
ggml_backend_buffer_free(b);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (buffers.size() == 1) {
|
||||
return buffers[0];
|
||||
}
|
||||
@@ -1200,7 +1250,6 @@ struct ggml_hexagon_opbatch {
|
||||
ggml_hexagon_session* sess;
|
||||
|
||||
std::vector<htp_opnode> ops; // htp_opnode of ops
|
||||
|
||||
std::vector<htp_buf_desc> h_bufs; // htp buffer descriptors
|
||||
std::vector<htp_tensor> h_tens; // htp tensor descriptors
|
||||
std::vector<htp_op_desc> h_ops; // htp op descriptors
|
||||
@@ -1529,45 +1578,45 @@ struct ggml_hexagon_registry {
|
||||
struct ggml_hexagon_opqueue {
|
||||
// Shared buffer for storing batches
|
||||
ggml_hexagon_shared_buffer *shm_buf;
|
||||
size_t shm_blk_size;
|
||||
uint8_t * data;
|
||||
size_t slot_size;
|
||||
size_t depth;
|
||||
|
||||
using opvec = std::vector<htp_opnode>;
|
||||
|
||||
std::vector<opvec> op_cache; // per batch op cache
|
||||
|
||||
ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) {
|
||||
static size_t size_of(size_t batch_size, size_t n_threads, size_t depth = 1) {
|
||||
size_t n_bufs = HTP_OP_MAX_BUFS;
|
||||
size_t n_ops = batch_size;
|
||||
size_t n_tensors = std::min<size_t>(n_ops * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS);
|
||||
|
||||
size_t tr_size = 0;
|
||||
if (opt_profile == 3) {
|
||||
tr_size = sess->n_threads * opt_optrace * sizeof(htp_trace_desc);
|
||||
tr_size = n_threads * opt_optrace * sizeof(htp_trace_desc);
|
||||
}
|
||||
|
||||
shm_blk_size = sizeof(htp_buf_desc) * n_bufs +
|
||||
sizeof(htp_tensor) * n_tensors +
|
||||
sizeof(htp_op_desc) * n_ops +
|
||||
sizeof(htp_prof_desc) * n_ops +
|
||||
tr_size;
|
||||
|
||||
shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */);
|
||||
size_t slot_size = sizeof(htp_buf_desc) * n_bufs +
|
||||
sizeof(htp_tensor) * n_tensors +
|
||||
sizeof(htp_op_desc) * n_ops +
|
||||
sizeof(htp_prof_desc) * n_ops +
|
||||
tr_size;
|
||||
return slot_size * depth;
|
||||
}
|
||||
|
||||
ggml_hexagon_opqueue(ggml_hexagon_shared_buffer *shm_buf, size_t offset, size_t batch_size, size_t depth)
|
||||
: shm_buf(shm_buf),
|
||||
data(shm_buf->base() + offset),
|
||||
slot_size(size_of(batch_size, shm_buf->sess->n_threads)),
|
||||
depth(depth) {
|
||||
op_cache.resize(depth);
|
||||
|
||||
if (opt_verbose) {
|
||||
GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",
|
||||
sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size);
|
||||
GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu slot-size %zu\n",
|
||||
shm_buf->c_name(), batch_size, depth, size_of(batch_size, shm_buf->sess->n_threads, depth), slot_size);
|
||||
}
|
||||
}
|
||||
|
||||
~ggml_hexagon_opqueue() {
|
||||
delete shm_buf;
|
||||
}
|
||||
|
||||
size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; }
|
||||
|
||||
// push new batch
|
||||
bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) {
|
||||
static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8");
|
||||
@@ -1603,13 +1652,13 @@ struct ggml_hexagon_opqueue {
|
||||
req.n_traces = 0;
|
||||
}
|
||||
|
||||
dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size);
|
||||
dbuf.ptr = data + ((size_t) slot * slot_size);
|
||||
dbuf.fd = shm_buf->fd();
|
||||
dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
|
||||
dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base();
|
||||
dbuf.size = b_size + t_size + o_size + p_size + tr_size;
|
||||
|
||||
GGML_ASSERT(dbuf.size <= shm_blk_size);
|
||||
GGML_ASSERT(dbuf.size <= slot_size);
|
||||
|
||||
uint8_t * m_ptr = (uint8_t*) dbuf.ptr;
|
||||
uint8_t * b_ptr = m_ptr; m_ptr += b_size;
|
||||
@@ -1657,7 +1706,7 @@ struct ggml_hexagon_opqueue {
|
||||
}
|
||||
|
||||
const size_t m_size = b_size + t_size + o_size + p_size + tr_size;
|
||||
GGML_ASSERT(m_size <= shm_blk_size);
|
||||
GGML_ASSERT(m_size <= slot_size);
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",
|
||||
shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops,
|
||||
@@ -1775,6 +1824,7 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) {
|
||||
if (op_batch->n_ops < min_ops) { return; }
|
||||
|
||||
op_batch->sort_buffers();
|
||||
this->mmap_buffers();
|
||||
|
||||
htp_opbatch_req req {};
|
||||
dspqueue_buffer dbuf{};
|
||||
@@ -1819,10 +1869,17 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) {
|
||||
}
|
||||
|
||||
void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
|
||||
auto clone_tensor_buffer = [this](const ggml_tensor * t) {
|
||||
auto sbuf = this->mmap_tensor(t);
|
||||
if (!sbuf) return;
|
||||
if (sbuf->sess != this) {
|
||||
auto register_tensor = [&](const ggml_tensor * t) {
|
||||
if (!t) return;
|
||||
auto * sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
|
||||
if (ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
|
||||
sbuf->extended = opt_dma64;
|
||||
}
|
||||
if (sbuf->sess == this) {
|
||||
if (!sbuf->mapped) {
|
||||
this->buffers.insert(sbuf);
|
||||
}
|
||||
} else {
|
||||
this->clone_buffer(sbuf);
|
||||
}
|
||||
for (auto & sub : this->mdev.sessions) {
|
||||
@@ -1830,11 +1887,11 @@ void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {
|
||||
}
|
||||
};
|
||||
|
||||
for (auto t : node.get_inputs()) {
|
||||
clone_tensor_buffer(t);
|
||||
for (const auto * src : node.get_inputs()) {
|
||||
register_tensor(src);
|
||||
}
|
||||
for (auto t : node.get_outputs()) {
|
||||
clone_tensor_buffer(t);
|
||||
for (const auto * dst : node.get_outputs()) {
|
||||
register_tensor(dst);
|
||||
}
|
||||
|
||||
const bool fits_tensors = op_batch->fit_tensors(node);
|
||||
@@ -1945,46 +2002,61 @@ void ggml_hexagon_session::enqueue_allreduce(
|
||||
this->enqueue_op(ar_node);
|
||||
}
|
||||
|
||||
ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) {
|
||||
if (!t) return nullptr;
|
||||
void ggml_hexagon_session::mmap_buffers() {
|
||||
auto collect_session = [](ggml_hexagon_session * sess, std::unordered_set<ggml_hexagon_shared_buffer *> & unmapped) {
|
||||
for (auto * sbuf : sess->buffers) {
|
||||
if (!sbuf->mapped) {
|
||||
unmapped.insert(sbuf);
|
||||
}
|
||||
}
|
||||
for (auto & [fd, clone] : sess->cloned_buffers) {
|
||||
if (!clone->mapped) {
|
||||
unmapped.insert(clone.get());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
|
||||
if (!sbuf->mapped) {
|
||||
const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
|
||||
const bool extended = opt_dma64 && is_weight;
|
||||
sbuf->mmap(extended);
|
||||
std::unordered_set<ggml_hexagon_shared_buffer *> unmapped;
|
||||
collect_session(this, unmapped);
|
||||
for (auto & sub : this->mdev.sessions) {
|
||||
collect_session(sub.get(), unmapped);
|
||||
}
|
||||
|
||||
if (!unmapped.empty()) {
|
||||
std::vector<ggml_hexagon_shared_buffer *> sorted(unmapped.begin(), unmapped.end());
|
||||
std::sort(sorted.begin(), sorted.end(), [](const auto * a, const auto * b) {
|
||||
if (a->size() != b->size()) {
|
||||
return a->size() > b->size();
|
||||
}
|
||||
if (a->fd() != b->fd()) {
|
||||
return a->fd() < b->fd();
|
||||
}
|
||||
return a->sess->domain_id < b->sess->domain_id;
|
||||
});
|
||||
for (auto * sbuf : sorted) {
|
||||
sbuf->mmap();
|
||||
}
|
||||
}
|
||||
return sbuf;
|
||||
}
|
||||
|
||||
bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)
|
||||
void ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)
|
||||
{
|
||||
GGML_ASSERT(sbuf && sbuf->mem);
|
||||
if (sbuf->sess == this) return true;
|
||||
if (sbuf->sess == this) return;
|
||||
|
||||
auto mem = sbuf->mem;
|
||||
int fd = mem->fd;
|
||||
|
||||
GGML_ASSERT(fd >= 0);
|
||||
|
||||
if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true;
|
||||
|
||||
GGML_ASSERT(sbuf->mapped);
|
||||
if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return;
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(),
|
||||
sbuf->c_name(), sbuf->base(), sbuf->size(), fd);
|
||||
|
||||
auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf);
|
||||
try {
|
||||
clone->mmap(sbuf->extended);
|
||||
} catch (const std::exception & exc) {
|
||||
GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what());
|
||||
return false;
|
||||
}
|
||||
|
||||
this->cloned_buffers[fd] = std::move(clone);
|
||||
mem->mapped_clones.insert(this);
|
||||
return true;
|
||||
mem->clones.insert(this);
|
||||
}
|
||||
|
||||
void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) {
|
||||
@@ -2000,14 +2072,14 @@ void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbu
|
||||
auto clone = std::move(it->second);
|
||||
this->cloned_buffers.erase(it);
|
||||
}
|
||||
mem->mapped_clones.erase(this);
|
||||
mem->clones.erase(this);
|
||||
}
|
||||
|
||||
void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) {
|
||||
GGML_ASSERT(sbuf && sbuf->mem);
|
||||
|
||||
auto mem = sbuf->mem;
|
||||
std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end());
|
||||
std::vector<ggml_hexagon_session *> sessions(mem->clones.begin(), mem->clones.end());
|
||||
|
||||
for (auto * sess : sessions) {
|
||||
sess->release_buffer(sbuf);
|
||||
@@ -2276,9 +2348,14 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
|
||||
}
|
||||
|
||||
// Allocate buffers and state for op batching
|
||||
this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);
|
||||
const size_t n_fences = 512;
|
||||
const size_t opq_size = ggml_hexagon_opqueue::size_of(opt_opbatch, this->n_threads, opt_opqueue);
|
||||
const size_t fence_offset = GGML_PAD(opq_size, HEX_L2_LINE_SIZE);
|
||||
const size_t fence_size = ggml_hexagon_fence_buffer::size_of(n_fences);
|
||||
|
||||
this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024);
|
||||
this->shm_buf = new ggml_hexagon_shared_buffer(this, fence_offset + fence_size, true /* pinned */);
|
||||
this->op_queue = new ggml_hexagon_opqueue(this->shm_buf, 0, opt_opbatch, opt_opqueue);
|
||||
this->fence_buf = new ggml_hexagon_fence_buffer(this->shm_buf, &dev_ctx->fence_buffer_type, fence_offset, n_fences);
|
||||
if (this->mdev.count > 1) {
|
||||
this->mdev_fence_slot = this->alloc_fence(this->mdev.count);
|
||||
}
|
||||
@@ -2287,7 +2364,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
|
||||
opt_vmem = ggml_hexagon_measure_max_vmem(this);
|
||||
GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem);
|
||||
}
|
||||
const size_t shm_size = this->op_queue->shm_size();
|
||||
const size_t shm_size = this->shm_buf->size();
|
||||
this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem;
|
||||
|
||||
this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
|
||||
@@ -2328,8 +2405,6 @@ void ggml_hexagon_session::release() noexcept(true) {
|
||||
}
|
||||
}
|
||||
|
||||
delete this->op_batch;
|
||||
delete this->op_queue;
|
||||
for (auto & it : this->cpy_fence_slots) {
|
||||
free_fence((void *) it.second, 1);
|
||||
}
|
||||
@@ -2340,6 +2415,14 @@ void ggml_hexagon_session::release() noexcept(true) {
|
||||
delete this->fence_buf;
|
||||
this->fence_buf = nullptr;
|
||||
}
|
||||
|
||||
delete this->op_batch;
|
||||
delete this->op_queue;
|
||||
delete this->shm_buf;
|
||||
this->op_batch = nullptr;
|
||||
this->op_queue = nullptr;
|
||||
this->shm_buf = nullptr;
|
||||
|
||||
while (!this->cloned_buffers.empty()) {
|
||||
release_buffer(this->cloned_buffers.begin()->second.get());
|
||||
}
|
||||
@@ -2734,19 +2817,12 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_
|
||||
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
|
||||
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
|
||||
|
||||
sess_src->mmap_tensor(src);
|
||||
auto sbuf_dst = sess_dst->mmap_tensor(dst);
|
||||
|
||||
if (!sess_src->clone_buffer(sbuf_dst)) { return false; }
|
||||
|
||||
const uint64_t src_key = ggml_hexagon_session_key(sess_src);
|
||||
auto & fence_slot = sess_dst->cpy_fence_slots[src_key];
|
||||
if (!fence_slot) {
|
||||
fence_slot = (volatile uint32_t *) sess_dst->alloc_fence(1);
|
||||
}
|
||||
|
||||
if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; }
|
||||
|
||||
uint32_t fence_seq = sess_dst->next_fence_seq();
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n",
|
||||
@@ -2766,11 +2842,6 @@ static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_
|
||||
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
|
||||
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
|
||||
|
||||
auto sbuf_src = sess_src->mmap_tensor(src);
|
||||
sess_dst->mmap_tensor(dst);
|
||||
|
||||
if (!sess_dst->clone_buffer(sbuf_src)) { return false; }
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n",
|
||||
sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src));
|
||||
|
||||
@@ -2893,7 +2964,6 @@ static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend
|
||||
return;
|
||||
}
|
||||
|
||||
sess->clone_buffer(hex_event->fence_sess->fence_buf);
|
||||
sess->add_peer(hex_event->sess);
|
||||
sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ true);
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@ struct htp_ops_context {
|
||||
struct htp_context {
|
||||
dspqueue_t dsp_queue;
|
||||
|
||||
struct htp_mmap shm;
|
||||
struct htp_mmap mmap[HTP_MAX_MMAPS];
|
||||
dma_queue_t dma[HTP_MAX_NTHREADS];
|
||||
struct htp_thread_trace trace[HTP_MAX_NTHREADS];
|
||||
|
||||
@@ -1089,7 +1089,12 @@ static int execute_op(struct htp_ops_context * octx) {
|
||||
}
|
||||
|
||||
static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) {
|
||||
b->base = NULL;
|
||||
if (b->fd == ctx->shm.fd) {
|
||||
b->base = ctx->shm.base;
|
||||
return true;
|
||||
}
|
||||
|
||||
b->base = 0;
|
||||
|
||||
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
|
||||
struct htp_mmap *m = ctx->mmap + i;
|
||||
@@ -1203,7 +1208,7 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin
|
||||
|
||||
for (uint32_t i=0; i < n_bufs; i++) {
|
||||
struct htp_buf_desc *b = bufs + i;
|
||||
if (!(b->flags & HTP_BUF_EXTENDED)) {
|
||||
if (!(b->flags & HTP_BUF_EXTENDED) && b->fd != ctx->shm.fd) {
|
||||
b->base = 0;
|
||||
}
|
||||
if (!mmap_buf(ctx, b)) {
|
||||
@@ -1316,6 +1321,9 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
|
||||
n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size);
|
||||
|
||||
// Setup descriptor pointers
|
||||
ctx->shm.fd = dbuf->fd;
|
||||
ctx->shm.base = (uintptr_t) dbuf->ptr - dbuf->offset;
|
||||
|
||||
uint8_t * m_ptr = dbuf->ptr;
|
||||
struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size;
|
||||
struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size;
|
||||
|
||||
Reference in new issue
Block a user