hexagon: Implement set_tensor_n and improve repacking/tiling robustness (#31744)

* hex-repack: move repacking/tiling into graph-optimize

* hex-tiling: simplify tiling rules and remove obsolete helpers

* hex-ops: use a hash for de-duplicating tensor descriptors

* hex-bufs: improve tensor hash

* hex-ops: accept empty (zero-elements) ops

* hex-bufs: enable hostbuf by default and reject tensors that do not fit into max_vmem

* hex-bufs: improve error message and update readme

* hex-bufs: make sure to sync flush opbatch in all sync set_tensor paths
This commit is contained in:
Max Krasnyansky authored and GitHub committed 2026-11-05 18:15:58 -08:00
1 parent e14a779719
commit 67107ac34f
6 files changed
+231 -222

No files matched your search

+7 -3
View File
@@ -325,10 +325,14 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs
Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version).
- `GGML_HEXAGON_VTCM_SIZE=0`
Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB.
The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled)
Enables allocating host buffers for debugging. By default, host buffers are disabled.
- `GGML_HEXAGON_HOSTBUF=0` (default: 1, enabled)
Controls host buffer (`HTP*-HOST`) allocation in shared FastRPC memory to avoid host-device tensor copies.
CPU-assigned weights (input embeddings like `token_embd` and `per_layer_token_embd`, and tensors overridden with `-ot ...=CPU`)
are placed in host buffers as well. On older drivers/FW, large models can run out of IOVA space when these are mapped.
Pass `--no-host` to keep CPU-assigned weights in regular CPU memory (`-ot ...=CPU` alone is not sufficient).
- `GGML_HEXAGON_DMA64=0` (default: enabled on v81+)
Disables 64-bit DMA for model weights. Set to `1` to enable it explicitly on a supported architecture.
+169 -156
View File
@@ -90,7 +90,7 @@ size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for
int opt_etm = 0;
int opt_verbose = 0;
int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)
bool opt_hostbuf = false;
bool opt_hostbuf = true;
bool opt_dma64 = false;
int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start
@@ -150,7 +150,7 @@ int opt_opqueue = 8; // max number of pending batches
int opt_optrace = 0; // trace buffer size per thread (0 means default)
int opt_oppoll = 0; // polling for batch completions
int opt_opfusion = 1; // enable/disable op fusion
int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = always
int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = all tileable weights (default)
static std::regex* opt_opfilter = NULL; // regex of ops to not claim
@@ -463,7 +463,6 @@ struct ggml_hexagon_rpcmem_block {
struct ggml_hexagon_shared_buffer {
ggml_hexagon_session * sess;
std::shared_ptr<ggml_hexagon_rpcmem_block> mem;
std::vector<htp_tensor_extra *> tensor_extra;
bool mapped;
bool pinned;
bool extended;
@@ -486,6 +485,9 @@ struct ggml_hexagon_shared_buffer {
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(),
sess->domain_id, size(), fd(), (unsigned) err);
if (!this->pinned) {
GGML_LOG_ERROR("ggml-hex: %s possibly out of IOVA space, try --no-host\n", sess->c_name());
}
throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)");
}
@@ -561,9 +563,6 @@ struct ggml_hexagon_shared_buffer {
~ggml_hexagon_shared_buffer() {
free();
for (auto * extra : tensor_extra) {
delete extra;
}
}
};
@@ -622,11 +621,9 @@ inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) {
}
inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * slot, uint32_t n_slots) const {
static htp_tensor_extra fence_extra { {}, 0, HTP_TENSOR_FENCE };
ggml_tensor t {};
t.buffer = &this->fence_buf->backend_buffer;
t.extra = &fence_extra;
t.extra = (void *) (uintptr_t) HTP_TENSOR_FENCE;
t.data = const_cast<void *>(slot);
t.type = GGML_TYPE_I8;
t.ne[0] = HTP_FENCE_SLOT_SIZE;
@@ -659,10 +656,7 @@ static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buf
HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(),
tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor));
auto extra = new htp_tensor_extra();
sbuf->tensor_extra.push_back(extra);
tensor->extra = extra;
tensor->extra = (void *) (uintptr_t) 0;
return GGML_STATUS_SUCCESS;
}
@@ -672,42 +666,19 @@ static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,
const void * data,
size_t offset,
size_t size) {
auto extra = (htp_tensor_extra *) tensor->extra;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
sess->flush_sync();
if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
extra->flags |= HTP_TENSOR_WEIGHT;
if (htp_is_tiled_weight_tensor(tensor)) {
extra->flags |= HTP_TENSOR_TILED;
}
ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT);
}
HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor));
if ((extra->flags & HTP_TENSOR_TILED) == 0) {
memcpy((char *) tensor->data + offset, data, size);
return;
}
if (offset == 0 && size == ggml_nbytes(tensor) && extra->shadow_buf.empty()) {
htp_repack_tensor_tiled(tensor, data, size);
return;
}
if (extra->shadow_buf.size() < ggml_nbytes(tensor)) {
extra->shadow_buf.resize(ggml_nbytes(tensor));
}
memcpy(extra->shadow_buf.data() + offset, data, size);
extra->shadow_size += size;
if (extra->shadow_size >= ggml_nbytes(tensor)) {
htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size());
extra->shadow_buf.clear();
extra->shadow_buf.shrink_to_fit();
extra->shadow_size = 0;
}
memcpy((char *) tensor->data + offset, data, size);
}
static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
@@ -715,14 +686,14 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
void * data,
size_t offset,
size_t size) {
auto extra = (htp_tensor_extra *) tensor->extra;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor);
HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, flags);
if ((extra->flags & HTP_TENSOR_TILED) == 0) {
if ((flags & HTP_TENSOR_TILED) == 0) {
memcpy(data, (const char *) tensor->data + offset, size);
return;
}
@@ -742,47 +713,27 @@ static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t bu
}
static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,
ggml_tensor * tensor,
const void * data,
size_t offset,
size_t size,
size_t n_copies,
size_t stride_tensor,
size_t stride_data) {
auto extra = (htp_tensor_extra *) tensor->extra;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
ggml_tensor * tensor,
const void * data,
size_t offset,
size_t size,
size_t n_copies,
size_t stride_tensor,
size_t stride_data) {
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
sess->flush_sync();
if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
extra->flags |= HTP_TENSOR_WEIGHT;
if (htp_is_tiled_weight_tensor(tensor)) {
extra->flags |= HTP_TENSOR_TILED;
}
ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT);
}
HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n",
sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags);
sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor));
if ((extra->flags & HTP_TENSOR_TILED) == 0) {
for (size_t i = 0; i < n_copies; i++) {
memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
}
return;
}
if (extra->shadow_buf.size() < ggml_nbytes(tensor)) {
extra->shadow_buf.resize(ggml_nbytes(tensor));
}
for (size_t i = 0; i < n_copies; i++) {
memcpy(extra->shadow_buf.data() + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
}
extra->shadow_size += n_copies * size;
if (extra->shadow_size >= ggml_nbytes(tensor)) {
htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size());
extra->shadow_buf.clear();
extra->shadow_buf.shrink_to_fit();
extra->shadow_size = 0;
memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
}
}
@@ -794,14 +745,14 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
size_t n_copies,
size_t stride_tensor,
size_t stride_data) {
auto extra = (htp_tensor_extra *) tensor->extra;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor);
HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n",
sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage);
if ((extra->flags & HTP_TENSOR_TILED) == 0) {
if ((flags & HTP_TENSOR_TILED) == 0) {
for (size_t i = 0; i < n_copies; i++) {
memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size);
}
@@ -825,6 +776,29 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
}
}
static void ggml_backend_hexagon_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
struct ggml_tensor * const * tensors,
const void * const * data,
const size_t * offsets,
const size_t * sizes,
size_t n_tensors) {
const bool is_weights = ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
sess->flush_sync();
for (size_t i = 0; i < n_tensors; i++) {
if (sizes[i] == 0) {
continue;
}
if (is_weights) {
ggml_hexagon_tensor_set_flags(tensors[i], HTP_TENSOR_WEIGHT);
}
memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]);
}
}
static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
@@ -844,7 +818,7 @@ static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,
/* .clear = */ ggml_backend_hexagon_buffer_clear,
/* .reset = */ NULL,
/* .set_tensor_n = */ NULL,
/* .set_tensor_n = */ ggml_backend_hexagon_buffer_set_tensor_n,
/* .get_tensor_n = */ NULL,
/* .cpy_tensor_n = */ NULL,
};
@@ -856,8 +830,12 @@ static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t bu
const void * data,
size_t offset,
size_t size) {
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
sess->flush_sync();
memcpy((char *) tensor->data + offset, data, size);
GGML_UNUSED(buffer);
}
static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer,
@@ -869,6 +847,25 @@ static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t bu
GGML_UNUSED(buffer);
}
static void ggml_backend_hexagon_host_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
struct ggml_tensor * const * tensors,
const void * const * data,
const size_t * offsets,
const size_t * sizes,
size_t n_tensors) {
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
auto sess = sbuf->sess;
sess->flush_sync();
for (size_t i = 0; i < n_tensors; i++) {
if (sizes[i] == 0) {
continue;
}
memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]);
}
}
static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = {
/* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer,
/* .get_base = */ ggml_backend_hexagon_buffer_get_base,
@@ -881,7 +878,7 @@ static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,
/* .clear = */ ggml_backend_hexagon_buffer_clear,
/* .reset = */ NULL,
/* .set_tensor_n = */ NULL,
/* .set_tensor_n = */ ggml_backend_hexagon_host_buffer_set_tensor_n,
/* .get_tensor_n = */ NULL,
/* .cpy_tensor_n = */ NULL,
};
@@ -896,7 +893,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer(
ggml_backend_buffer_type_t buffer_type, size_t size) {
auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx;
auto sess = dev_ctx->session();
if (sess && sess->max_vmem && size > sess->max_vmem) {
if (size > sess->max_vmem) {
GGML_LOG_ERROR("ggml-hex: %s buffer size %zu exceeds max_vmem %zu\n",
dev_ctx->c_name(), size, sess->max_vmem);
return nullptr;
@@ -914,7 +911,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer(
ggml_backend_buffer_type_t buffer_type, size_t size) {
auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx;
auto sess = dev_ctx->session();
if (sess && sess->max_vmem && size > sess->max_vmem) {
if (size > sess->max_vmem) {
GGML_LOG_ERROR("ggml-hex: %s host buffer size %zu exceeds max_vmem %zu\n",
dev_ctx->c_name(), size, sess->max_vmem);
return nullptr;
@@ -934,7 +931,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer
}
static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) {
if (!ggml_backend_buft_is_host(buft) && htp_is_tiled_weight_tensor(t)) {
if (opt_rt_repack != 0 && !ggml_backend_buft_is_host(buft) && ggml_op_is_empty(t->op) && htp_is_tiled_weight_tensor(t)) {
int64_t ne0 = hex_round_up(t->ne[0], 32);
int64_t ne1 = hex_round_up(t->ne[1], 32);
int64_t ne2 = t->ne[2];
@@ -1212,8 +1209,7 @@ struct ggml_hexagon_opbatch {
std::vector<htp_tensor> h_tens; // htp tensor descriptors
std::vector<htp_op_desc> h_ops; // htp op descriptors
std::unordered_map<const ggml_tensor*, int> t_map; // tensor ptr to index
std::unordered_multimap<void*, int> d_map; // tensor data to index
std::unordered_map<uint64_t, int> t_map; // tensor hash to index
unsigned int n_bufs; // num buffers in the batch
unsigned int n_tens; // num tensors ...
@@ -1232,7 +1228,6 @@ struct ggml_hexagon_opbatch {
b_vmem = 0;
t_map.clear();
d_map.clear();
ops.resize(n_ops_max);
}
@@ -1252,7 +1247,6 @@ struct ggml_hexagon_opbatch {
h_ops.resize(n_ops_max);
t_map.reserve(n_tens_max);
d_map.reserve(n_tens_max);
GGML_LOG_INFO("ggml-hex: %s op batching: n-bufs %u n-tensors %u n-ops %u vmem %zu\n",
sess->c_name(), n_bufs_max, n_tens_max, n_ops_max, b_vmem_max);
@@ -1298,7 +1292,7 @@ struct ggml_hexagon_opbatch {
}
static void init_tensor_shape(htp_tensor & h, const ggml_tensor * t) {
auto extra = (htp_tensor_extra *) t->extra;
const uint32_t flags = ggml_hexagon_tensor_get_flags(t);
h.type = t->type;
h.ne[0] = t->ne[0];
@@ -1306,7 +1300,7 @@ struct ggml_hexagon_opbatch {
h.ne[2] = t->ne[2];
h.ne[3] = t->ne[3];
if (extra->flags & HTP_TENSOR_TILED) {
if (flags & HTP_TENSOR_TILED) {
if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16) {
h.type = GGML_TYPE_F16;
h.nb[0] = sizeof(ggml_fp16_t);
@@ -1327,31 +1321,42 @@ struct ggml_hexagon_opbatch {
h.size = ggml_nbytes(t);
}
h.flags = extra->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE);
h.flags = flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE);
}
static bool same_shape(const htp_tensor & a, const htp_tensor & b) {
return (a.type == b.type) &&
(a.ne[0] == b.ne[0]) && (a.ne[1] == b.ne[1]) && (a.ne[2] == b.ne[2]) && (a.ne[3] == b.ne[3]) &&
(a.nb[0] == b.nb[0]) && (a.nb[1] == b.nb[1]) && (a.nb[2] == b.nb[2]) && (a.nb[3] == b.nb[3]);
static uint64_t hash_tensor(const ggml_tensor * t, const htp_tensor & shape) {
uint64_t hash = 1469598103934665603ull;
auto hash_value = [&hash](uint64_t value) {
hash ^= value;
hash *= 1099511628211ull;
};
hash_value((uintptr_t) t->data);
hash_value(shape.type);
hash_value(shape.size);
hash_value(shape.flags);
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
hash_value(shape.ne[i]);
hash_value(shape.nb[i]);
}
// avalanche high bits down to low bits for power-of-two table indexing
hash ^= hash >> 33;
hash *= 0xff51afd7ed558ccdull;
hash ^= hash >> 33;
return hash;
}
// add tensor and return its index
int add_tensor(const ggml_tensor * t) {
// Lookup by tensor ptr
auto it = t_map.find(t);
if (it != t_map.end()) { return it->second; }
htp_tensor shape {};
init_tensor_shape(shape, t);
// Lookup by tensor data and shape
auto range = d_map.equal_range(t->data);
for (auto dit = range.first; dit != range.second; ++dit) {
if (same_shape(h_tens[dit->second], shape)) {
t_map.insert({t, dit->second});
return dit->second;
}
const uint64_t hash = hash_tensor(t, shape);
auto it = t_map.find(hash);
if (it != t_map.end()) {
return it->second;
}
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
@@ -1360,8 +1365,7 @@ struct ggml_hexagon_opbatch {
int ti = n_tens++;
GGML_ASSERT(n_tens <= n_tens_max);
t_map.insert({t, ti});
d_map.insert({t->data, ti});
t_map[hash] = ti;
uint64_t t_offset = (uint8_t *) t->data - sbuf->base();
@@ -1391,8 +1395,11 @@ struct ggml_hexagon_opbatch {
auto fit_tensor = [&](const ggml_tensor * t) {
if (!t) return;
if (check_tens) {
if (t_map.count(t)) return;
htp_tensor shape {};
init_tensor_shape(shape, t);
if (t_map.count(hash_tensor(t, shape))) return;
extra_tens++;
}
@@ -2364,11 +2371,33 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) {
delete backend;
}
static inline bool op_is_compute(ggml_tensor *node)
{
static inline bool ggml_hexagon_op_is_compute(const struct ggml_tensor * node) {
return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE);
}
static void ggml_hexagon_tensor_repack(ggml_tensor * tensor, ggml_hexagon_session * sess) {
if (tensor->data == nullptr || ggml_hexagon_tensor_is_tiled(tensor) || !ggml_hexagon_tensor_can_tile(tensor)) {
return;
}
HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(tensor, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(tensor));
std::vector<uint8_t> raw(ggml_nbytes(tensor));
memcpy(raw.data(), tensor->data, raw.size());
htp_repack_tensor_tiled(tensor, raw.data(), raw.size());
ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_TILED);
}
static void ggml_hexagon_graph_repack(const ggml_cgraph * graph, ggml_hexagon_session * sess) {
for (int i = 0; i < graph->n_nodes; ++i) {
ggml_tensor * n = graph->nodes[i];
if (!ggml_hexagon_op_is_compute(n)) {
continue;
}
if (n->op == GGML_OP_MUL_MAT || n->op == GGML_OP_MUL_MAT_ID) {
ggml_hexagon_tensor_repack(n->src[0], sess);
}
}
}
static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) {
auto sess = static_cast<ggml_hexagon_session *>(backend->context);
@@ -2385,13 +2414,15 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
if (cache_hit) {
nodes_ptr = &sess->cached_nodes;
} else {
ggml_hexagon_graph_repack(graph, sess);
htp_fusion_tag_graph(graph);
computed_nodes.reserve(graph->n_nodes);
for (int i = 0; i < graph->n_nodes; ++i) {
ggml_tensor * n = graph->nodes[i];
if (!op_is_compute(n)) {
if (!ggml_hexagon_op_is_compute(n)) {
continue;
}
@@ -2401,22 +2432,7 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
}
htp_opnode node(opcode, n);
if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || node.opcode == HTP_OP_GET_ROWS) {
ggml_tensor * src0 = node.node->src[0];
if (htp_needs_repack(src0)) {
HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(src0, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(src0));
std::vector<uint8_t> raw(ggml_nbytes(src0));
memcpy(raw.data(), src0->data, raw.size());
htp_repack_tensor_tiled(src0, raw.data(), raw.size());
auto * extra = (htp_tensor_extra *) src0->extra;
extra->flags |= HTP_TENSOR_TILED;
}
}
GGML_ASSERT(htp_kparams_solve(sess, node));
if (node.opcode == HTP_OP_CPY && node.kparams<htp_copy_kernel_params>()->total_elems == 0) {
continue;
}
computed_nodes.push_back(std::move(node));
}
@@ -2626,7 +2642,11 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<no
}
static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgraph * gf, ggml_backend_graph_optimize_params * params) {
GGML_UNUSED(params);
auto sess = static_cast<ggml_hexagon_session *>(backend->context);
HEX_VERBOSE("ggml-hex: %s graph-optimize n_nodes %d\n", sess->c_name(), gf->n_nodes);
ggml_hexagon_graph_repack(gf, sess);
const int n = gf->n_nodes;
@@ -2661,7 +2681,7 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr
}
}
GGML_UNUSED(backend);
GGML_UNUSED(params);
}
static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) {
@@ -2723,14 +2743,7 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg
return false;
}
// FIXME: ggml-meta needs to call init_tensor on auxiliary tensors
if (!dst->extra) {
ggml_backend_buffer_init_tensor(dst->buffer, dst);
}
auto * dst_extra = static_cast<htp_tensor_extra *>(dst->extra);
const auto * src_extra = static_cast<const htp_tensor_extra *>(src->extra);
dst_extra->flags = src_extra->flags & ~HTP_TENSOR_FUSEABLE;
dst->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(src) & ~HTP_TENSOR_FUSEABLE);
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
@@ -3003,18 +3016,18 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
}
bool supp = false;
switch (op->op) {
case GGML_OP_NONE:
case GGML_OP_RESHAPE:
case GGML_OP_VIEW:
case GGML_OP_PERMUTE:
case GGML_OP_TRANSPOSE:
supp = true;
break;
default:
if (ggml_op_is_empty(op->op) || ggml_is_empty(op)) {
supp = true;
} else {
supp = ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op) <= sess->max_vmem;
for (int i = 0; supp && i < GGML_MAX_SRC; i++) {
if (op->src[i] && ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op->src[i]) > sess->max_vmem) {
supp = false;
}
}
if (supp) {
supp = htp_kparams_check(sess, op);
break;
}
}
ggml_hexagon_dump_op_supp(sess->c_name(), op, supp);
+27 -28
View File
@@ -4,12 +4,6 @@
#include "htp-opnode.h"
#include "htp-repack.h"
struct htp_tensor_extra {
std::vector<uint8_t> shadow_buf;
size_t shadow_size { 0 };
uint32_t flags { 0 };
};
extern int opt_arch;
extern size_t opt_ndev;
extern size_t opt_nhvx;
@@ -46,36 +40,41 @@ extern int opt_rt_repack;
if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__)
#endif
static inline bool htp_tensor_is_fuseable(const struct ggml_tensor * t) {
if (!t || !t->extra) return false;
const auto * extra = (const struct htp_tensor_extra *) t->extra;
return (extra->flags & HTP_TENSOR_FUSEABLE) != 0;
static inline uint32_t ggml_hexagon_tensor_get_flags(const struct ggml_tensor * t) {
return (uint32_t) (uintptr_t) t->extra;
}
static inline bool htp_can_rt_repack(const struct ggml_tensor * t) {
if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) {
return false;
}
if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) {
return false;
}
return opt_rt_repack > 1 || ggml_is_quantized(t->type);
static inline void ggml_hexagon_tensor_set_flags(struct ggml_tensor * t, uint32_t flags) {
t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) | flags);
}
static inline bool htp_tensor_is_tiled(const struct ggml_tensor * t) {
const auto * extra = (const struct htp_tensor_extra *) t->extra;
if (extra && (extra->flags & HTP_TENSOR_TILED)) {
return true;
}
return htp_can_rt_repack(t);
static inline void ggml_hexagon_tensor_clear_flags(struct ggml_tensor * t, uint32_t flags) {
t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) & ~flags);
}
static inline bool htp_needs_repack(const struct ggml_tensor * t) {
const auto * extra = (const struct htp_tensor_extra *) t->extra;
if (extra && (extra->flags & HTP_TENSOR_TILED)) {
static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) {
return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_FUSEABLE) != 0;
}
static inline bool ggml_hexagon_tensor_is_tiled(const struct ggml_tensor * t) {
return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_TILED) != 0;
}
static inline bool ggml_hexagon_tensor_can_tile(const struct ggml_tensor * t) {
if (opt_rt_repack == 0) {
return false;
}
return htp_can_rt_repack(t);
if (t->buffer) {
if (ggml_backend_buft_is_host(t->buffer->buft) ||
ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
return false;
}
}
return htp_is_tiled_weight_tensor(t);
}
static inline bool ggml_hexagon_tensor_will_tile(const struct ggml_tensor * t) {
return ggml_hexagon_tensor_is_tiled(t) || ggml_hexagon_tensor_can_tile(t);
}
static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) {
+11 -13
View File
@@ -35,7 +35,7 @@ static inline bool htp_match_consumer(
return false;
}
const ggml_tensor * prev_out = prev.dst();
if (!htp_tensor_is_fuseable(prev_out)) {
if (!ggml_hexagon_tensor_is_fuseable(prev_out)) {
return false;
}
if (htp_same_tensor_data(cur.src0(), prev_out)) {
@@ -53,14 +53,14 @@ static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struc
if (kparams->n_hmx) {
return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
}
return htp_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW);
return ggml_hexagon_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW);
}
static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {
if (kparams->n_hmx) {
return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
}
return htp_tensor_is_tiled(src0) &&
return ggml_hexagon_tensor_is_tiled(src0) &&
(kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK);
}
@@ -74,7 +74,7 @@ static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & p
const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks;
const ggml_tensor * ar_local = prev.inputs[rank];
if (!htp_tensor_is_fuseable(ar_local)) return false;
if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false;
const ggml_tensor * add_dst = cur.dst();
const ggml_tensor * res_tensor = nullptr;
@@ -250,7 +250,7 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev
if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 ||
htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) ||
orig_kparams->n_hmx != cur_kparams->n_hmx) {
return false;
}
@@ -295,7 +295,7 @@ static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & p
if (x_in != x || ids_in != ids || w_in->type != w0->type ||
w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] ||
htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) ||
orig_kparams->n_hmx != cur_kparams->n_hmx) {
return false;
}
@@ -598,19 +598,17 @@ int htp_fusion_max(const struct ggml_cgraph * graph, int idx) {
void htp_fusion_tag_graph(struct ggml_cgraph * graph) {
for (int i = 0; i < graph->n_nodes; i++) {
auto * extra = (htp_tensor_extra *) graph->nodes[i]->extra;
if (!extra) continue;
ggml_tensor * node = graph->nodes[i];
ggml_hexagon_tensor_clear_flags(node, HTP_TENSOR_FUSEABLE);
extra->flags &= ~HTP_TENSOR_FUSEABLE;
const ggml_op op = graph->nodes[i]->op;
const ggml_op op = node->op;
if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT ||
op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB ||
op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID ||
op == GGML_OP_SSM_CONV ||
(op == GGML_OP_MUL_MAT && !htp_is_hadamard(graph->nodes[i]))) &&
(op == GGML_OP_MUL_MAT && !htp_is_hadamard(node))) &&
ggml_node_has_n_uses(graph, i, 1)) {
extra->flags |= HTP_TENSOR_FUSEABLE;
ggml_hexagon_tensor_set_flags(node, HTP_TENSOR_FUSEABLE);
}
}
}
+6 -6
View File
@@ -628,7 +628,7 @@ static bool htp_matmul_is_hmx_eligible(
return false;
}
if (!htp_is_hmx_weight_type((ggml_type) wtype)) {
if (!htp_is_tiled_weight_type((ggml_type) wtype)) {
return false;
}
@@ -636,7 +636,7 @@ static bool htp_matmul_is_hmx_eligible(
return false;
}
const bool is_tiled = htp_tensor_is_tiled(src0);
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) {
return false;
}
@@ -674,7 +674,7 @@ static bool htp_kparams_solve_hmx_mm(
size_t vtcm_budget,
struct htp_mm_kernel_params * kparams
) {
const bool is_tiled = htp_tensor_is_tiled(src0);
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0;
const int n_threads = (int) htp_session_n_threads(sess);
const uint32_t mdev_count = htp_session_mdev_count(sess);
@@ -893,7 +893,7 @@ static bool htp_kparams_solve_matmul_impl(
const int ne12 = src1->ne[2];
const int ne13 = src1->ne[3];
const bool is_tiled = htp_tensor_is_tiled(src0);
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00;
const int ne11_padded = hex_round_up(ne11, 32);
@@ -1109,7 +1109,7 @@ bool htp_kparams_solve_mmnx(
const int ne12 = act->ne[2];
const int ne13 = act->ne[3];
const bool is_tiled = htp_tensor_is_tiled(src0);
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00;
const int ne11_padded = hex_round_up(ne11, 32);
@@ -1683,7 +1683,7 @@ bool htp_kparams_solve_repeat_get_rows(
const uint32_t ne12 = src1->ne[2];
const uint32_t nr = ne10 * ne11 * ne12;
const bool tiled = htp_tensor_is_tiled(src0);
const bool tiled = ggml_hexagon_tensor_is_tiled(src0);
const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
if (tiled) {
+11 -16
View File
@@ -8,31 +8,30 @@
#include "htp/htp-tensor.h"
static inline bool htp_is_tiled_weight_type(enum ggml_type type) {
return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 ||
type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL ||
type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0 ||
type == GGML_TYPE_Q6_K ||
return type == GGML_TYPE_Q1_0 || type == GGML_TYPE_Q2_0 ||
type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 ||
type == GGML_TYPE_MXFP4 || type == GGML_TYPE_IQ4_NL ||
type == GGML_TYPE_Q8_0 || type == GGML_TYPE_Q6_K ||
type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K ||
type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K ||
type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K ||
type == GGML_TYPE_Q2_0 ||
type == GGML_TYPE_F16 || type == GGML_TYPE_F32 ||
type == GGML_TYPE_BF16;
}
static inline bool htp_is_tiled_weight_tensor(const struct ggml_tensor * t) {
if (!ggml_is_contiguous(t) || ggml_is_view(t)) {
return false;
}
if (!htp_is_tiled_weight_type(t->type)) {
return false;
}
if (t->ne[0] < 32) {
if (!ggml_is_contiguous(t) || ggml_is_view(t)) {
return false;
}
if (t->ne[0] < 32 || (t->ne[0] % 32 != 0) || t->ne[3] != 1) {
return false;
}
if (!ggml_is_quantized(t->type)) {
return (t->ne[0] % 32 == 0) && (t->ne[1] % 32 == 0) && (t->ne[2] == 1) && (t->ne[3] == 1);
return t->ne[1] > 1;
}
return t->ne[3] == 1;
return true;
}
// Size of one tiled row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins,
@@ -59,10 +58,6 @@ static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) {
return ggml_row_size(type, ne0);
}
static inline bool htp_is_hmx_weight_type(enum ggml_type type) {
return htp_is_tiled_weight_type(type);
}
void htp_repack_tensor_tiled(struct ggml_tensor * t, const void * data, size_t size);
void htp_unpack_tensor_tiled(void * data, const struct ggml_tensor * t, size_t offset, size_t size);