From 67107ac34f62882d8b53ea41ad68010ea635b9be Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Thu, 5 Nov 2026 18:15:58 -0800 Subject: [PATCH] hexagon: Implement set_tensor_n and improve repacking/tiling robustness (#31744) * hex-repack: move repacking/tiling into graph-optimize * hex-tiling: simplify tiling rules and remove obsolete helpers * hex-ops: use a hash for de-duplicating tensor descriptors * hex-bufs: improve tensor hash * hex-ops: accept empty (zero-elements) ops * hex-bufs: enable hostbuf by default and reject tensors that do not fit into max_vmem * hex-bufs: improve error message and update readme * hex-bufs: make sure to sync flush opbatch in all sync set_tensor paths --- docs/backend/snapdragon/README.md | 10 +- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 325 +++++++++++++------------ ggml/src/ggml-hexagon/htp-common.h | 55 ++--- ggml/src/ggml-hexagon/htp-fusion.cpp | 24 +- ggml/src/ggml-hexagon/htp-kparams.cpp | 12 +- ggml/src/ggml-hexagon/htp-repack.h | 27 +- 6 files changed, 231 insertions(+), 222 deletions(-) diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md index ea4e3f386..bd9703705 100644 --- a/docs/backend/snapdragon/README.md +++ b/docs/backend/snapdragon/README.md @@ -325,10 +325,14 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version). - `GGML_HEXAGON_VTCM_SIZE=0` - Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session. + Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. + The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session. -- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled) - Enables allocating host buffers for debugging. By default, host buffers are disabled. +- `GGML_HEXAGON_HOSTBUF=0` (default: 1, enabled) + Controls host buffer (`HTP*-HOST`) allocation in shared FastRPC memory to avoid host-device tensor copies. + CPU-assigned weights (input embeddings like `token_embd` and `per_layer_token_embd`, and tensors overridden with `-ot ...=CPU`) + are placed in host buffers as well. On older drivers/FW, large models can run out of IOVA space when these are mapped. + Pass `--no-host` to keep CPU-assigned weights in regular CPU memory (`-ot ...=CPU` alone is not sufficient). - `GGML_HEXAGON_DMA64=0` (default: enabled on v81+) Disables 64-bit DMA for model weights. Set to `1` to enable it explicitly on a supported architecture. diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index a883c585c..9b2600c66 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -90,7 +90,7 @@ size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for int opt_etm = 0; int opt_verbose = 0; int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu) -bool opt_hostbuf = false; +bool opt_hostbuf = true; bool opt_dma64 = false; int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start @@ -150,7 +150,7 @@ int opt_opqueue = 8; // max number of pending batches int opt_optrace = 0; // trace buffer size per thread (0 means default) int opt_oppoll = 0; // polling for batch completions int opt_opfusion = 1; // enable/disable op fusion -int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = always +int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = all tileable weights (default) static std::regex* opt_opfilter = NULL; // regex of ops to not claim @@ -463,7 +463,6 @@ struct ggml_hexagon_rpcmem_block { struct ggml_hexagon_shared_buffer { ggml_hexagon_session * sess; std::shared_ptr mem; - std::vector tensor_extra; bool mapped; bool pinned; bool extended; @@ -486,6 +485,9 @@ struct ggml_hexagon_shared_buffer { if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(), sess->domain_id, size(), fd(), (unsigned) err); + if (!this->pinned) { + GGML_LOG_ERROR("ggml-hex: %s possibly out of IOVA space, try --no-host\n", sess->c_name()); + } throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)"); } @@ -561,9 +563,6 @@ struct ggml_hexagon_shared_buffer { ~ggml_hexagon_shared_buffer() { free(); - for (auto * extra : tensor_extra) { - delete extra; - } } }; @@ -622,11 +621,9 @@ inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) { } inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * slot, uint32_t n_slots) const { - static htp_tensor_extra fence_extra { {}, 0, HTP_TENSOR_FENCE }; - ggml_tensor t {}; t.buffer = &this->fence_buf->backend_buffer; - t.extra = &fence_extra; + t.extra = (void *) (uintptr_t) HTP_TENSOR_FENCE; t.data = const_cast(slot); t.type = GGML_TYPE_I8; t.ne[0] = HTP_FENCE_SLOT_SIZE; @@ -659,10 +656,7 @@ static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buf HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(), tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor)); - auto extra = new htp_tensor_extra(); - sbuf->tensor_extra.push_back(extra); - - tensor->extra = extra; + tensor->extra = (void *) (uintptr_t) 0; return GGML_STATUS_SUCCESS; } @@ -672,42 +666,19 @@ static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, const void * data, size_t offset, size_t size) { - auto extra = (htp_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - extra->flags |= HTP_TENSOR_WEIGHT; - if (htp_is_tiled_weight_tensor(tensor)) { - extra->flags |= HTP_TENSOR_TILED; - } + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT); } HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor)); - if ((extra->flags & HTP_TENSOR_TILED) == 0) { - memcpy((char *) tensor->data + offset, data, size); - return; - } - - if (offset == 0 && size == ggml_nbytes(tensor) && extra->shadow_buf.empty()) { - htp_repack_tensor_tiled(tensor, data, size); - return; - } - - if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { - extra->shadow_buf.resize(ggml_nbytes(tensor)); - } - memcpy(extra->shadow_buf.data() + offset, data, size); - extra->shadow_size += size; - - if (extra->shadow_size >= ggml_nbytes(tensor)) { - htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); - extra->shadow_buf.clear(); - extra->shadow_buf.shrink_to_fit(); - extra->shadow_size = 0; - } + memcpy((char *) tensor->data + offset, data, size); } static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, @@ -715,14 +686,14 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, void * data, size_t offset, size_t size) { - auto extra = (htp_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor); HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, flags); - if ((extra->flags & HTP_TENSOR_TILED) == 0) { + if ((flags & HTP_TENSOR_TILED) == 0) { memcpy(data, (const char *) tensor->data + offset, size); return; } @@ -742,47 +713,27 @@ static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t bu } static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, - ggml_tensor * tensor, - const void * data, - size_t offset, - size_t size, - size_t n_copies, - size_t stride_tensor, - size_t stride_data) { - auto extra = (htp_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - extra->flags |= HTP_TENSOR_WEIGHT; - if (htp_is_tiled_weight_tensor(tensor)) { - extra->flags |= HTP_TENSOR_TILED; - } + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT); } HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n", - sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags); + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor)); - if ((extra->flags & HTP_TENSOR_TILED) == 0) { - for (size_t i = 0; i < n_copies; i++) { - memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); - } - return; - } - - if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { - extra->shadow_buf.resize(ggml_nbytes(tensor)); - } for (size_t i = 0; i < n_copies; i++) { - memcpy(extra->shadow_buf.data() + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); - } - extra->shadow_size += n_copies * size; - - if (extra->shadow_size >= ggml_nbytes(tensor)) { - htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); - extra->shadow_buf.clear(); - extra->shadow_buf.shrink_to_fit(); - extra->shadow_size = 0; + memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); } } @@ -794,14 +745,14 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff size_t n_copies, size_t stride_tensor, size_t stride_data) { - auto extra = (htp_tensor_extra *) tensor->extra; - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor); HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage); - if ((extra->flags & HTP_TENSOR_TILED) == 0) { + if ((flags & HTP_TENSOR_TILED) == 0) { for (size_t i = 0; i < n_copies; i++) { memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size); } @@ -825,6 +776,29 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff } } +static void ggml_backend_hexagon_buffer_set_tensor_n(ggml_backend_buffer_t buffer, + struct ggml_tensor * const * tensors, + const void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + const bool is_weights = ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + if (is_weights) { + ggml_hexagon_tensor_set_flags(tensors[i], HTP_TENSOR_WEIGHT); + } + memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]); + } +} + static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; auto sess = sbuf->sess; @@ -844,7 +818,7 @@ static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, /* .clear = */ ggml_backend_hexagon_buffer_clear, /* .reset = */ NULL, - /* .set_tensor_n = */ NULL, + /* .set_tensor_n = */ ggml_backend_hexagon_buffer_set_tensor_n, /* .get_tensor_n = */ NULL, /* .cpy_tensor_n = */ NULL, }; @@ -856,8 +830,12 @@ static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t bu const void * data, size_t offset, size_t size) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); + memcpy((char *) tensor->data + offset, data, size); - GGML_UNUSED(buffer); } static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer, @@ -869,6 +847,25 @@ static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t bu GGML_UNUSED(buffer); } +static void ggml_backend_hexagon_host_buffer_set_tensor_n(ggml_backend_buffer_t buffer, + struct ggml_tensor * const * tensors, + const void * const * data, + const size_t * offsets, + const size_t * sizes, + size_t n_tensors) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + sess->flush_sync(); + + for (size_t i = 0; i < n_tensors; i++) { + if (sizes[i] == 0) { + continue; + } + memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]); + } +} + static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = { /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, /* .get_base = */ ggml_backend_hexagon_buffer_get_base, @@ -881,7 +878,7 @@ static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = { /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, /* .clear = */ ggml_backend_hexagon_buffer_clear, /* .reset = */ NULL, - /* .set_tensor_n = */ NULL, + /* .set_tensor_n = */ ggml_backend_hexagon_host_buffer_set_tensor_n, /* .get_tensor_n = */ NULL, /* .cpy_tensor_n = */ NULL, }; @@ -896,7 +893,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); - if (sess && sess->max_vmem && size > sess->max_vmem) { + if (size > sess->max_vmem) { GGML_LOG_ERROR("ggml-hex: %s buffer size %zu exceeds max_vmem %zu\n", dev_ctx->c_name(), size, sess->max_vmem); return nullptr; @@ -914,7 +911,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); - if (sess && sess->max_vmem && size > sess->max_vmem) { + if (size > sess->max_vmem) { GGML_LOG_ERROR("ggml-hex: %s host buffer size %zu exceeds max_vmem %zu\n", dev_ctx->c_name(), size, sess->max_vmem); return nullptr; @@ -934,7 +931,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer } static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) { - if (!ggml_backend_buft_is_host(buft) && htp_is_tiled_weight_tensor(t)) { + if (opt_rt_repack != 0 && !ggml_backend_buft_is_host(buft) && ggml_op_is_empty(t->op) && htp_is_tiled_weight_tensor(t)) { int64_t ne0 = hex_round_up(t->ne[0], 32); int64_t ne1 = hex_round_up(t->ne[1], 32); int64_t ne2 = t->ne[2]; @@ -1212,8 +1209,7 @@ struct ggml_hexagon_opbatch { std::vector h_tens; // htp tensor descriptors std::vector h_ops; // htp op descriptors - std::unordered_map t_map; // tensor ptr to index - std::unordered_multimap d_map; // tensor data to index + std::unordered_map t_map; // tensor hash to index unsigned int n_bufs; // num buffers in the batch unsigned int n_tens; // num tensors ... @@ -1232,7 +1228,6 @@ struct ggml_hexagon_opbatch { b_vmem = 0; t_map.clear(); - d_map.clear(); ops.resize(n_ops_max); } @@ -1252,7 +1247,6 @@ struct ggml_hexagon_opbatch { h_ops.resize(n_ops_max); t_map.reserve(n_tens_max); - d_map.reserve(n_tens_max); GGML_LOG_INFO("ggml-hex: %s op batching: n-bufs %u n-tensors %u n-ops %u vmem %zu\n", sess->c_name(), n_bufs_max, n_tens_max, n_ops_max, b_vmem_max); @@ -1298,7 +1292,7 @@ struct ggml_hexagon_opbatch { } static void init_tensor_shape(htp_tensor & h, const ggml_tensor * t) { - auto extra = (htp_tensor_extra *) t->extra; + const uint32_t flags = ggml_hexagon_tensor_get_flags(t); h.type = t->type; h.ne[0] = t->ne[0]; @@ -1306,7 +1300,7 @@ struct ggml_hexagon_opbatch { h.ne[2] = t->ne[2]; h.ne[3] = t->ne[3]; - if (extra->flags & HTP_TENSOR_TILED) { + if (flags & HTP_TENSOR_TILED) { if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16) { h.type = GGML_TYPE_F16; h.nb[0] = sizeof(ggml_fp16_t); @@ -1327,31 +1321,42 @@ struct ggml_hexagon_opbatch { h.size = ggml_nbytes(t); } - h.flags = extra->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE); + h.flags = flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE); } - static bool same_shape(const htp_tensor & a, const htp_tensor & b) { - return (a.type == b.type) && - (a.ne[0] == b.ne[0]) && (a.ne[1] == b.ne[1]) && (a.ne[2] == b.ne[2]) && (a.ne[3] == b.ne[3]) && - (a.nb[0] == b.nb[0]) && (a.nb[1] == b.nb[1]) && (a.nb[2] == b.nb[2]) && (a.nb[3] == b.nb[3]); + static uint64_t hash_tensor(const ggml_tensor * t, const htp_tensor & shape) { + uint64_t hash = 1469598103934665603ull; + auto hash_value = [&hash](uint64_t value) { + hash ^= value; + hash *= 1099511628211ull; + }; + + hash_value((uintptr_t) t->data); + hash_value(shape.type); + hash_value(shape.size); + hash_value(shape.flags); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + hash_value(shape.ne[i]); + hash_value(shape.nb[i]); + } + + // avalanche high bits down to low bits for power-of-two table indexing + hash ^= hash >> 33; + hash *= 0xff51afd7ed558ccdull; + hash ^= hash >> 33; + + return hash; } // add tensor and return its index int add_tensor(const ggml_tensor * t) { - // Lookup by tensor ptr - auto it = t_map.find(t); - if (it != t_map.end()) { return it->second; } - htp_tensor shape {}; init_tensor_shape(shape, t); - // Lookup by tensor data and shape - auto range = d_map.equal_range(t->data); - for (auto dit = range.first; dit != range.second; ++dit) { - if (same_shape(h_tens[dit->second], shape)) { - t_map.insert({t, dit->second}); - return dit->second; - } + const uint64_t hash = hash_tensor(t, shape); + auto it = t_map.find(hash); + if (it != t_map.end()) { + return it->second; } auto sbuf = static_cast(t->buffer->context); @@ -1360,8 +1365,7 @@ struct ggml_hexagon_opbatch { int ti = n_tens++; GGML_ASSERT(n_tens <= n_tens_max); - t_map.insert({t, ti}); - d_map.insert({t->data, ti}); + t_map[hash] = ti; uint64_t t_offset = (uint8_t *) t->data - sbuf->base(); @@ -1391,8 +1395,11 @@ struct ggml_hexagon_opbatch { auto fit_tensor = [&](const ggml_tensor * t) { if (!t) return; + if (check_tens) { - if (t_map.count(t)) return; + htp_tensor shape {}; + init_tensor_shape(shape, t); + if (t_map.count(hash_tensor(t, shape))) return; extra_tens++; } @@ -2364,11 +2371,33 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) { delete backend; } -static inline bool op_is_compute(ggml_tensor *node) -{ +static inline bool ggml_hexagon_op_is_compute(const struct ggml_tensor * node) { return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE); } +static void ggml_hexagon_tensor_repack(ggml_tensor * tensor, ggml_hexagon_session * sess) { + if (tensor->data == nullptr || ggml_hexagon_tensor_is_tiled(tensor) || !ggml_hexagon_tensor_can_tile(tensor)) { + return; + } + HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(tensor, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(tensor)); + std::vector raw(ggml_nbytes(tensor)); + memcpy(raw.data(), tensor->data, raw.size()); + htp_repack_tensor_tiled(tensor, raw.data(), raw.size()); + ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_TILED); +} + +static void ggml_hexagon_graph_repack(const ggml_cgraph * graph, ggml_hexagon_session * sess) { + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * n = graph->nodes[i]; + if (!ggml_hexagon_op_is_compute(n)) { + continue; + } + if (n->op == GGML_OP_MUL_MAT || n->op == GGML_OP_MUL_MAT_ID) { + ggml_hexagon_tensor_repack(n->src[0], sess); + } + } +} + static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { auto sess = static_cast(backend->context); @@ -2385,13 +2414,15 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg if (cache_hit) { nodes_ptr = &sess->cached_nodes; } else { + ggml_hexagon_graph_repack(graph, sess); + htp_fusion_tag_graph(graph); computed_nodes.reserve(graph->n_nodes); for (int i = 0; i < graph->n_nodes; ++i) { ggml_tensor * n = graph->nodes[i]; - if (!op_is_compute(n)) { + if (!ggml_hexagon_op_is_compute(n)) { continue; } @@ -2401,22 +2432,7 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg } htp_opnode node(opcode, n); - if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || node.opcode == HTP_OP_GET_ROWS) { - ggml_tensor * src0 = node.node->src[0]; - if (htp_needs_repack(src0)) { - HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(src0, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(src0)); - std::vector raw(ggml_nbytes(src0)); - memcpy(raw.data(), src0->data, raw.size()); - htp_repack_tensor_tiled(src0, raw.data(), raw.size()); - auto * extra = (htp_tensor_extra *) src0->extra; - extra->flags |= HTP_TENSOR_TILED; - } - } - GGML_ASSERT(htp_kparams_solve(sess, node)); - if (node.opcode == HTP_OP_CPY && node.kparams()->total_elems == 0) { - continue; - } computed_nodes.push_back(std::move(node)); } @@ -2626,7 +2642,11 @@ static std::vector ggml_hexagon_graph_optimize_reorder(const std::vector(backend->context); + + HEX_VERBOSE("ggml-hex: %s graph-optimize n_nodes %d\n", sess->c_name(), gf->n_nodes); + + ggml_hexagon_graph_repack(gf, sess); const int n = gf->n_nodes; @@ -2661,7 +2681,7 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr } } - GGML_UNUSED(backend); + GGML_UNUSED(params); } static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) { @@ -2723,14 +2743,7 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg return false; } - // FIXME: ggml-meta needs to call init_tensor on auxiliary tensors - if (!dst->extra) { - ggml_backend_buffer_init_tensor(dst->buffer, dst); - } - - auto * dst_extra = static_cast(dst->extra); - const auto * src_extra = static_cast(src->extra); - dst_extra->flags = src_extra->flags & ~HTP_TENSOR_FUSEABLE; + dst->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(src) & ~HTP_TENSOR_FUSEABLE); auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); @@ -3003,18 +3016,18 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons } bool supp = false; - switch (op->op) { - case GGML_OP_NONE: - case GGML_OP_RESHAPE: - case GGML_OP_VIEW: - case GGML_OP_PERMUTE: - case GGML_OP_TRANSPOSE: - supp = true; - break; - - default: + if (ggml_op_is_empty(op->op) || ggml_is_empty(op)) { + supp = true; + } else { + supp = ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op) <= sess->max_vmem; + for (int i = 0; supp && i < GGML_MAX_SRC; i++) { + if (op->src[i] && ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op->src[i]) > sess->max_vmem) { + supp = false; + } + } + if (supp) { supp = htp_kparams_check(sess, op); - break; + } } ggml_hexagon_dump_op_supp(sess->c_name(), op, supp); diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.h index 472892be9..815a021b8 100644 --- a/ggml/src/ggml-hexagon/htp-common.h +++ b/ggml/src/ggml-hexagon/htp-common.h @@ -4,12 +4,6 @@ #include "htp-opnode.h" #include "htp-repack.h" -struct htp_tensor_extra { - std::vector shadow_buf; - size_t shadow_size { 0 }; - uint32_t flags { 0 }; -}; - extern int opt_arch; extern size_t opt_ndev; extern size_t opt_nhvx; @@ -46,36 +40,41 @@ extern int opt_rt_repack; if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__) #endif -static inline bool htp_tensor_is_fuseable(const struct ggml_tensor * t) { - if (!t || !t->extra) return false; - const auto * extra = (const struct htp_tensor_extra *) t->extra; - return (extra->flags & HTP_TENSOR_FUSEABLE) != 0; +static inline uint32_t ggml_hexagon_tensor_get_flags(const struct ggml_tensor * t) { + return (uint32_t) (uintptr_t) t->extra; } -static inline bool htp_can_rt_repack(const struct ggml_tensor * t) { - if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) { - return false; - } - if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) { - return false; - } - return opt_rt_repack > 1 || ggml_is_quantized(t->type); +static inline void ggml_hexagon_tensor_set_flags(struct ggml_tensor * t, uint32_t flags) { + t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) | flags); } -static inline bool htp_tensor_is_tiled(const struct ggml_tensor * t) { - const auto * extra = (const struct htp_tensor_extra *) t->extra; - if (extra && (extra->flags & HTP_TENSOR_TILED)) { - return true; - } - return htp_can_rt_repack(t); +static inline void ggml_hexagon_tensor_clear_flags(struct ggml_tensor * t, uint32_t flags) { + t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) & ~flags); } -static inline bool htp_needs_repack(const struct ggml_tensor * t) { - const auto * extra = (const struct htp_tensor_extra *) t->extra; - if (extra && (extra->flags & HTP_TENSOR_TILED)) { +static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) { + return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_FUSEABLE) != 0; +} + +static inline bool ggml_hexagon_tensor_is_tiled(const struct ggml_tensor * t) { + return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_TILED) != 0; +} + +static inline bool ggml_hexagon_tensor_can_tile(const struct ggml_tensor * t) { + if (opt_rt_repack == 0) { return false; } - return htp_can_rt_repack(t); + if (t->buffer) { + if (ggml_backend_buft_is_host(t->buffer->buft) || + ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + return false; + } + } + return htp_is_tiled_weight_tensor(t); +} + +static inline bool ggml_hexagon_tensor_will_tile(const struct ggml_tensor * t) { + return ggml_hexagon_tensor_is_tiled(t) || ggml_hexagon_tensor_can_tile(t); } static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) { diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp index ba2dd6410..4de6dd8b7 100644 --- a/ggml/src/ggml-hexagon/htp-fusion.cpp +++ b/ggml/src/ggml-hexagon/htp-fusion.cpp @@ -35,7 +35,7 @@ static inline bool htp_match_consumer( return false; } const ggml_tensor * prev_out = prev.dst(); - if (!htp_tensor_is_fuseable(prev_out)) { + if (!ggml_hexagon_tensor_is_fuseable(prev_out)) { return false; } if (htp_same_tensor_data(cur.src0(), prev_out)) { @@ -53,14 +53,14 @@ static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struc if (kparams->n_hmx) { return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; } - return htp_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW); + return ggml_hexagon_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW); } static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { if (kparams->n_hmx) { return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; } - return htp_tensor_is_tiled(src0) && + return ggml_hexagon_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK); } @@ -74,7 +74,7 @@ static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & p const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; const ggml_tensor * ar_local = prev.inputs[rank]; - if (!htp_tensor_is_fuseable(ar_local)) return false; + if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false; const ggml_tensor * add_dst = cur.dst(); const ggml_tensor * res_tensor = nullptr; @@ -250,7 +250,7 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 || - htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) || + ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) || orig_kparams->n_hmx != cur_kparams->n_hmx) { return false; } @@ -295,7 +295,7 @@ static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & p if (x_in != x || ids_in != ids || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] || - htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) || + ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) || orig_kparams->n_hmx != cur_kparams->n_hmx) { return false; } @@ -598,19 +598,17 @@ int htp_fusion_max(const struct ggml_cgraph * graph, int idx) { void htp_fusion_tag_graph(struct ggml_cgraph * graph) { for (int i = 0; i < graph->n_nodes; i++) { - auto * extra = (htp_tensor_extra *) graph->nodes[i]->extra; - if (!extra) continue; + ggml_tensor * node = graph->nodes[i]; + ggml_hexagon_tensor_clear_flags(node, HTP_TENSOR_FUSEABLE); - extra->flags &= ~HTP_TENSOR_FUSEABLE; - - const ggml_op op = graph->nodes[i]->op; + const ggml_op op = node->op; if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT || op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB || op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID || op == GGML_OP_SSM_CONV || - (op == GGML_OP_MUL_MAT && !htp_is_hadamard(graph->nodes[i]))) && + (op == GGML_OP_MUL_MAT && !htp_is_hadamard(node))) && ggml_node_has_n_uses(graph, i, 1)) { - extra->flags |= HTP_TENSOR_FUSEABLE; + ggml_hexagon_tensor_set_flags(node, HTP_TENSOR_FUSEABLE); } } } diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index 822d1eb08..4fa900bed 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -628,7 +628,7 @@ static bool htp_matmul_is_hmx_eligible( return false; } - if (!htp_is_hmx_weight_type((ggml_type) wtype)) { + if (!htp_is_tiled_weight_type((ggml_type) wtype)) { return false; } @@ -636,7 +636,7 @@ static bool htp_matmul_is_hmx_eligible( return false; } - const bool is_tiled = htp_tensor_is_tiled(src0); + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) { return false; } @@ -674,7 +674,7 @@ static bool htp_kparams_solve_hmx_mm( size_t vtcm_budget, struct htp_mm_kernel_params * kparams ) { - const bool is_tiled = htp_tensor_is_tiled(src0); + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0; const int n_threads = (int) htp_session_n_threads(sess); const uint32_t mdev_count = htp_session_mdev_count(sess); @@ -893,7 +893,7 @@ static bool htp_kparams_solve_matmul_impl( const int ne12 = src1->ne[2]; const int ne13 = src1->ne[3]; - const bool is_tiled = htp_tensor_is_tiled(src0); + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00; const int ne11_padded = hex_round_up(ne11, 32); @@ -1109,7 +1109,7 @@ bool htp_kparams_solve_mmnx( const int ne12 = act->ne[2]; const int ne13 = act->ne[3]; - const bool is_tiled = htp_tensor_is_tiled(src0); + const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00; const int ne11_padded = hex_round_up(ne11, 32); @@ -1683,7 +1683,7 @@ bool htp_kparams_solve_repeat_get_rows( const uint32_t ne12 = src1->ne[2]; const uint32_t nr = ne10 * ne11 * ne12; - const bool tiled = htp_tensor_is_tiled(src0); + const bool tiled = ggml_hexagon_tensor_is_tiled(src0); const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; if (tiled) { diff --git a/ggml/src/ggml-hexagon/htp-repack.h b/ggml/src/ggml-hexagon/htp-repack.h index f656e9b8d..f3206bc87 100644 --- a/ggml/src/ggml-hexagon/htp-repack.h +++ b/ggml/src/ggml-hexagon/htp-repack.h @@ -8,31 +8,30 @@ #include "htp/htp-tensor.h" static inline bool htp_is_tiled_weight_type(enum ggml_type type) { - return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || - type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL || - type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0 || - type == GGML_TYPE_Q6_K || + return type == GGML_TYPE_Q1_0 || type == GGML_TYPE_Q2_0 || + type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || + type == GGML_TYPE_MXFP4 || type == GGML_TYPE_IQ4_NL || + type == GGML_TYPE_Q8_0 || type == GGML_TYPE_Q6_K || + type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || - type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K || - type == GGML_TYPE_Q2_0 || type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || type == GGML_TYPE_BF16; } static inline bool htp_is_tiled_weight_tensor(const struct ggml_tensor * t) { - if (!ggml_is_contiguous(t) || ggml_is_view(t)) { - return false; - } if (!htp_is_tiled_weight_type(t->type)) { return false; } - if (t->ne[0] < 32) { + if (!ggml_is_contiguous(t) || ggml_is_view(t)) { + return false; + } + if (t->ne[0] < 32 || (t->ne[0] % 32 != 0) || t->ne[3] != 1) { return false; } if (!ggml_is_quantized(t->type)) { - return (t->ne[0] % 32 == 0) && (t->ne[1] % 32 == 0) && (t->ne[2] == 1) && (t->ne[3] == 1); + return t->ne[1] > 1; } - return t->ne[3] == 1; + return true; } // Size of one tiled row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins, @@ -59,10 +58,6 @@ static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) { return ggml_row_size(type, ne0); } -static inline bool htp_is_hmx_weight_type(enum ggml_type type) { - return htp_is_tiled_weight_type(type); -} - void htp_repack_tensor_tiled(struct ggml_tensor * t, const void * data, size_t size); void htp_unpack_tensor_tiled(void * data, const struct ggml_tensor * t, size_t offset, size_t size);