hexagon: Implement set_tensor_n and improve repacking/tiling robustness (#31744)
* hex-repack: move repacking/tiling into graph-optimize * hex-tiling: simplify tiling rules and remove obsolete helpers * hex-ops: use a hash for de-duplicating tensor descriptors * hex-bufs: improve tensor hash * hex-ops: accept empty (zero-elements) ops * hex-bufs: enable hostbuf by default and reject tensors that do not fit into max_vmem * hex-bufs: improve error message and update readme * hex-bufs: make sure to sync flush opbatch in all sync set_tensor paths
This commit is contained in:
1 parent
e14a779719
commit
67107ac34f
6 files changed
+231
-222
No files matched your search
@@ -325,10 +325,14 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs
|
||||
Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version).
|
||||
|
||||
- `GGML_HEXAGON_VTCM_SIZE=0`
|
||||
Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
|
||||
Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB.
|
||||
The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
|
||||
|
||||
- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled)
|
||||
Enables allocating host buffers for debugging. By default, host buffers are disabled.
|
||||
- `GGML_HEXAGON_HOSTBUF=0` (default: 1, enabled)
|
||||
Controls host buffer (`HTP*-HOST`) allocation in shared FastRPC memory to avoid host-device tensor copies.
|
||||
CPU-assigned weights (input embeddings like `token_embd` and `per_layer_token_embd`, and tensors overridden with `-ot ...=CPU`)
|
||||
are placed in host buffers as well. On older drivers/FW, large models can run out of IOVA space when these are mapped.
|
||||
Pass `--no-host` to keep CPU-assigned weights in regular CPU memory (`-ot ...=CPU` alone is not sufficient).
|
||||
|
||||
- `GGML_HEXAGON_DMA64=0` (default: enabled on v81+)
|
||||
Disables 64-bit DMA for model weights. Set to `1` to enable it explicitly on a supported architecture.
|
||||
|
||||
@@ -90,7 +90,7 @@ size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for
|
||||
int opt_etm = 0;
|
||||
int opt_verbose = 0;
|
||||
int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)
|
||||
bool opt_hostbuf = false;
|
||||
bool opt_hostbuf = true;
|
||||
bool opt_dma64 = false;
|
||||
int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start
|
||||
|
||||
@@ -150,7 +150,7 @@ int opt_opqueue = 8; // max number of pending batches
|
||||
int opt_optrace = 0; // trace buffer size per thread (0 means default)
|
||||
int opt_oppoll = 0; // polling for batch completions
|
||||
int opt_opfusion = 1; // enable/disable op fusion
|
||||
int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = always
|
||||
int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = all tileable weights (default)
|
||||
|
||||
static std::regex* opt_opfilter = NULL; // regex of ops to not claim
|
||||
|
||||
@@ -463,7 +463,6 @@ struct ggml_hexagon_rpcmem_block {
|
||||
struct ggml_hexagon_shared_buffer {
|
||||
ggml_hexagon_session * sess;
|
||||
std::shared_ptr<ggml_hexagon_rpcmem_block> mem;
|
||||
std::vector<htp_tensor_extra *> tensor_extra;
|
||||
bool mapped;
|
||||
bool pinned;
|
||||
bool extended;
|
||||
@@ -486,6 +485,9 @@ struct ggml_hexagon_shared_buffer {
|
||||
if (err != 0) {
|
||||
GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(),
|
||||
sess->domain_id, size(), fd(), (unsigned) err);
|
||||
if (!this->pinned) {
|
||||
GGML_LOG_ERROR("ggml-hex: %s possibly out of IOVA space, try --no-host\n", sess->c_name());
|
||||
}
|
||||
throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)");
|
||||
}
|
||||
|
||||
@@ -561,9 +563,6 @@ struct ggml_hexagon_shared_buffer {
|
||||
|
||||
~ggml_hexagon_shared_buffer() {
|
||||
free();
|
||||
for (auto * extra : tensor_extra) {
|
||||
delete extra;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -622,11 +621,9 @@ inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) {
|
||||
}
|
||||
|
||||
inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * slot, uint32_t n_slots) const {
|
||||
static htp_tensor_extra fence_extra { {}, 0, HTP_TENSOR_FENCE };
|
||||
|
||||
ggml_tensor t {};
|
||||
t.buffer = &this->fence_buf->backend_buffer;
|
||||
t.extra = &fence_extra;
|
||||
t.extra = (void *) (uintptr_t) HTP_TENSOR_FENCE;
|
||||
t.data = const_cast<void *>(slot);
|
||||
t.type = GGML_TYPE_I8;
|
||||
t.ne[0] = HTP_FENCE_SLOT_SIZE;
|
||||
@@ -659,10 +656,7 @@ static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buf
|
||||
HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(),
|
||||
tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor));
|
||||
|
||||
auto extra = new htp_tensor_extra();
|
||||
sbuf->tensor_extra.push_back(extra);
|
||||
|
||||
tensor->extra = extra;
|
||||
tensor->extra = (void *) (uintptr_t) 0;
|
||||
|
||||
return GGML_STATUS_SUCCESS;
|
||||
}
|
||||
@@ -672,42 +666,19 @@ static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,
|
||||
const void * data,
|
||||
size_t offset,
|
||||
size_t size) {
|
||||
auto extra = (htp_tensor_extra *) tensor->extra;
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
|
||||
sess->flush_sync();
|
||||
|
||||
if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
|
||||
extra->flags |= HTP_TENSOR_WEIGHT;
|
||||
if (htp_is_tiled_weight_tensor(tensor)) {
|
||||
extra->flags |= HTP_TENSOR_TILED;
|
||||
}
|
||||
ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT);
|
||||
}
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",
|
||||
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);
|
||||
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor));
|
||||
|
||||
if ((extra->flags & HTP_TENSOR_TILED) == 0) {
|
||||
memcpy((char *) tensor->data + offset, data, size);
|
||||
return;
|
||||
}
|
||||
|
||||
if (offset == 0 && size == ggml_nbytes(tensor) && extra->shadow_buf.empty()) {
|
||||
htp_repack_tensor_tiled(tensor, data, size);
|
||||
return;
|
||||
}
|
||||
|
||||
if (extra->shadow_buf.size() < ggml_nbytes(tensor)) {
|
||||
extra->shadow_buf.resize(ggml_nbytes(tensor));
|
||||
}
|
||||
memcpy(extra->shadow_buf.data() + offset, data, size);
|
||||
extra->shadow_size += size;
|
||||
|
||||
if (extra->shadow_size >= ggml_nbytes(tensor)) {
|
||||
htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size());
|
||||
extra->shadow_buf.clear();
|
||||
extra->shadow_buf.shrink_to_fit();
|
||||
extra->shadow_size = 0;
|
||||
}
|
||||
memcpy((char *) tensor->data + offset, data, size);
|
||||
}
|
||||
|
||||
static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
|
||||
@@ -715,14 +686,14 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
|
||||
void * data,
|
||||
size_t offset,
|
||||
size_t size) {
|
||||
auto extra = (htp_tensor_extra *) tensor->extra;
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor);
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",
|
||||
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);
|
||||
sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, flags);
|
||||
|
||||
if ((extra->flags & HTP_TENSOR_TILED) == 0) {
|
||||
if ((flags & HTP_TENSOR_TILED) == 0) {
|
||||
memcpy(data, (const char *) tensor->data + offset, size);
|
||||
return;
|
||||
}
|
||||
@@ -742,47 +713,27 @@ static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t bu
|
||||
}
|
||||
|
||||
static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,
|
||||
ggml_tensor * tensor,
|
||||
const void * data,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
size_t n_copies,
|
||||
size_t stride_tensor,
|
||||
size_t stride_data) {
|
||||
auto extra = (htp_tensor_extra *) tensor->extra;
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
ggml_tensor * tensor,
|
||||
const void * data,
|
||||
size_t offset,
|
||||
size_t size,
|
||||
size_t n_copies,
|
||||
size_t stride_tensor,
|
||||
size_t stride_data) {
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
|
||||
sess->flush_sync();
|
||||
|
||||
if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
|
||||
extra->flags |= HTP_TENSOR_WEIGHT;
|
||||
if (htp_is_tiled_weight_tensor(tensor)) {
|
||||
extra->flags |= HTP_TENSOR_TILED;
|
||||
}
|
||||
ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_WEIGHT);
|
||||
}
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n",
|
||||
sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags);
|
||||
sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, ggml_hexagon_tensor_get_flags(tensor));
|
||||
|
||||
if ((extra->flags & HTP_TENSOR_TILED) == 0) {
|
||||
for (size_t i = 0; i < n_copies; i++) {
|
||||
memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (extra->shadow_buf.size() < ggml_nbytes(tensor)) {
|
||||
extra->shadow_buf.resize(ggml_nbytes(tensor));
|
||||
}
|
||||
for (size_t i = 0; i < n_copies; i++) {
|
||||
memcpy(extra->shadow_buf.data() + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
|
||||
}
|
||||
extra->shadow_size += n_copies * size;
|
||||
|
||||
if (extra->shadow_size >= ggml_nbytes(tensor)) {
|
||||
htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size());
|
||||
extra->shadow_buf.clear();
|
||||
extra->shadow_buf.shrink_to_fit();
|
||||
extra->shadow_size = 0;
|
||||
memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -794,14 +745,14 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
|
||||
size_t n_copies,
|
||||
size_t stride_tensor,
|
||||
size_t stride_data) {
|
||||
auto extra = (htp_tensor_extra *) tensor->extra;
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
const uint32_t flags = ggml_hexagon_tensor_get_flags(tensor);
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n",
|
||||
sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage);
|
||||
|
||||
if ((extra->flags & HTP_TENSOR_TILED) == 0) {
|
||||
if ((flags & HTP_TENSOR_TILED) == 0) {
|
||||
for (size_t i = 0; i < n_copies; i++) {
|
||||
memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size);
|
||||
}
|
||||
@@ -825,6 +776,29 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_backend_hexagon_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
|
||||
struct ggml_tensor * const * tensors,
|
||||
const void * const * data,
|
||||
const size_t * offsets,
|
||||
const size_t * sizes,
|
||||
size_t n_tensors) {
|
||||
const bool is_weights = ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
|
||||
sess->flush_sync();
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
if (is_weights) {
|
||||
ggml_hexagon_tensor_set_flags(tensors[i], HTP_TENSOR_WEIGHT);
|
||||
}
|
||||
memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
@@ -844,7 +818,7 @@ static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = {
|
||||
/* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_hexagon_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .set_tensor_n = */ ggml_backend_hexagon_buffer_set_tensor_n,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
@@ -856,8 +830,12 @@ static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t bu
|
||||
const void * data,
|
||||
size_t offset,
|
||||
size_t size) {
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
|
||||
sess->flush_sync();
|
||||
|
||||
memcpy((char *) tensor->data + offset, data, size);
|
||||
GGML_UNUSED(buffer);
|
||||
}
|
||||
|
||||
static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer,
|
||||
@@ -869,6 +847,25 @@ static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t bu
|
||||
GGML_UNUSED(buffer);
|
||||
}
|
||||
|
||||
static void ggml_backend_hexagon_host_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
|
||||
struct ggml_tensor * const * tensors,
|
||||
const void * const * data,
|
||||
const size_t * offsets,
|
||||
const size_t * sizes,
|
||||
size_t n_tensors) {
|
||||
auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;
|
||||
auto sess = sbuf->sess;
|
||||
|
||||
sess->flush_sync();
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = {
|
||||
/* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer,
|
||||
/* .get_base = */ ggml_backend_hexagon_buffer_get_base,
|
||||
@@ -881,7 +878,7 @@ static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = {
|
||||
/* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_hexagon_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .set_tensor_n = */ ggml_backend_hexagon_host_buffer_set_tensor_n,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
@@ -896,7 +893,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer(
|
||||
ggml_backend_buffer_type_t buffer_type, size_t size) {
|
||||
auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx;
|
||||
auto sess = dev_ctx->session();
|
||||
if (sess && sess->max_vmem && size > sess->max_vmem) {
|
||||
if (size > sess->max_vmem) {
|
||||
GGML_LOG_ERROR("ggml-hex: %s buffer size %zu exceeds max_vmem %zu\n",
|
||||
dev_ctx->c_name(), size, sess->max_vmem);
|
||||
return nullptr;
|
||||
@@ -914,7 +911,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer(
|
||||
ggml_backend_buffer_type_t buffer_type, size_t size) {
|
||||
auto dev_ctx = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context)->dev_ctx;
|
||||
auto sess = dev_ctx->session();
|
||||
if (sess && sess->max_vmem && size > sess->max_vmem) {
|
||||
if (size > sess->max_vmem) {
|
||||
GGML_LOG_ERROR("ggml-hex: %s host buffer size %zu exceeds max_vmem %zu\n",
|
||||
dev_ctx->c_name(), size, sess->max_vmem);
|
||||
return nullptr;
|
||||
@@ -934,7 +931,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer
|
||||
}
|
||||
|
||||
static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) {
|
||||
if (!ggml_backend_buft_is_host(buft) && htp_is_tiled_weight_tensor(t)) {
|
||||
if (opt_rt_repack != 0 && !ggml_backend_buft_is_host(buft) && ggml_op_is_empty(t->op) && htp_is_tiled_weight_tensor(t)) {
|
||||
int64_t ne0 = hex_round_up(t->ne[0], 32);
|
||||
int64_t ne1 = hex_round_up(t->ne[1], 32);
|
||||
int64_t ne2 = t->ne[2];
|
||||
@@ -1212,8 +1209,7 @@ struct ggml_hexagon_opbatch {
|
||||
std::vector<htp_tensor> h_tens; // htp tensor descriptors
|
||||
std::vector<htp_op_desc> h_ops; // htp op descriptors
|
||||
|
||||
std::unordered_map<const ggml_tensor*, int> t_map; // tensor ptr to index
|
||||
std::unordered_multimap<void*, int> d_map; // tensor data to index
|
||||
std::unordered_map<uint64_t, int> t_map; // tensor hash to index
|
||||
|
||||
unsigned int n_bufs; // num buffers in the batch
|
||||
unsigned int n_tens; // num tensors ...
|
||||
@@ -1232,7 +1228,6 @@ struct ggml_hexagon_opbatch {
|
||||
b_vmem = 0;
|
||||
|
||||
t_map.clear();
|
||||
d_map.clear();
|
||||
ops.resize(n_ops_max);
|
||||
}
|
||||
|
||||
@@ -1252,7 +1247,6 @@ struct ggml_hexagon_opbatch {
|
||||
h_ops.resize(n_ops_max);
|
||||
|
||||
t_map.reserve(n_tens_max);
|
||||
d_map.reserve(n_tens_max);
|
||||
|
||||
GGML_LOG_INFO("ggml-hex: %s op batching: n-bufs %u n-tensors %u n-ops %u vmem %zu\n",
|
||||
sess->c_name(), n_bufs_max, n_tens_max, n_ops_max, b_vmem_max);
|
||||
@@ -1298,7 +1292,7 @@ struct ggml_hexagon_opbatch {
|
||||
}
|
||||
|
||||
static void init_tensor_shape(htp_tensor & h, const ggml_tensor * t) {
|
||||
auto extra = (htp_tensor_extra *) t->extra;
|
||||
const uint32_t flags = ggml_hexagon_tensor_get_flags(t);
|
||||
|
||||
h.type = t->type;
|
||||
h.ne[0] = t->ne[0];
|
||||
@@ -1306,7 +1300,7 @@ struct ggml_hexagon_opbatch {
|
||||
h.ne[2] = t->ne[2];
|
||||
h.ne[3] = t->ne[3];
|
||||
|
||||
if (extra->flags & HTP_TENSOR_TILED) {
|
||||
if (flags & HTP_TENSOR_TILED) {
|
||||
if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16) {
|
||||
h.type = GGML_TYPE_F16;
|
||||
h.nb[0] = sizeof(ggml_fp16_t);
|
||||
@@ -1327,31 +1321,42 @@ struct ggml_hexagon_opbatch {
|
||||
h.size = ggml_nbytes(t);
|
||||
}
|
||||
|
||||
h.flags = extra->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE);
|
||||
h.flags = flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE);
|
||||
}
|
||||
|
||||
static bool same_shape(const htp_tensor & a, const htp_tensor & b) {
|
||||
return (a.type == b.type) &&
|
||||
(a.ne[0] == b.ne[0]) && (a.ne[1] == b.ne[1]) && (a.ne[2] == b.ne[2]) && (a.ne[3] == b.ne[3]) &&
|
||||
(a.nb[0] == b.nb[0]) && (a.nb[1] == b.nb[1]) && (a.nb[2] == b.nb[2]) && (a.nb[3] == b.nb[3]);
|
||||
static uint64_t hash_tensor(const ggml_tensor * t, const htp_tensor & shape) {
|
||||
uint64_t hash = 1469598103934665603ull;
|
||||
auto hash_value = [&hash](uint64_t value) {
|
||||
hash ^= value;
|
||||
hash *= 1099511628211ull;
|
||||
};
|
||||
|
||||
hash_value((uintptr_t) t->data);
|
||||
hash_value(shape.type);
|
||||
hash_value(shape.size);
|
||||
hash_value(shape.flags);
|
||||
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
|
||||
hash_value(shape.ne[i]);
|
||||
hash_value(shape.nb[i]);
|
||||
}
|
||||
|
||||
// avalanche high bits down to low bits for power-of-two table indexing
|
||||
hash ^= hash >> 33;
|
||||
hash *= 0xff51afd7ed558ccdull;
|
||||
hash ^= hash >> 33;
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
// add tensor and return its index
|
||||
int add_tensor(const ggml_tensor * t) {
|
||||
// Lookup by tensor ptr
|
||||
auto it = t_map.find(t);
|
||||
if (it != t_map.end()) { return it->second; }
|
||||
|
||||
htp_tensor shape {};
|
||||
init_tensor_shape(shape, t);
|
||||
|
||||
// Lookup by tensor data and shape
|
||||
auto range = d_map.equal_range(t->data);
|
||||
for (auto dit = range.first; dit != range.second; ++dit) {
|
||||
if (same_shape(h_tens[dit->second], shape)) {
|
||||
t_map.insert({t, dit->second});
|
||||
return dit->second;
|
||||
}
|
||||
const uint64_t hash = hash_tensor(t, shape);
|
||||
auto it = t_map.find(hash);
|
||||
if (it != t_map.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);
|
||||
@@ -1360,8 +1365,7 @@ struct ggml_hexagon_opbatch {
|
||||
int ti = n_tens++;
|
||||
GGML_ASSERT(n_tens <= n_tens_max);
|
||||
|
||||
t_map.insert({t, ti});
|
||||
d_map.insert({t->data, ti});
|
||||
t_map[hash] = ti;
|
||||
|
||||
uint64_t t_offset = (uint8_t *) t->data - sbuf->base();
|
||||
|
||||
@@ -1391,8 +1395,11 @@ struct ggml_hexagon_opbatch {
|
||||
|
||||
auto fit_tensor = [&](const ggml_tensor * t) {
|
||||
if (!t) return;
|
||||
|
||||
if (check_tens) {
|
||||
if (t_map.count(t)) return;
|
||||
htp_tensor shape {};
|
||||
init_tensor_shape(shape, t);
|
||||
if (t_map.count(hash_tensor(t, shape))) return;
|
||||
extra_tens++;
|
||||
}
|
||||
|
||||
@@ -2364,11 +2371,33 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) {
|
||||
delete backend;
|
||||
}
|
||||
|
||||
static inline bool op_is_compute(ggml_tensor *node)
|
||||
{
|
||||
static inline bool ggml_hexagon_op_is_compute(const struct ggml_tensor * node) {
|
||||
return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE);
|
||||
}
|
||||
|
||||
static void ggml_hexagon_tensor_repack(ggml_tensor * tensor, ggml_hexagon_session * sess) {
|
||||
if (tensor->data == nullptr || ggml_hexagon_tensor_is_tiled(tensor) || !ggml_hexagon_tensor_can_tile(tensor)) {
|
||||
return;
|
||||
}
|
||||
HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(tensor, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(tensor));
|
||||
std::vector<uint8_t> raw(ggml_nbytes(tensor));
|
||||
memcpy(raw.data(), tensor->data, raw.size());
|
||||
htp_repack_tensor_tiled(tensor, raw.data(), raw.size());
|
||||
ggml_hexagon_tensor_set_flags(tensor, HTP_TENSOR_TILED);
|
||||
}
|
||||
|
||||
static void ggml_hexagon_graph_repack(const ggml_cgraph * graph, ggml_hexagon_session * sess) {
|
||||
for (int i = 0; i < graph->n_nodes; ++i) {
|
||||
ggml_tensor * n = graph->nodes[i];
|
||||
if (!ggml_hexagon_op_is_compute(n)) {
|
||||
continue;
|
||||
}
|
||||
if (n->op == GGML_OP_MUL_MAT || n->op == GGML_OP_MUL_MAT_ID) {
|
||||
ggml_hexagon_tensor_repack(n->src[0], sess);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) {
|
||||
auto sess = static_cast<ggml_hexagon_session *>(backend->context);
|
||||
|
||||
@@ -2385,13 +2414,15 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
|
||||
if (cache_hit) {
|
||||
nodes_ptr = &sess->cached_nodes;
|
||||
} else {
|
||||
ggml_hexagon_graph_repack(graph, sess);
|
||||
|
||||
htp_fusion_tag_graph(graph);
|
||||
|
||||
computed_nodes.reserve(graph->n_nodes);
|
||||
|
||||
for (int i = 0; i < graph->n_nodes; ++i) {
|
||||
ggml_tensor * n = graph->nodes[i];
|
||||
if (!op_is_compute(n)) {
|
||||
if (!ggml_hexagon_op_is_compute(n)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -2401,22 +2432,7 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
|
||||
}
|
||||
|
||||
htp_opnode node(opcode, n);
|
||||
if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || node.opcode == HTP_OP_GET_ROWS) {
|
||||
ggml_tensor * src0 = node.node->src[0];
|
||||
if (htp_needs_repack(src0)) {
|
||||
HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(src0, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(src0));
|
||||
std::vector<uint8_t> raw(ggml_nbytes(src0));
|
||||
memcpy(raw.data(), src0->data, raw.size());
|
||||
htp_repack_tensor_tiled(src0, raw.data(), raw.size());
|
||||
auto * extra = (htp_tensor_extra *) src0->extra;
|
||||
extra->flags |= HTP_TENSOR_TILED;
|
||||
}
|
||||
}
|
||||
|
||||
GGML_ASSERT(htp_kparams_solve(sess, node));
|
||||
if (node.opcode == HTP_OP_CPY && node.kparams<htp_copy_kernel_params>()->total_elems == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
computed_nodes.push_back(std::move(node));
|
||||
}
|
||||
@@ -2626,7 +2642,11 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<no
|
||||
}
|
||||
|
||||
static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgraph * gf, ggml_backend_graph_optimize_params * params) {
|
||||
GGML_UNUSED(params);
|
||||
auto sess = static_cast<ggml_hexagon_session *>(backend->context);
|
||||
|
||||
HEX_VERBOSE("ggml-hex: %s graph-optimize n_nodes %d\n", sess->c_name(), gf->n_nodes);
|
||||
|
||||
ggml_hexagon_graph_repack(gf, sess);
|
||||
|
||||
const int n = gf->n_nodes;
|
||||
|
||||
@@ -2661,7 +2681,7 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr
|
||||
}
|
||||
}
|
||||
|
||||
GGML_UNUSED(backend);
|
||||
GGML_UNUSED(params);
|
||||
}
|
||||
|
||||
static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) {
|
||||
@@ -2723,14 +2743,7 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg
|
||||
return false;
|
||||
}
|
||||
|
||||
// FIXME: ggml-meta needs to call init_tensor on auxiliary tensors
|
||||
if (!dst->extra) {
|
||||
ggml_backend_buffer_init_tensor(dst->buffer, dst);
|
||||
}
|
||||
|
||||
auto * dst_extra = static_cast<htp_tensor_extra *>(dst->extra);
|
||||
const auto * src_extra = static_cast<const htp_tensor_extra *>(src->extra);
|
||||
dst_extra->flags = src_extra->flags & ~HTP_TENSOR_FUSEABLE;
|
||||
dst->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(src) & ~HTP_TENSOR_FUSEABLE);
|
||||
|
||||
auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);
|
||||
auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);
|
||||
@@ -3003,18 +3016,18 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
}
|
||||
|
||||
bool supp = false;
|
||||
switch (op->op) {
|
||||
case GGML_OP_NONE:
|
||||
case GGML_OP_RESHAPE:
|
||||
case GGML_OP_VIEW:
|
||||
case GGML_OP_PERMUTE:
|
||||
case GGML_OP_TRANSPOSE:
|
||||
supp = true;
|
||||
break;
|
||||
|
||||
default:
|
||||
if (ggml_op_is_empty(op->op) || ggml_is_empty(op)) {
|
||||
supp = true;
|
||||
} else {
|
||||
supp = ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op) <= sess->max_vmem;
|
||||
for (int i = 0; supp && i < GGML_MAX_SRC; i++) {
|
||||
if (op->src[i] && ggml_backend_hexagon_buffer_type_get_alloc_size(&dev_ctx->buffer_type, op->src[i]) > sess->max_vmem) {
|
||||
supp = false;
|
||||
}
|
||||
}
|
||||
if (supp) {
|
||||
supp = htp_kparams_check(sess, op);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ggml_hexagon_dump_op_supp(sess->c_name(), op, supp);
|
||||
|
||||
@@ -4,12 +4,6 @@
|
||||
#include "htp-opnode.h"
|
||||
#include "htp-repack.h"
|
||||
|
||||
struct htp_tensor_extra {
|
||||
std::vector<uint8_t> shadow_buf;
|
||||
size_t shadow_size { 0 };
|
||||
uint32_t flags { 0 };
|
||||
};
|
||||
|
||||
extern int opt_arch;
|
||||
extern size_t opt_ndev;
|
||||
extern size_t opt_nhvx;
|
||||
@@ -46,36 +40,41 @@ extern int opt_rt_repack;
|
||||
if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__)
|
||||
#endif
|
||||
|
||||
static inline bool htp_tensor_is_fuseable(const struct ggml_tensor * t) {
|
||||
if (!t || !t->extra) return false;
|
||||
const auto * extra = (const struct htp_tensor_extra *) t->extra;
|
||||
return (extra->flags & HTP_TENSOR_FUSEABLE) != 0;
|
||||
static inline uint32_t ggml_hexagon_tensor_get_flags(const struct ggml_tensor * t) {
|
||||
return (uint32_t) (uintptr_t) t->extra;
|
||||
}
|
||||
|
||||
static inline bool htp_can_rt_repack(const struct ggml_tensor * t) {
|
||||
if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) {
|
||||
return false;
|
||||
}
|
||||
if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) {
|
||||
return false;
|
||||
}
|
||||
return opt_rt_repack > 1 || ggml_is_quantized(t->type);
|
||||
static inline void ggml_hexagon_tensor_set_flags(struct ggml_tensor * t, uint32_t flags) {
|
||||
t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) | flags);
|
||||
}
|
||||
|
||||
static inline bool htp_tensor_is_tiled(const struct ggml_tensor * t) {
|
||||
const auto * extra = (const struct htp_tensor_extra *) t->extra;
|
||||
if (extra && (extra->flags & HTP_TENSOR_TILED)) {
|
||||
return true;
|
||||
}
|
||||
return htp_can_rt_repack(t);
|
||||
static inline void ggml_hexagon_tensor_clear_flags(struct ggml_tensor * t, uint32_t flags) {
|
||||
t->extra = (void *) (uintptr_t) (ggml_hexagon_tensor_get_flags(t) & ~flags);
|
||||
}
|
||||
|
||||
static inline bool htp_needs_repack(const struct ggml_tensor * t) {
|
||||
const auto * extra = (const struct htp_tensor_extra *) t->extra;
|
||||
if (extra && (extra->flags & HTP_TENSOR_TILED)) {
|
||||
static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) {
|
||||
return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_FUSEABLE) != 0;
|
||||
}
|
||||
|
||||
static inline bool ggml_hexagon_tensor_is_tiled(const struct ggml_tensor * t) {
|
||||
return (ggml_hexagon_tensor_get_flags(t) & HTP_TENSOR_TILED) != 0;
|
||||
}
|
||||
|
||||
static inline bool ggml_hexagon_tensor_can_tile(const struct ggml_tensor * t) {
|
||||
if (opt_rt_repack == 0) {
|
||||
return false;
|
||||
}
|
||||
return htp_can_rt_repack(t);
|
||||
if (t->buffer) {
|
||||
if (ggml_backend_buft_is_host(t->buffer->buft) ||
|
||||
ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return htp_is_tiled_weight_tensor(t);
|
||||
}
|
||||
|
||||
static inline bool ggml_hexagon_tensor_will_tile(const struct ggml_tensor * t) {
|
||||
return ggml_hexagon_tensor_is_tiled(t) || ggml_hexagon_tensor_can_tile(t);
|
||||
}
|
||||
|
||||
static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) {
|
||||
|
||||
@@ -35,7 +35,7 @@ static inline bool htp_match_consumer(
|
||||
return false;
|
||||
}
|
||||
const ggml_tensor * prev_out = prev.dst();
|
||||
if (!htp_tensor_is_fuseable(prev_out)) {
|
||||
if (!ggml_hexagon_tensor_is_fuseable(prev_out)) {
|
||||
return false;
|
||||
}
|
||||
if (htp_same_tensor_data(cur.src0(), prev_out)) {
|
||||
@@ -53,14 +53,14 @@ static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struc
|
||||
if (kparams->n_hmx) {
|
||||
return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
|
||||
}
|
||||
return htp_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW);
|
||||
return ggml_hexagon_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW);
|
||||
}
|
||||
|
||||
static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {
|
||||
if (kparams->n_hmx) {
|
||||
return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
|
||||
}
|
||||
return htp_tensor_is_tiled(src0) &&
|
||||
return ggml_hexagon_tensor_is_tiled(src0) &&
|
||||
(kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK);
|
||||
}
|
||||
|
||||
@@ -74,7 +74,7 @@ static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & p
|
||||
const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks;
|
||||
|
||||
const ggml_tensor * ar_local = prev.inputs[rank];
|
||||
if (!htp_tensor_is_fuseable(ar_local)) return false;
|
||||
if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false;
|
||||
|
||||
const ggml_tensor * add_dst = cur.dst();
|
||||
const ggml_tensor * res_tensor = nullptr;
|
||||
@@ -250,7 +250,7 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev
|
||||
|
||||
if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
|
||||
w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 ||
|
||||
htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
|
||||
ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) ||
|
||||
orig_kparams->n_hmx != cur_kparams->n_hmx) {
|
||||
return false;
|
||||
}
|
||||
@@ -295,7 +295,7 @@ static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & p
|
||||
|
||||
if (x_in != x || ids_in != ids || w_in->type != w0->type ||
|
||||
w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] ||
|
||||
htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
|
||||
ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0) ||
|
||||
orig_kparams->n_hmx != cur_kparams->n_hmx) {
|
||||
return false;
|
||||
}
|
||||
@@ -598,19 +598,17 @@ int htp_fusion_max(const struct ggml_cgraph * graph, int idx) {
|
||||
|
||||
void htp_fusion_tag_graph(struct ggml_cgraph * graph) {
|
||||
for (int i = 0; i < graph->n_nodes; i++) {
|
||||
auto * extra = (htp_tensor_extra *) graph->nodes[i]->extra;
|
||||
if (!extra) continue;
|
||||
ggml_tensor * node = graph->nodes[i];
|
||||
ggml_hexagon_tensor_clear_flags(node, HTP_TENSOR_FUSEABLE);
|
||||
|
||||
extra->flags &= ~HTP_TENSOR_FUSEABLE;
|
||||
|
||||
const ggml_op op = graph->nodes[i]->op;
|
||||
const ggml_op op = node->op;
|
||||
if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT ||
|
||||
op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB ||
|
||||
op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID ||
|
||||
op == GGML_OP_SSM_CONV ||
|
||||
(op == GGML_OP_MUL_MAT && !htp_is_hadamard(graph->nodes[i]))) &&
|
||||
(op == GGML_OP_MUL_MAT && !htp_is_hadamard(node))) &&
|
||||
ggml_node_has_n_uses(graph, i, 1)) {
|
||||
extra->flags |= HTP_TENSOR_FUSEABLE;
|
||||
ggml_hexagon_tensor_set_flags(node, HTP_TENSOR_FUSEABLE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -628,7 +628,7 @@ static bool htp_matmul_is_hmx_eligible(
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!htp_is_hmx_weight_type((ggml_type) wtype)) {
|
||||
if (!htp_is_tiled_weight_type((ggml_type) wtype)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -636,7 +636,7 @@ static bool htp_matmul_is_hmx_eligible(
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool is_tiled = htp_tensor_is_tiled(src0);
|
||||
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
|
||||
if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) {
|
||||
return false;
|
||||
}
|
||||
@@ -674,7 +674,7 @@ static bool htp_kparams_solve_hmx_mm(
|
||||
size_t vtcm_budget,
|
||||
struct htp_mm_kernel_params * kparams
|
||||
) {
|
||||
const bool is_tiled = htp_tensor_is_tiled(src0);
|
||||
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
|
||||
const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0;
|
||||
const int n_threads = (int) htp_session_n_threads(sess);
|
||||
const uint32_t mdev_count = htp_session_mdev_count(sess);
|
||||
@@ -893,7 +893,7 @@ static bool htp_kparams_solve_matmul_impl(
|
||||
const int ne12 = src1->ne[2];
|
||||
const int ne13 = src1->ne[3];
|
||||
|
||||
const bool is_tiled = htp_tensor_is_tiled(src0);
|
||||
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
|
||||
const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
|
||||
const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00;
|
||||
const int ne11_padded = hex_round_up(ne11, 32);
|
||||
@@ -1109,7 +1109,7 @@ bool htp_kparams_solve_mmnx(
|
||||
const int ne12 = act->ne[2];
|
||||
const int ne13 = act->ne[3];
|
||||
|
||||
const bool is_tiled = htp_tensor_is_tiled(src0);
|
||||
const bool is_tiled = ggml_hexagon_tensor_will_tile(src0);
|
||||
const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
|
||||
const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00;
|
||||
const int ne11_padded = hex_round_up(ne11, 32);
|
||||
@@ -1683,7 +1683,7 @@ bool htp_kparams_solve_repeat_get_rows(
|
||||
const uint32_t ne12 = src1->ne[2];
|
||||
const uint32_t nr = ne10 * ne11 * ne12;
|
||||
|
||||
const bool tiled = htp_tensor_is_tiled(src0);
|
||||
const bool tiled = ggml_hexagon_tensor_is_tiled(src0);
|
||||
const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
|
||||
|
||||
if (tiled) {
|
||||
|
||||
@@ -8,31 +8,30 @@
|
||||
#include "htp/htp-tensor.h"
|
||||
|
||||
static inline bool htp_is_tiled_weight_type(enum ggml_type type) {
|
||||
return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 ||
|
||||
type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL ||
|
||||
type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0 ||
|
||||
type == GGML_TYPE_Q6_K ||
|
||||
return type == GGML_TYPE_Q1_0 || type == GGML_TYPE_Q2_0 ||
|
||||
type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 ||
|
||||
type == GGML_TYPE_MXFP4 || type == GGML_TYPE_IQ4_NL ||
|
||||
type == GGML_TYPE_Q8_0 || type == GGML_TYPE_Q6_K ||
|
||||
type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K ||
|
||||
type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K ||
|
||||
type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K ||
|
||||
type == GGML_TYPE_Q2_0 ||
|
||||
type == GGML_TYPE_F16 || type == GGML_TYPE_F32 ||
|
||||
type == GGML_TYPE_BF16;
|
||||
}
|
||||
|
||||
static inline bool htp_is_tiled_weight_tensor(const struct ggml_tensor * t) {
|
||||
if (!ggml_is_contiguous(t) || ggml_is_view(t)) {
|
||||
return false;
|
||||
}
|
||||
if (!htp_is_tiled_weight_type(t->type)) {
|
||||
return false;
|
||||
}
|
||||
if (t->ne[0] < 32) {
|
||||
if (!ggml_is_contiguous(t) || ggml_is_view(t)) {
|
||||
return false;
|
||||
}
|
||||
if (t->ne[0] < 32 || (t->ne[0] % 32 != 0) || t->ne[3] != 1) {
|
||||
return false;
|
||||
}
|
||||
if (!ggml_is_quantized(t->type)) {
|
||||
return (t->ne[0] % 32 == 0) && (t->ne[1] % 32 == 0) && (t->ne[2] == 1) && (t->ne[3] == 1);
|
||||
return t->ne[1] > 1;
|
||||
}
|
||||
return t->ne[3] == 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Size of one tiled row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins,
|
||||
@@ -59,10 +58,6 @@ static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) {
|
||||
return ggml_row_size(type, ne0);
|
||||
}
|
||||
|
||||
static inline bool htp_is_hmx_weight_type(enum ggml_type type) {
|
||||
return htp_is_tiled_weight_type(type);
|
||||
}
|
||||
|
||||
void htp_repack_tensor_tiled(struct ggml_tensor * t, const void * data, size_t size);
|
||||
void htp_unpack_tensor_tiled(void * data, const struct ggml_tensor * t, size_t offset, size_t size);
|
||||
|
||||
|
||||
Reference in new issue
Block a user