hexagon: support for multi-NPU devices (IQ9, IQ10) and fully asynchronous backend (#26501)
* hexagon: use non-host bufs by default and make the backend fully async * hex-hb: remove optional hostbuf support and fix async copy * hex-unary: relax supported unary check * hex-bufs: use same get_alignment for host bufs * snapdragon: bump android_platform to 34 * hex-rows: super hacky get/set rows for q8_0 * hex-get-rows: fix q8_0 * hex-get-rows: supprot for f16 and cleanup for q8_0 * hex-get-rows: generic macros and specialized thread funcs * hex-get-rows: add DMA pipeline, vtcm_layout and kernel params * hex-set-rows: fix q8_0 support, add dma and tracing * hex-tests: override nmse threshold for HTP of Q8_0 quants * hex-fa: add support for Q8_0 with inplace dequantizers * hex-get-rows: simplify type dispatch * hex-rows: simplify GET/SET_ROWS DMA pipeline * hex-async: add events, set/get-tensor-async and rest of the async api support * hex-repack: use slice instead of expert in repack functions * hex-cpy: update event/async-cpy logging * hex-set-rows: optimize smaller tensors * hex-geglu: fix perf regression with larger tensors * hex-get-rows: add missing header * hex-set-rows: add missing header * hex-bufs: ressurect GGML_HEXAGON_HOSTBUF but disable it by default * hexagon: do not reject ops with non-heaxon buffers * hex-get-rows: apply >=32 restriction only for q8_0 * hex-res: bump vtcm acquire timeout to 10 seconds * hex-bufs: add support for cloning buffers between sessions to speed up tensor copies * hex-async: rework event recording and batch flushing and integrate with meta backend * hex-bufs: improved handling of repacked tensors * hex-repack: handle get_tensor_2d offsets * hex-dev: add support for devices with multiple NPUs * hex-sync: add support for sync tokens to synchronize npu devices for async splits * hex-mmap: cleanup mmap calls and add a retry for robustness * hex-sync: add failsafe if sync wait gets stuck * hex-sync: use sync_seq to check for completed events * hex-sync: rotate tokens for extra robustness * hex-devs: add supprot for legacy device names for now * hex-bufs: add support for auto-cloning buffers from diff sessions * hex-fusion: simplify and optimize htp-opnode fusion handling * hex-sync: override opnode name so that it shows up in the profiles * hex-trace: update scripts to handle multiple devices * hex-sync: bump the size of the opbatch queue and number of sync tokens * hex-cpy-sync: do not explicitly flush opbatches in cpy_tensor_async and add support for cpy-dma * hex-sync: add graph-flush threshold to avoid single op batches * hex-sync: add sync_peer so that we can flush peers we depend on during cross-device ops * hex-bufs: introduce tensor->extra and shadow_bufs for repacking * hex-l2: flush tiny tensors inline * hex-sync: use explicit l2flush for sync tokens * hex-extra: track weight flags via tensor extra * hex-fence: rename sync to fence * hex-repack: proper handling of set-tensor-2d in the shadow_buf * hex-trace: remove obsolete opstage mask that we used for profiling * hex-env: remove obsolete use_hmx variable * hexagon: new unified run.py and build.py and updated docs * snapdragon: update run script to auto-escapt test-backend-op -p argument * hex-scripts: fix trailing spaces * hex-scripts: fix flake8 warnings * snapdragon: cleanup dst lib/bin dirs before copying new build * hex-ops: add support for allreduce * hex-ar: improved allreduce with dma pipeline * hex-ar: align macros * hex-ar: consistent use of fence_seq * hex-ar: add AR_SELECT env var to select ALLREDUCE kernel or fallback * hex-ar: add proper synchronize handling for ALLREDUCE * hex-opbatch: looks like we now just rely on backend.synchronise to flush the batches, no need to flush them by threshold * hex-ar: bump block size to improve dma efficiency * hex-ar: fused ALLREDUCE+ADD * hex-ar: cleaner fence buffer management * hex-ar: futher allreduce tweaking to remove race conditions * hex-ar: add simple solver and remove non-dma kernels * hex-ar: add row-broadcast to fuse with bias ADD * hex-fence: pass seq numbers via op_params * hex-ar: allow for both entry/exit seq for completing entry wait * hex-ar: align macros * hex-ar: do not refetch broadcast row * hex-fusion: move all fusion into opbatch::add_op for consistency with ALLREDUCE and things * hex-fusion: fix incorrect MUL_MAT reordering * hex-mm: make fused 2x and 3x matmuls more generic * hex-fusion: move tensor fusion tagging to graph_compute * hexagon: make sure to copy tensor->extra by value * hex-get-rows: fix offset calc with row-chunking * hex-repack: get_tensor_2d fixes for non-zero offsets * snapdragon: make profile/trace scripts more robust and donot mix stdout/stderr by default * hex-devices: use legacy device nameing by default to ease the transition * hex-devices: hardcode CDSP domain IDs for current devices for now * hex-optrace: improve multi-NPU timestamp alignment and overall handling of cycle values * hex-optrace: more robust handling of the fence events
This commit is contained in:
Max Krasnyansky
+2278
-760
File diff suppressed because it is too large
Load Diff
@@ -8,60 +8,107 @@
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <stdio.h>
|
||||
#include "htp-ops.h"
|
||||
#include "htp/matmul-ops.h"
|
||||
#include "htp/flash-attn-ops.h"
|
||||
#include "htp/unary-ops.h"
|
||||
#include "htp/allreduce-ops.h"
|
||||
|
||||
struct htp_opnode {
|
||||
ggml_tensor * node = nullptr;
|
||||
ggml_tensor * node { nullptr };
|
||||
htp_op_code opcode { HTP_OP_INVALID };
|
||||
int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0};
|
||||
|
||||
std::vector<ggml_tensor *> fused;
|
||||
std::vector<ggml_tensor *> fused;
|
||||
std::vector<std::shared_ptr<ggml_tensor>> dummy;
|
||||
|
||||
htp_op_code opcode = HTP_OP_INVALID;
|
||||
std::vector<const ggml_tensor *> inputs;
|
||||
std::vector<const ggml_tensor *> outputs;
|
||||
std::string name;
|
||||
|
||||
std::vector<ggml_tensor *> extra_dsts;
|
||||
|
||||
int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] = {0};
|
||||
|
||||
htp_opnode(ggml_tensor * node = nullptr, std::vector<ggml_tensor *> fused = {}, htp_op_code opcode = HTP_OP_INVALID, std::vector<ggml_tensor *> extra_dsts = {})
|
||||
: node(node), fused(std::move(fused)), opcode(opcode), extra_dsts(std::move(extra_dsts)) {}
|
||||
|
||||
ggml_op op() const {
|
||||
return node->op;
|
||||
int n_active_src(const ggml_tensor * t) const {
|
||||
if (!t) return 0;
|
||||
for (int i = GGML_MAX_SRC - 1; i >= 0; i--) {
|
||||
if (t->src[i]) {
|
||||
return i + 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const ggml_tensor * dst() const {
|
||||
return fused.empty() ? node : fused.back();
|
||||
void init(ggml_tensor * node) {
|
||||
this->node = node;
|
||||
if (this->node) {
|
||||
this->name = ggml_op_desc(this->node);
|
||||
|
||||
// Build inputs (preserving optional nullptrs)
|
||||
int n_inputs = n_active_src(this->node);
|
||||
this->inputs.resize(n_inputs, nullptr);
|
||||
for (int i = 0; i < n_inputs; i++) {
|
||||
this->inputs[i] = this->node->src[i];
|
||||
}
|
||||
|
||||
// Build outputs
|
||||
this->outputs.push_back(this->dst());
|
||||
}
|
||||
}
|
||||
|
||||
htp_opnode(htp_op_code opcode = HTP_OP_INVALID, ggml_tensor * node = nullptr) : opcode(opcode) {
|
||||
init(node);
|
||||
}
|
||||
|
||||
ggml_op op() const { return node->op; }
|
||||
const ggml_tensor * src0() const { return node->src[0]; }
|
||||
const ggml_tensor * src1() const { return node->src[1]; }
|
||||
const ggml_tensor * dst() const { return outputs.empty() ? node : outputs.back(); }
|
||||
|
||||
ggml_tensor * add_dummy(const ggml_tensor & t) {
|
||||
dummy.push_back(std::make_shared<ggml_tensor>(t));
|
||||
return dummy.back().get();
|
||||
}
|
||||
|
||||
void add_fused(ggml_tensor * t, bool extra_dst = false) {
|
||||
fused.push_back(t);
|
||||
if (extra_dst) {
|
||||
extra_dsts.push_back(t);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<const ggml_tensor *> get_outputs() const {
|
||||
std::vector<const ggml_tensor *> res;
|
||||
if (extra_dsts.empty()) {
|
||||
res.push_back(dst());
|
||||
name += "+";
|
||||
name += ggml_op_desc(t);
|
||||
|
||||
if (extra_dst) {
|
||||
outputs.push_back(t);
|
||||
} else {
|
||||
res.push_back(node);
|
||||
for (const auto * x : extra_dsts) {
|
||||
res.push_back(x);
|
||||
outputs.clear();
|
||||
outputs.push_back(t);
|
||||
}
|
||||
|
||||
// Remove the newly fused intermediate output tensor t from inputs (if it was there)
|
||||
inputs.erase(std::remove(inputs.begin(), inputs.end(), t), inputs.end());
|
||||
|
||||
// Append new inputs from t, preserving middle nullptrs
|
||||
int n_inputs = n_active_src(t);
|
||||
for (int i = 0; i < n_inputs; i++) {
|
||||
const auto * src = t->src[i];
|
||||
if (!src) {
|
||||
inputs.push_back(nullptr);
|
||||
} else if (src != node &&
|
||||
std::find(fused.begin(), fused.end(), src) == fused.end() &&
|
||||
std::find(inputs.begin(), inputs.end(), src) == inputs.end()) {
|
||||
inputs.push_back(src);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
const ggml_tensor * src0() const {
|
||||
return node->src[0];
|
||||
const std::vector<const ggml_tensor *> & get_inputs() const {
|
||||
return inputs;
|
||||
}
|
||||
|
||||
const ggml_tensor * src1() const {
|
||||
return node->src[1];
|
||||
const std::vector<const ggml_tensor *> & get_outputs() const {
|
||||
return outputs;
|
||||
}
|
||||
|
||||
std::string op_name() const {
|
||||
return name;
|
||||
}
|
||||
|
||||
bool is_empty() const {
|
||||
@@ -81,75 +128,6 @@ struct htp_opnode {
|
||||
bool same_input(const htp_opnode& n) const {
|
||||
return n.src1() == this->src1();
|
||||
}
|
||||
|
||||
std::vector<const ggml_tensor *> get_inputs() const {
|
||||
if (fused.empty()) {
|
||||
int last_non_null = -1;
|
||||
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
||||
if (node->src[i]) {
|
||||
last_non_null = i;
|
||||
}
|
||||
}
|
||||
std::vector<const ggml_tensor *> inputs(last_non_null + 1, nullptr);
|
||||
for (int i = 0; i <= last_non_null; i++) {
|
||||
inputs[i] = node->src[i];
|
||||
}
|
||||
return inputs;
|
||||
}
|
||||
|
||||
std::vector<const ggml_tensor *> inputs(GGML_MAX_SRC, nullptr);
|
||||
std::vector<const ggml_tensor *> outputs;
|
||||
outputs.push_back(node);
|
||||
for (const auto * f : fused) {
|
||||
outputs.push_back(f);
|
||||
}
|
||||
|
||||
auto contains = [&](const std::vector<const ggml_tensor *> & vec, const ggml_tensor * t) {
|
||||
for (const auto * x : vec) {
|
||||
if (x == t) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
int count = 0;
|
||||
auto add_input = [&](const ggml_tensor * t) {
|
||||
if (t && !contains(outputs, t) && !contains(inputs, t)) {
|
||||
if (count < (int)inputs.size()) {
|
||||
inputs[count++] = t;
|
||||
} else {
|
||||
inputs.push_back(t);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
||||
if (node->src[i]) {
|
||||
add_input(node->src[i]);
|
||||
}
|
||||
}
|
||||
for (const auto * f : fused) {
|
||||
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
||||
if (f->src[i]) {
|
||||
add_input(f->src[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inputs.resize(count);
|
||||
return inputs;
|
||||
}
|
||||
|
||||
std::string op_name() const {
|
||||
if (fused.empty()) {
|
||||
return ggml_op_desc(node);
|
||||
}
|
||||
std::string name = ggml_op_desc(node);
|
||||
for (const auto * f : fused) {
|
||||
name += "+";
|
||||
name += ggml_op_desc(f);
|
||||
}
|
||||
return name;
|
||||
}
|
||||
};
|
||||
|
||||
struct htp_opformat {
|
||||
@@ -337,8 +315,7 @@ struct htp_opformat {
|
||||
}
|
||||
void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) {
|
||||
if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID ||
|
||||
node.opcode == HTP_OP_MUL_MAT_QKV || node.opcode == HTP_OP_MUL_MAT_FFN ||
|
||||
node.opcode == HTP_OP_MUL_MAT_ADD) {
|
||||
node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ADD) {
|
||||
const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params;
|
||||
const char * path = "unknown";
|
||||
int32_t type = kparams->kernel_type;
|
||||
|
||||
@@ -43,6 +43,7 @@ add_library(${HTP_LIB} SHARED
|
||||
pad-ops.c
|
||||
argsort-ops.c
|
||||
im2col-ops.c
|
||||
allreduce-ops.c
|
||||
)
|
||||
|
||||
target_compile_definitions(${HTP_LIB} PRIVATE
|
||||
|
||||
@@ -183,6 +183,53 @@ static void swiglu_oai_f32(const float * restrict src0,
|
||||
static const float GELU_COEF_A = 0.044715f;
|
||||
static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
|
||||
|
||||
static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) {
|
||||
v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
|
||||
v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3));
|
||||
|
||||
HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
|
||||
HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
|
||||
HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x);
|
||||
|
||||
HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2));
|
||||
v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
|
||||
|
||||
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1));
|
||||
v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx);
|
||||
v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x);
|
||||
|
||||
HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
|
||||
v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
|
||||
|
||||
HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
|
||||
HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
|
||||
|
||||
// Newton-Raphson with 2 iterations
|
||||
HVX_Vector two_sf = hvx_vec_splat_f32(2.0f);
|
||||
HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5);
|
||||
HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(
|
||||
i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));
|
||||
r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
|
||||
r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));
|
||||
HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf);
|
||||
|
||||
res = Q6_Vqf32_vmpy_VsfVsf(v3, res);
|
||||
|
||||
return Q6_Vsf_equals_Vqf32(res);
|
||||
}
|
||||
|
||||
static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v,
|
||||
HVX_Vector one,
|
||||
HVX_Vector max_exp,
|
||||
HVX_Vector min_exp) {
|
||||
const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v);
|
||||
const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp);
|
||||
|
||||
HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v);
|
||||
out = Q6_V_vmux_QVV(pred_max, out, one);
|
||||
return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero());
|
||||
}
|
||||
|
||||
static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
|
||||
assert((unsigned long) dst % 128 == 0);
|
||||
assert((unsigned long) src0 % 128 == 0);
|
||||
@@ -200,20 +247,13 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest
|
||||
|
||||
const HVX_Vector v_coef_a_times_sqrt = hvx_vec_splat_f32(GELU_COEF_A_TIMES_SQRT);
|
||||
const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f32(SQRT_2_OVER_PI);
|
||||
const HVX_Vector v_half = hvx_vec_splat_f32(0.5f);
|
||||
const HVX_Vector v_one = hvx_vec_splat_f32(1.0f);
|
||||
const HVX_Vector v_two = hvx_vec_splat_f32(2.0f);
|
||||
|
||||
// Hoisted fast sigmoid / inverse constants to avoid loop-internal overhead
|
||||
const HVX_Vector v_log2f = Q6_V_vsplat_R(FAST_SIGMOID_LOG2F);
|
||||
const HVX_Vector v_c1 = Q6_V_vsplat_R(FAST_SIGMOID_C1);
|
||||
const HVX_Vector v_c2 = Q6_V_vsplat_R(FAST_SIGMOID_C2);
|
||||
const HVX_Vector v_inv_aprox = Q6_V_vsplat_R(0x7EEEEBB3);
|
||||
const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f);
|
||||
const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f);
|
||||
|
||||
uint32_t i = 0;
|
||||
|
||||
_Pragma("unroll(4)")
|
||||
for (; i < nvec; i++) {
|
||||
HVX_Vector x = vsrc0[i];
|
||||
HVX_Vector g = vsrc1[i];
|
||||
@@ -223,56 +263,13 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest
|
||||
coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi);
|
||||
HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef);
|
||||
|
||||
// y2 = 2 * inner
|
||||
HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two);
|
||||
// y2 = 2 * inner = inner + inner
|
||||
HVX_Vector y2 = hvx_vec_add_f32_f32(inner, inner);
|
||||
|
||||
// Sigmoid guard check predicates
|
||||
HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2);
|
||||
HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp);
|
||||
|
||||
// Fast sigmoid approximation
|
||||
HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f);
|
||||
v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half);
|
||||
|
||||
HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
|
||||
HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
|
||||
HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig);
|
||||
|
||||
HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2);
|
||||
v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f);
|
||||
|
||||
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1);
|
||||
v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig);
|
||||
v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig);
|
||||
|
||||
HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
|
||||
v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
|
||||
|
||||
HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
|
||||
HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
|
||||
|
||||
// Fast division (Newton-Raphson with 2 iterations)
|
||||
HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5);
|
||||
HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(
|
||||
i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));
|
||||
r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
|
||||
r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));
|
||||
HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf);
|
||||
|
||||
HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv));
|
||||
|
||||
// Sigmoid guards
|
||||
sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one);
|
||||
sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero());
|
||||
|
||||
// tanh(inner) = 2 * sigmoid(2 * inner) - 1
|
||||
HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two);
|
||||
tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one);
|
||||
|
||||
HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one);
|
||||
HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half);
|
||||
HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one);
|
||||
// Fast sigmoid approximation (2 iterations)
|
||||
HVX_Vector sig2y = hvx_vec_fast_sigmoid_f32_guard_2it(y2, v_one, v_max_exp, v_min_exp);
|
||||
|
||||
HVX_Vector gelu_x = hvx_vec_mul_f32_f32(x, sig2y);
|
||||
vdst[i] = hvx_vec_mul_f32_f32(gelu_x, g);
|
||||
}
|
||||
|
||||
@@ -285,50 +282,11 @@ static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * rest
|
||||
coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi);
|
||||
HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef);
|
||||
|
||||
HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two);
|
||||
HVX_Vector y2 = hvx_vec_add_f32_f32(inner, inner);
|
||||
|
||||
HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2);
|
||||
HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp);
|
||||
|
||||
HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f);
|
||||
v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half);
|
||||
|
||||
HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
|
||||
HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
|
||||
HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig);
|
||||
|
||||
HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2);
|
||||
v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f);
|
||||
|
||||
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1);
|
||||
v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig);
|
||||
v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig);
|
||||
|
||||
HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
|
||||
v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
|
||||
|
||||
HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
|
||||
HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
|
||||
|
||||
HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5);
|
||||
HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(
|
||||
i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));
|
||||
r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
|
||||
r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));
|
||||
HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf);
|
||||
|
||||
HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv));
|
||||
|
||||
sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one);
|
||||
sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero());
|
||||
|
||||
HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two);
|
||||
tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one);
|
||||
|
||||
HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one);
|
||||
HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half);
|
||||
HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one);
|
||||
HVX_Vector sig2y = hvx_vec_fast_sigmoid_f32_guard_2it(y2, v_one, v_max_exp, v_min_exp);
|
||||
|
||||
HVX_Vector gelu_x = hvx_vec_mul_f32_f32(x, sig2y);
|
||||
HVX_Vector res = hvx_vec_mul_f32_f32(gelu_x, g);
|
||||
hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
#pragma clang diagnostic ignored "-Wunused-variable"
|
||||
#pragma clang diagnostic ignored "-Wunused-function"
|
||||
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
|
||||
|
||||
#include <HAP_farf.h>
|
||||
#include <HAP_perf.h>
|
||||
#include <stdatomic.h>
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
|
||||
#define GGML_COMMON_DECL_C
|
||||
#include "ggml-common.h"
|
||||
#include "htp-ctx.h"
|
||||
#include "htp-ops.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "htp-tensor.h"
|
||||
#include "hex-dma.h"
|
||||
#include "hex-profile.h"
|
||||
#include "allreduce-ops.h"
|
||||
|
||||
struct htp_allreduce_context {
|
||||
struct htp_ops_context * octx;
|
||||
uint32_t n_ranks;
|
||||
uint32_t n_dsts;
|
||||
uint32_t nelem;
|
||||
uint32_t ne0;
|
||||
uint32_t ne1;
|
||||
uint32_t row_size_aligned;
|
||||
uint32_t rank_elem_start;
|
||||
uint32_t rank_nelem;
|
||||
uint32_t elems_per_thread;
|
||||
uint32_t block_elems;
|
||||
uint32_t vtcm_size_per_thread;
|
||||
bool is_row_bcast;
|
||||
uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS];
|
||||
uint8_t * dst_spad_base;
|
||||
uint8_t * res_spad_base;
|
||||
};
|
||||
|
||||
#define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \
|
||||
static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
|
||||
struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \
|
||||
struct htp_ops_context * octx = actx->octx; \
|
||||
\
|
||||
const uint32_t n_ranks = actx->n_ranks; \
|
||||
const uint32_t n_dsts = actx->n_dsts; \
|
||||
const uint32_t block_elems = actx->block_elems; \
|
||||
\
|
||||
const uint32_t dr = actx->elems_per_thread; \
|
||||
const uint32_t ir0 = actx->rank_elem_start + dr * ith; \
|
||||
const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \
|
||||
if (ir0 >= ir1) return; \
|
||||
\
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
|
||||
dma_queue * q = octx->ctx->dma[ith]; \
|
||||
\
|
||||
uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \
|
||||
for (uint32_t s = 0; s < n_ranks; s++) { \
|
||||
src_spad_base[s] = actx->src_spad_base[s] + (ith * actx->vtcm_size_per_thread); \
|
||||
} \
|
||||
uint8_t * dst_spad_base = actx->dst_spad_base + (ith * actx->vtcm_size_per_thread); \
|
||||
uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + (ith * actx->vtcm_size_per_thread)) : NULL; \
|
||||
\
|
||||
const size_t spad_half = actx->vtcm_size_per_thread / 2; \
|
||||
uint32_t ir_prefetch = ir0; \
|
||||
int spad_idx = 0; \
|
||||
\
|
||||
for (int k = 0; k < 2 && ir_prefetch < ir1; k++) { \
|
||||
uint32_t cur_elems = MIN(block_elems, ir1 - ir_prefetch); \
|
||||
size_t cur_bytes = cur_elems * sizeof(TYPE); \
|
||||
uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \
|
||||
for (uint32_t d = 0; d < n_dsts; d++) { \
|
||||
uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + ir_prefetch * sizeof(TYPE); \
|
||||
dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \
|
||||
} \
|
||||
for (uint32_t s = 0; s < n_ranks; s++) { \
|
||||
uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \
|
||||
const uint8_t * s_ddr = (const uint8_t *) octx->src[s]->data + ir_prefetch * sizeof(TYPE); \
|
||||
dma_queue_push(q, dma_make_ptr(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \
|
||||
} \
|
||||
if (HAS_ADD) { \
|
||||
uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \
|
||||
const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \
|
||||
dma_queue_push(q, dma_make_ptr(r_spad, r_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \
|
||||
} \
|
||||
ir_prefetch += cur_elems; \
|
||||
spad_idx ^= 1; \
|
||||
} \
|
||||
\
|
||||
for (uint32_t ir = ir0; ir < ir1; ) { \
|
||||
uint32_t cur_elems = MIN(block_elems, ir1 - ir); \
|
||||
size_t cur_bytes = cur_elems * sizeof(TYPE); \
|
||||
uint8_t * d_spad = NULL; \
|
||||
for (uint32_t d = 0; d < n_dsts; d++) { \
|
||||
d_spad = (uint8_t *) dma_queue_pop(q).src; \
|
||||
} \
|
||||
uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \
|
||||
for (uint32_t s = 0; s < n_ranks; s++) { \
|
||||
s_spad[s] = (uint8_t *) dma_queue_pop(q).dst; \
|
||||
} \
|
||||
uint8_t * r_spad = HAS_ADD ? (uint8_t *) dma_queue_pop(q).dst : NULL; \
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \
|
||||
HVX_ADD_FN(d_spad, s_spad[0], s_spad[1], cur_elems); \
|
||||
for (uint32_t s = 2; s < n_ranks; s++) { \
|
||||
HVX_ADD_FN(d_spad, d_spad, s_spad[s], cur_elems); \
|
||||
} \
|
||||
if (HAS_ADD) { \
|
||||
HVX_ADD_FN(d_spad, d_spad, r_spad, cur_elems); \
|
||||
} \
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \
|
||||
for (uint32_t d = 0; d < n_dsts; d++) { \
|
||||
uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + ir * sizeof(TYPE); \
|
||||
dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 1); \
|
||||
} \
|
||||
if (ir_prefetch < ir1) { \
|
||||
uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \
|
||||
size_t next_bytes = next_elems * sizeof(TYPE); \
|
||||
for (uint32_t s = 0; s < n_ranks; s++) { \
|
||||
const uint8_t * s_next = (const uint8_t *) octx->src[s]->data + ir_prefetch * sizeof(TYPE); \
|
||||
dma_queue_push(q, dma_make_ptr(s_spad[s], s_next), next_bytes, next_bytes, next_bytes, 1); \
|
||||
} \
|
||||
if (HAS_ADD) { \
|
||||
const uint8_t * r_next = (const uint8_t *) octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \
|
||||
dma_queue_push(q, dma_make_ptr(r_spad, r_next), next_bytes, next_bytes, next_bytes, 1); \
|
||||
} \
|
||||
ir_prefetch += next_elems; \
|
||||
} \
|
||||
ir += cur_elems; \
|
||||
} \
|
||||
dma_queue_flush(q); \
|
||||
}
|
||||
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_1D(f16, __fp16, hvx_add_f16_aaa, 0)
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_1D(f32, float, hvx_add_f32_aaa, 0)
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1)
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1)
|
||||
|
||||
#define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \
|
||||
static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
|
||||
struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \
|
||||
struct htp_ops_context * octx = actx->octx; \
|
||||
\
|
||||
const uint32_t n_ranks = actx->n_ranks; \
|
||||
const uint32_t n_dsts = actx->n_dsts; \
|
||||
const uint32_t ne0 = actx->ne0; \
|
||||
const uint32_t block_rows = actx->block_elems; \
|
||||
const uint32_t row_size_aligned = actx->row_size_aligned; \
|
||||
const uint32_t row_bytes = ne0 * sizeof(TYPE); \
|
||||
\
|
||||
const uint32_t dr = actx->elems_per_thread; \
|
||||
const uint32_t r0 = actx->rank_elem_start + dr * ith; \
|
||||
const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \
|
||||
if (r0 >= r1) return; \
|
||||
\
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
|
||||
dma_queue * q = octx->ctx->dma[ith]; \
|
||||
\
|
||||
uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; \
|
||||
for (uint32_t s = 0; s < n_ranks; s++) { \
|
||||
src_spad_base[s] = actx->src_spad_base[s] + (ith * actx->vtcm_size_per_thread); \
|
||||
} \
|
||||
uint8_t * dst_spad_base = actx->dst_spad_base + (ith * actx->vtcm_size_per_thread); \
|
||||
uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + (ith * actx->vtcm_size_per_thread))) : NULL; \
|
||||
\
|
||||
const size_t spad_half = actx->vtcm_size_per_thread / 2; \
|
||||
uint32_t r_prefetch = r0; \
|
||||
int spad_idx = 0; \
|
||||
\
|
||||
for (int k = 0; k < 2 && r_prefetch < r1; k++) { \
|
||||
uint32_t cur_rows = MIN(block_rows, r1 - r_prefetch); \
|
||||
uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \
|
||||
for (uint32_t d = 0; d < n_dsts; d++) { \
|
||||
uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + r_prefetch * octx->dsts[d]->nb[1]; \
|
||||
dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \
|
||||
} \
|
||||
for (uint32_t s = 0; s < n_ranks; s++) { \
|
||||
uint8_t * s_spad = src_spad_base[s] + spad_idx * spad_half; \
|
||||
const uint8_t * s_ddr = (const uint8_t *) octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \
|
||||
dma_queue_push(q, dma_make_ptr(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \
|
||||
} \
|
||||
if (HAS_ADD && !IS_ROW_BCAST) { \
|
||||
uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \
|
||||
const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \
|
||||
dma_queue_push(q, dma_make_ptr(r_spad, r_ddr), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, cur_rows); \
|
||||
} \
|
||||
r_prefetch += cur_rows; \
|
||||
spad_idx ^= 1; \
|
||||
} \
|
||||
\
|
||||
for (uint32_t r = r0; r < r1; ) { \
|
||||
uint32_t cur_rows = MIN(block_rows, r1 - r); \
|
||||
uint8_t * d_spad = NULL; \
|
||||
for (uint32_t d = 0; d < n_dsts; d++) { \
|
||||
d_spad = (uint8_t *) dma_queue_pop(q).src; \
|
||||
} \
|
||||
uint8_t * s_spad[HTP_ALLREDUCE_MAX_RANKS]; \
|
||||
for (uint32_t s = 0; s < n_ranks; s++) { \
|
||||
s_spad[s] = (uint8_t *) dma_queue_pop(q).dst; \
|
||||
} \
|
||||
uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (uint8_t *) dma_queue_pop(q).dst : NULL; \
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
|
||||
for (uint32_t row = 0; row < cur_rows; row++) { \
|
||||
uint8_t * d_row = d_spad + row * row_size_aligned; \
|
||||
const uint8_t * s0_row = s_spad[0] + row * row_size_aligned; \
|
||||
const uint8_t * s1_row = s_spad[1] + row * row_size_aligned; \
|
||||
HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \
|
||||
for (uint32_t s = 2; s < n_ranks; s++) { \
|
||||
const uint8_t * ss_row = s_spad[s] + row * row_size_aligned; \
|
||||
HVX_ADD_FN(d_row, d_row, ss_row, ne0); \
|
||||
} \
|
||||
if (HAS_ADD) { \
|
||||
const uint8_t * res_row = IS_ROW_BCAST ? res_spad_base : (r_spad + row * row_size_aligned); \
|
||||
HVX_ADD_FN(d_row, d_row, res_row, ne0); \
|
||||
} \
|
||||
} \
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
|
||||
for (uint32_t d = 0; d < n_dsts; d++) { \
|
||||
uint8_t * d_ddr = (uint8_t *) octx->dsts[d]->data + r * octx->dsts[d]->nb[1]; \
|
||||
dma_queue_push(q, dma_make_ptr(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, cur_rows); \
|
||||
} \
|
||||
if (r_prefetch < r1) { \
|
||||
uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \
|
||||
for (uint32_t s = 0; s < n_ranks; s++) { \
|
||||
const uint8_t * s_next = (const uint8_t *) octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \
|
||||
dma_queue_push(q, dma_make_ptr(s_spad[s], s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \
|
||||
} \
|
||||
if (HAS_ADD && !IS_ROW_BCAST) { \
|
||||
const uint8_t * r_next = (const uint8_t *) octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \
|
||||
dma_queue_push(q, dma_make_ptr(r_spad, r_next), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, next_rows); \
|
||||
} \
|
||||
r_prefetch += next_rows; \
|
||||
} \
|
||||
r += cur_rows; \
|
||||
} \
|
||||
dma_queue_flush(q); \
|
||||
}
|
||||
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_2D(f16, __fp16, hvx_add_f16_aaa, 0, 0)
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_2D(f32, float, hvx_add_f32_aaa, 0, 0)
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f16, __fp16, hvx_add_f16_aaa, 1, 0)
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0)
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1)
|
||||
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1)
|
||||
|
||||
int op_allreduce(struct htp_ops_context * octx) {
|
||||
const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params;
|
||||
const struct htp_tensor * dst = octx->dst;
|
||||
|
||||
const uint32_t rank = (uint32_t) kparams->rank;
|
||||
const uint32_t n_ranks = (uint32_t) kparams->n_ranks;
|
||||
|
||||
if (n_ranks < 2 || n_ranks > HTP_ALLREDUCE_MAX_RANKS || rank >= n_ranks) {
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
|
||||
if (dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_F32) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3];
|
||||
const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0];
|
||||
const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1];
|
||||
|
||||
// 1. Entry Barrier: Synchronize all ranks before reading
|
||||
struct htp_thread_trace * tr0 = &octx->ctx->trace[0];
|
||||
htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
|
||||
|
||||
const struct htp_tensor * my_sync = octx->src[n_ranks + rank];
|
||||
atomic_uint * my_fence = (atomic_uint *) my_sync->data;
|
||||
|
||||
atomic_store(&my_fence[0], fence_seq_entry);
|
||||
asm volatile ("syncht" : : : "memory");
|
||||
Q6_dccleaninva_A((void *) my_fence);
|
||||
|
||||
for (uint32_t j = 0; j < n_ranks; j++) {
|
||||
if (j == rank) continue;
|
||||
const struct htp_tensor * peer_sync = octx->src[n_ranks + j];
|
||||
atomic_uint * peer_fence = (atomic_uint *) peer_sync->data;
|
||||
uint64_t spins = 0;
|
||||
while (1) {
|
||||
Q6_dccleaninva_A((void *) peer_fence);
|
||||
uint32_t val = atomic_load(&peer_fence[0]);
|
||||
if (val == fence_seq_entry || val == fence_seq_exit) {
|
||||
break;
|
||||
}
|
||||
if (++spins > HTP_FENCE_TIMEOUT) {
|
||||
FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_entry);
|
||||
return HTP_STATUS_INTERNAL_ERR;
|
||||
}
|
||||
hex_pause();
|
||||
}
|
||||
}
|
||||
asm volatile ("syncht" : : : "memory");
|
||||
|
||||
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
|
||||
|
||||
// 2. Multi-threaded Reduction across assigned rank chunk
|
||||
if (nelem > 0) {
|
||||
const uint32_t n_threads = (uint32_t) kparams->n_threads;
|
||||
const uint32_t block_elems = (uint32_t) kparams->block_elems;
|
||||
const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread;
|
||||
const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread;
|
||||
|
||||
const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD);
|
||||
|
||||
struct htp_allreduce_context actx;
|
||||
actx.octx = octx;
|
||||
actx.n_ranks = n_ranks;
|
||||
actx.n_dsts = (uint32_t) kparams->n_dsts ? (uint32_t) kparams->n_dsts : n_ranks;
|
||||
actx.nelem = nelem;
|
||||
actx.ne0 = (uint32_t) kparams->ne0;
|
||||
actx.ne1 = (uint32_t) kparams->ne1;
|
||||
actx.row_size_aligned = (uint32_t) kparams->row_size_aligned;
|
||||
actx.rank_elem_start = (uint32_t) kparams->rank_elem_start;
|
||||
actx.rank_nelem = (uint32_t) kparams->rank_nelem;
|
||||
actx.elems_per_thread = elems_per_thread;
|
||||
actx.block_elems = block_elems;
|
||||
actx.vtcm_size_per_thread = vtcm_size_per_thread;
|
||||
actx.is_row_bcast = (kparams->is_row_bcast != 0);
|
||||
|
||||
work_queue_func_t reduce_fun = NULL;
|
||||
switch (kparams->kernel_type) {
|
||||
case HTP_ALLREDUCE_KERNEL_DMA_1D:
|
||||
if (has_add) {
|
||||
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_add_f16 : allreduce_thread_dma_1d_add_f32;
|
||||
} else {
|
||||
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_f16 : allreduce_thread_dma_1d_f32;
|
||||
}
|
||||
break;
|
||||
case HTP_ALLREDUCE_KERNEL_DMA_2D:
|
||||
if (has_add) {
|
||||
if (kparams->is_row_bcast) {
|
||||
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_bcast_f16 : allreduce_thread_dma_2d_add_bcast_f32;
|
||||
} else {
|
||||
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_f16 : allreduce_thread_dma_2d_add_f32;
|
||||
}
|
||||
} else {
|
||||
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_f16 : allreduce_thread_dma_2d_f32;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base;
|
||||
for (uint32_t s = 0; s < n_ranks; s++) {
|
||||
actx.src_spad_base[s] = vtcm_ptr;
|
||||
vtcm_ptr += n_threads * vtcm_size_per_thread;
|
||||
}
|
||||
actx.dst_spad_base = vtcm_ptr;
|
||||
vtcm_ptr += n_threads * vtcm_size_per_thread;
|
||||
if (has_add) {
|
||||
actx.res_spad_base = vtcm_ptr;
|
||||
vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread;
|
||||
}
|
||||
|
||||
if (has_add && actx.is_row_bcast) {
|
||||
const uint8_t * r_ddr = (const uint8_t *) octx->src[2 * n_ranks]->data;
|
||||
const uint32_t row_bytes = actx.ne0 * (dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float));
|
||||
dma_queue * q = octx->ctx->dma[0];
|
||||
dma_queue_push(q, dma_make_ptr(actx.res_spad_base, r_ddr), actx.row_size_aligned, 0, row_bytes, 1);
|
||||
dma_queue_pop(q);
|
||||
}
|
||||
|
||||
work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads);
|
||||
}
|
||||
|
||||
// 4. Exit Barrier: Synchronize all ranks after writing
|
||||
htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
|
||||
|
||||
atomic_store(&my_fence[0], fence_seq_exit);
|
||||
asm volatile ("syncht" : : : "memory");
|
||||
Q6_dccleaninva_A((void *) my_fence);
|
||||
|
||||
for (uint32_t j = 0; j < n_ranks; j++) {
|
||||
if (j == rank) continue;
|
||||
const struct htp_tensor * peer_sync = octx->src[n_ranks + j];
|
||||
atomic_uint * peer_fence = (atomic_uint *) peer_sync->data;
|
||||
uint64_t spins = 0;
|
||||
while (1) {
|
||||
Q6_dccleaninva_A((void *) peer_fence);
|
||||
uint32_t val = atomic_load(&peer_fence[0]);
|
||||
if (val == fence_seq_exit) {
|
||||
break;
|
||||
}
|
||||
if (++spins > HTP_FENCE_TIMEOUT) {
|
||||
FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_exit);
|
||||
return HTP_STATUS_INTERNAL_ERR;
|
||||
}
|
||||
hex_pause();
|
||||
}
|
||||
}
|
||||
asm volatile ("syncht" : : : "memory");
|
||||
|
||||
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
|
||||
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#ifndef ALLREDUCE_OPS_H
|
||||
#define ALLREDUCE_OPS_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define HTP_ALLREDUCE_MAX_RANKS 4
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
enum htp_allreduce_kernel_type {
|
||||
HTP_ALLREDUCE_KERNEL_UNSUPPORTED = 0,
|
||||
HTP_ALLREDUCE_KERNEL_DMA_1D,
|
||||
HTP_ALLREDUCE_KERNEL_DMA_2D,
|
||||
};
|
||||
|
||||
struct htp_allreduce_kernel_params {
|
||||
int32_t rank;
|
||||
int32_t n_ranks;
|
||||
int32_t n_threads;
|
||||
int32_t block_elems; // 1D: block_elems, 2D: block_rows
|
||||
int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread
|
||||
int32_t vtcm_size_per_thread;
|
||||
int32_t vtcm_size;
|
||||
int32_t kernel_type;
|
||||
int32_t ne0;
|
||||
int32_t ne1;
|
||||
int32_t row_size_aligned;
|
||||
int32_t rank_elem_start;
|
||||
int32_t rank_nelem;
|
||||
int32_t n_dsts;
|
||||
int32_t is_row_bcast;
|
||||
};
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ALLREDUCE_OPS_H */
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#include <HAP_farf.h>
|
||||
#include <HAP_perf.h>
|
||||
#include <qurt_memory.h>
|
||||
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
@@ -14,6 +15,7 @@
|
||||
#include "htp-ops.h"
|
||||
#include "htp-ops.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "htp-tensor.h"
|
||||
|
||||
struct htp_copy_context {
|
||||
struct htp_ops_context * octx;
|
||||
@@ -78,7 +80,7 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
|
||||
} \
|
||||
}
|
||||
|
||||
DEFINE_CPY_SAMESHAPE(f32, float, 4)
|
||||
DEFINE_CPY_SAMESHAPE(f32, float, 4)
|
||||
DEFINE_CPY_SAMESHAPE(f16, __fp16, 2)
|
||||
|
||||
#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
|
||||
@@ -179,7 +181,7 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
|
||||
} \
|
||||
}
|
||||
|
||||
DEFINE_CPY_RESHAPE(f32, float, 4)
|
||||
DEFINE_CPY_RESHAPE(f32, float, 4)
|
||||
DEFINE_CPY_RESHAPE(f16, __fp16, 2)
|
||||
|
||||
static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, void * data) {
|
||||
@@ -232,6 +234,41 @@ static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, voi
|
||||
}
|
||||
}
|
||||
|
||||
static inline void cpy_dma_sametype_sameshape(
|
||||
struct htp_ops_context * octx,
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_tensor * src0,
|
||||
uint32_t elem_size,
|
||||
uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03,
|
||||
uint32_t nb01, uint32_t nb02, uint32_t nb03,
|
||||
uint32_t nb1, uint32_t nb2, uint32_t nb3
|
||||
) {
|
||||
const bool contiguous_outer =
|
||||
(ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) &&
|
||||
(ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2));
|
||||
|
||||
dma_queue * q = octx->ctx->dma[0];
|
||||
|
||||
if (contiguous_outer) {
|
||||
dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03);
|
||||
dma_queue_pop(q);
|
||||
return;
|
||||
}
|
||||
|
||||
for (uint32_t i03 = 0; i03 < ne03; i03++) {
|
||||
for (uint32_t i02 = 0; i02 < ne02; i02++) {
|
||||
uint8_t* dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
|
||||
uint8_t* src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
|
||||
if (!dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01)) {
|
||||
dma_queue_flush(q);
|
||||
dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dma_queue_flush(q);
|
||||
}
|
||||
|
||||
int op_cpy(struct htp_ops_context * octx) {
|
||||
cpy_preamble;
|
||||
|
||||
@@ -264,14 +301,11 @@ int op_cpy(struct htp_ops_context * octx) {
|
||||
|
||||
ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads;
|
||||
|
||||
worker_callback_t copy_fun;
|
||||
worker_callback_t copy_fun = NULL;
|
||||
bool use_dma = false;
|
||||
|
||||
if (sametype && sameshape) {
|
||||
if (src0->type == HTP_TYPE_F32) {
|
||||
copy_fun = cpy_thread_f32_sameshape;
|
||||
} else {
|
||||
copy_fun = cpy_thread_f16_sameshape;
|
||||
}
|
||||
use_dma = true;
|
||||
} else if (sameshape) {
|
||||
/**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32)
|
||||
copy_fun = cpy_thread_f16_f32_sameshape;
|
||||
@@ -289,7 +323,28 @@ int op_cpy(struct htp_ops_context * octx) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads);
|
||||
if (use_dma) {
|
||||
cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);
|
||||
} else {
|
||||
worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads);
|
||||
}
|
||||
|
||||
const struct htp_tensor *sync = octx->src[1];
|
||||
if (sync) {
|
||||
if (!use_dma) {
|
||||
// htp_tensor_flush_all(octx->ctx, octx->dsts, 1);
|
||||
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
|
||||
}
|
||||
|
||||
atomic_uint * sync_fence = (atomic_uint *) sync->data;
|
||||
const uint32_t seq = (uint32_t) octx->op_params[0];
|
||||
|
||||
atomic_store(&sync_fence[0], seq);
|
||||
asm volatile ("syncht" : : : "memory");
|
||||
Q6_dccleaninva_A((void *) sync_fence);
|
||||
|
||||
FARF(HIGH, "ggml-hex: sync-release : fence %p seq %u\n", sync_fence, seq);
|
||||
}
|
||||
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
@@ -244,17 +244,18 @@ static inline dma_ptr dma_queue_pop(dma_queue * q) {
|
||||
return dptr;
|
||||
}
|
||||
|
||||
dma_descriptor_2d * desc = &r->desc[r->pop_idx];
|
||||
dptr = r->dptr[r->pop_idx];
|
||||
|
||||
volatile dma_descriptor_2d * desc = &r->desc[r->pop_idx];
|
||||
|
||||
// Wait for desc to complete
|
||||
if (!desc->done) {
|
||||
// FARF(ALWAYS, "dma-poll: idx %u dst %p src %p", r->pop_idx, dptr.dst, dptr.src);
|
||||
while (!desc->done) {
|
||||
dmpoll();
|
||||
}
|
||||
}
|
||||
|
||||
dptr = r->dptr[r->pop_idx];
|
||||
|
||||
htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx);
|
||||
|
||||
r->pop_idx = (r->pop_idx + 1) & r->idx_mask;
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
#include "ggml-common.h"
|
||||
#include "htp-ctx.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-tensor.h"
|
||||
#include "hvx-quant.h"
|
||||
|
||||
#include "flash-attn-ops.h"
|
||||
#include "hvx-fa-kernels.h"
|
||||
@@ -85,12 +87,17 @@ struct htp_fa_context {
|
||||
uint8_t * spad_m;
|
||||
uint8_t * spad_a;
|
||||
|
||||
const struct htp_tensor * k;
|
||||
const struct htp_tensor * v;
|
||||
|
||||
uint64_t t_start;
|
||||
};
|
||||
|
||||
struct hmx_fa_context {
|
||||
const struct htp_ops_context * octx;
|
||||
const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent
|
||||
const struct htp_tensor * k;
|
||||
const struct htp_tensor * v;
|
||||
bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2
|
||||
uint32_t n_threads;
|
||||
|
||||
@@ -214,8 +221,8 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
|
||||
const uint32_t DV = nev0;
|
||||
|
||||
const size_t size_q_row = DK * ((q->type == HTP_TYPE_F32) ? 4 : 2);
|
||||
const size_t size_k_row = DK * sizeof(__fp16);
|
||||
const size_t size_v_row = DV * sizeof(__fp16);
|
||||
const size_t size_k_row = htp_tensor_get_row_size(k->type, DK);
|
||||
const size_t size_v_row = htp_tensor_get_row_size(v->type, DV);
|
||||
|
||||
// Scratchpad buffers for Q, K, V, Mask, and VKQ32 accumulator
|
||||
uint8_t * spad_q = factx->spad_q + factx->size_q_block * ith;
|
||||
@@ -364,6 +371,23 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
|
||||
uint8_t * v_base = dma_queue_pop(dma).dst; // V
|
||||
__fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M
|
||||
|
||||
if (factx->k->type == HTP_TYPE_Q8_0) {
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
|
||||
for (uint32_t r = 0; r < current_block_size; ++r) {
|
||||
__fp16 * row_k = (__fp16 *)(k_base + r * factx->size_k_row_padded);
|
||||
hvx_dequantize_row_q8_0_f16(row_k, row_k, DK);
|
||||
}
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
|
||||
}
|
||||
if (factx->v->type == HTP_TYPE_Q8_0) {
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
|
||||
for (uint32_t r = 0; r < current_block_size; ++r) {
|
||||
__fp16 * row_v = (__fp16 *)(v_base + r * factx->size_v_row_padded);
|
||||
hvx_dequantize_row_q8_0_f16(row_v, row_v, DV);
|
||||
}
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
|
||||
}
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, ir);
|
||||
|
||||
// Inner loop processing the block from VTCM
|
||||
@@ -625,6 +649,12 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)
|
||||
|
||||
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
|
||||
if (factx->k->type == HTP_TYPE_Q8_0) {
|
||||
for (uint32_t r = start; r < end; ++r) {
|
||||
__fp16 * row_k = (__fp16 *)((char *)args->curr_k + r * args->src_stride * sizeof(__fp16));
|
||||
hvx_dequantize_row_q8_0_f16(row_k, row_k, factx->DK);
|
||||
}
|
||||
}
|
||||
hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK,
|
||||
args->src_stride, start, end);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
|
||||
@@ -673,6 +703,12 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data)
|
||||
|
||||
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
|
||||
if (factx->v->type == HTP_TYPE_Q8_0) {
|
||||
for (uint32_t r = start; r < end; ++r) {
|
||||
__fp16 * row_v = (__fp16 *)((char *)args->v_src + r * args->src_stride * sizeof(__fp16));
|
||||
hvx_dequantize_row_q8_0_f16(row_v, row_v, factx->DV);
|
||||
}
|
||||
}
|
||||
hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV,
|
||||
args->src_stride, (uint32_t) args->n_col_tiles, start, end);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
|
||||
@@ -1809,6 +1845,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
|
||||
memset(&factx, 0, sizeof(factx));
|
||||
factx.octx = octx;
|
||||
factx.sinks = octx->src[4]; // NULL if this op has no attention sinks
|
||||
factx.k = k;
|
||||
factx.v = v;
|
||||
factx.n_threads = kparams->n_threads;
|
||||
factx.DK = DK;
|
||||
factx.DV = DV;
|
||||
@@ -1853,10 +1891,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
|
||||
// ======== VTCM allocation (GQA-aware) ========
|
||||
// K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used
|
||||
// throughout the KV loop below.
|
||||
const size_t size_k_row = DK * sizeof(__fp16);
|
||||
const size_t size_v_row = DV * sizeof(__fp16);
|
||||
const size_t size_k_row_padded = hex_round_up(size_k_row, 128);
|
||||
const size_t size_v_row_padded = hex_round_up(size_v_row, 128);
|
||||
const size_t size_k_row = htp_tensor_get_row_size(k->type, DK);
|
||||
const size_t size_v_row = htp_tensor_get_row_size(v->type, DV);
|
||||
const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128);
|
||||
const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128);
|
||||
|
||||
// Build the VTCM layout once (shared with the host estimator) and place every
|
||||
// scratch buffer at its computed offset.
|
||||
@@ -2348,7 +2386,9 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
|
||||
const struct htp_tensor * dst = octx->dst;
|
||||
|
||||
// Check support
|
||||
if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) || k->type != HTP_TYPE_F16 || v->type != HTP_TYPE_F16) {
|
||||
if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) ||
|
||||
(k->type != HTP_TYPE_F16 && k->type != HTP_TYPE_Q8_0) ||
|
||||
(v->type != HTP_TYPE_F16 && v->type != HTP_TYPE_Q8_0)) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
@@ -2364,6 +2404,8 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
|
||||
|
||||
struct htp_fa_context factx;
|
||||
factx.octx = octx;
|
||||
factx.k = k;
|
||||
factx.v = v;
|
||||
|
||||
factx.t_start = HAP_perf_get_qtimer_count();
|
||||
|
||||
|
||||
@@ -12,18 +12,17 @@
|
||||
#include "ggml-common.h"
|
||||
#include "htp-ctx.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-tensor.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "hvx-quant.h"
|
||||
#include "get-rows-ops.h"
|
||||
#include "work-queue.h"
|
||||
|
||||
struct get_rows_context {
|
||||
struct htp_ops_context * octx;
|
||||
uint32_t tasks_per_thread;
|
||||
uint32_t total_tasks;
|
||||
uint32_t chunks_per_row;
|
||||
uint32_t chunk_size;
|
||||
struct fastdiv_values get_rows_div_ne10;
|
||||
struct fastdiv_values get_rows_div_ne10_ne11;
|
||||
struct fastdiv_values get_rows_div_chunks_per_row;
|
||||
const struct htp_get_rows_kernel_params * kparams;
|
||||
struct htp_get_rows_vtcm_layout vtcm_layout;
|
||||
uint8_t * vtcm_base;
|
||||
};
|
||||
|
||||
#define get_rows_preamble \
|
||||
@@ -56,102 +55,161 @@ struct get_rows_context {
|
||||
\
|
||||
const uint32_t nr = ne10 * ne11 * ne12;
|
||||
|
||||
static void get_rows_thread_f32_f32_dma(unsigned int nth, unsigned int ith, void *data) {
|
||||
struct get_rows_context * grctx = (struct get_rows_context *)data;
|
||||
struct htp_ops_context * octx = grctx->octx;
|
||||
get_rows_preamble;
|
||||
|
||||
uint64_t qt = HAP_perf_get_qtimer_count();
|
||||
|
||||
const uint32_t dr = grctx->tasks_per_thread;
|
||||
const uint32_t ir0 = dr * ith;
|
||||
if (ir0 >= grctx->total_tasks) {
|
||||
return;
|
||||
}
|
||||
const uint32_t ir1 = MIN(ir0 + dr, grctx->total_tasks);
|
||||
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
dma_queue * dma_queue = octx->ctx->dma[ith];
|
||||
for (uint32_t i = ir0; i < ir1; ++i) {
|
||||
const uint32_t i12 = fastdiv(i, &grctx->get_rows_div_ne10_ne11);
|
||||
const uint32_t rem = i - i12 * ne11 * ne10;
|
||||
const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10);
|
||||
const uint32_t i10 = rem - i11 * ne10;
|
||||
|
||||
const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
|
||||
uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
|
||||
|
||||
if (i01 >= ne01) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03;
|
||||
const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
|
||||
|
||||
while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, ne00 * sizeof(float), 1)) {
|
||||
dma_queue_pop(dma_queue);
|
||||
}
|
||||
}
|
||||
dma_queue_flush(dma_queue);
|
||||
|
||||
qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
|
||||
FARF(HIGH, "get-rows-f32-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
|
||||
ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
|
||||
#define GET_ROWS_THREAD_ST_FN(IDX_TYPE) \
|
||||
static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
|
||||
struct get_rows_context * grctx = (struct get_rows_context *)data; \
|
||||
struct htp_ops_context * octx = grctx->octx; \
|
||||
const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \
|
||||
get_rows_preamble; \
|
||||
const uint32_t dr = kparams->tasks_per_thread; \
|
||||
const uint32_t ir0 = dr * ith; \
|
||||
if (ir0 >= kparams->total_tasks) { \
|
||||
return; \
|
||||
} \
|
||||
const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
|
||||
const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \
|
||||
dma_queue * dma_queue = octx->ctx->dma[ith]; \
|
||||
for (uint32_t i = ir0; i < ir1; ++i) { \
|
||||
const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); \
|
||||
const uint32_t rem = i - i12 * ne11 * ne10; \
|
||||
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
|
||||
const uint32_t i10 = rem - i11 * ne10; \
|
||||
const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \
|
||||
const uint32_t i01 = (uint32_t)*src1_ptr; \
|
||||
assert(i01 < ne01); \
|
||||
const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \
|
||||
const uint32_t i02 = i11 - q02 * ne02; \
|
||||
const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \
|
||||
const uint32_t i03 = i12 - q03 * ne03; \
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; \
|
||||
const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; \
|
||||
while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, \
|
||||
row_size_bytes, 1)) { \
|
||||
dma_queue_pop(dma_queue); \
|
||||
} \
|
||||
} \
|
||||
dma_queue_flush(dma_queue); \
|
||||
}
|
||||
|
||||
static void get_rows_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) {
|
||||
struct get_rows_context * grctx = (struct get_rows_context *)data;
|
||||
struct htp_ops_context * octx = grctx->octx;
|
||||
get_rows_preamble;
|
||||
GET_ROWS_THREAD_ST_FN(int32_t)
|
||||
GET_ROWS_THREAD_ST_FN(int64_t)
|
||||
|
||||
uint64_t qt = HAP_perf_get_qtimer_count();
|
||||
|
||||
const uint32_t dr = grctx->tasks_per_thread;
|
||||
const uint32_t ir0 = dr * ith;
|
||||
if (ir0 >= grctx->total_tasks) {
|
||||
return;
|
||||
}
|
||||
const uint32_t ir1 = MIN(ir0 + dr, grctx->total_tasks);
|
||||
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
const uint32_t chunks_per_row = grctx->chunks_per_row;
|
||||
const uint32_t chunk_size = grctx->chunk_size;
|
||||
for (uint32_t i = ir0; i < ir1; ++i) {
|
||||
const uint32_t row_idx = fastdiv(i, &grctx->get_rows_div_chunks_per_row);
|
||||
const uint32_t chunk_idx = i - row_idx * chunks_per_row;
|
||||
|
||||
const uint32_t i12 = fastdiv(row_idx, &grctx->get_rows_div_ne10_ne11);
|
||||
const uint32_t rem = row_idx - i12 * ne11 * ne10;
|
||||
const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10);
|
||||
const uint32_t i10 = rem - i11 * ne10;
|
||||
|
||||
const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
|
||||
uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
|
||||
|
||||
if (i01 >= ne01) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const uint32_t offset = chunk_idx * chunk_size;
|
||||
if (offset < ne00) {
|
||||
const uint32_t copy_size = MIN(chunk_size, ne00 - offset);
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03 + offset * sizeof(float);
|
||||
const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float);
|
||||
hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, copy_size);
|
||||
}
|
||||
}
|
||||
|
||||
qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
|
||||
FARF(HIGH, "get-rows-f32-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
|
||||
ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
|
||||
#define GET_ROWS_THREAD_DT_FN(TYPE_NAME, SRC0_SIZE_EXPR, IDX_TYPE, COMPUTE_EXPR) \
|
||||
static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
|
||||
struct get_rows_context * grctx = (struct get_rows_context *)data; \
|
||||
struct htp_ops_context * octx = grctx->octx; \
|
||||
const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \
|
||||
get_rows_preamble; \
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
|
||||
const uint32_t dr = kparams->tasks_per_thread; \
|
||||
const uint32_t ir0 = dr * ith; \
|
||||
if (ir0 >= kparams->total_tasks) { \
|
||||
return; \
|
||||
} \
|
||||
const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
|
||||
const uint32_t chunks_per_row = kparams->chunks_per_row; \
|
||||
const uint32_t chunk_size = kparams->chunk_size; \
|
||||
dma_queue * dma_queue = octx->ctx->dma[ith]; \
|
||||
const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; \
|
||||
uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \
|
||||
uint8_t * vtcm_dst = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \
|
||||
for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { \
|
||||
const uint32_t i = ir0 + step; \
|
||||
const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \
|
||||
const uint32_t chunk_idx = i - row_idx * chunks_per_row; \
|
||||
const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \
|
||||
const uint32_t rem = row_idx - i12 * ne11 * ne10; \
|
||||
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
|
||||
const uint32_t i10 = rem - i11 * ne10; \
|
||||
const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \
|
||||
const uint32_t i01 = (uint32_t)*src1_ptr; \
|
||||
assert(i01 < ne01); \
|
||||
const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \
|
||||
const uint32_t i02 = i11 - q02 * ne02; \
|
||||
const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \
|
||||
const uint32_t i03 = i12 - q03 * ne03; \
|
||||
const uint32_t offset = chunk_idx * chunk_size; \
|
||||
const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \
|
||||
const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems); \
|
||||
const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)(uintptr_t)octx->dst->data, \
|
||||
vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \
|
||||
cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \
|
||||
(const void *)src0_ptr), \
|
||||
vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1); \
|
||||
} \
|
||||
for (uint32_t step = 0; step < ir1 - ir0; ++step) { \
|
||||
const uint32_t i = ir0 + step; \
|
||||
void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \
|
||||
void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \
|
||||
const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \
|
||||
const uint32_t chunk_idx = i - row_idx * chunks_per_row; \
|
||||
const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \
|
||||
const uint32_t rem = row_idx - i12 * ne11 * ne10; \
|
||||
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
|
||||
const uint32_t i10 = rem - i11 * ne10; \
|
||||
const uint32_t offset = chunk_idx * chunk_size; \
|
||||
const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \
|
||||
const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i); \
|
||||
COMPUTE_EXPR; \
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i); \
|
||||
const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \
|
||||
cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1); \
|
||||
const uint32_t next_step = step + 2; \
|
||||
if (next_step < ir1 - ir0) { \
|
||||
const uint32_t pi = ir0 + next_step; \
|
||||
const uint32_t prow_idx = fastdiv(pi, &kparams->div_chunks_per_row); \
|
||||
const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row; \
|
||||
const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11); \
|
||||
const uint32_t prem = prow_idx - pi12 * ne11 * ne10; \
|
||||
const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10); \
|
||||
const uint32_t pi10 = prem - pi11 * ne10; \
|
||||
const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \
|
||||
const uint32_t pi01 = (uint32_t)*psrc1_ptr; \
|
||||
assert(pi01 < ne01); \
|
||||
const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02); \
|
||||
const uint32_t pi02 = pi11 - pq02 * ne02; \
|
||||
const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03); \
|
||||
const uint32_t pi03 = pi12 - pq03 * ne03; \
|
||||
const uint32_t poffset = pchunk_idx * chunk_size; \
|
||||
const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0; \
|
||||
const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems); \
|
||||
const uintptr_t psrc0_ptr = \
|
||||
octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \
|
||||
vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1); \
|
||||
} \
|
||||
} \
|
||||
dma_queue_flush(dma_queue); \
|
||||
}
|
||||
|
||||
#define F32_BYTES(n) ((n) * sizeof(float))
|
||||
#define F16_BYTES(n) ((n) * sizeof(__fp16))
|
||||
#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0))
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int32_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
|
||||
GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int64_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
|
||||
GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
|
||||
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
|
||||
|
||||
int op_get_rows(struct htp_ops_context * octx) {
|
||||
get_rows_preamble;
|
||||
const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;
|
||||
|
||||
if (octx->src[0]->type != HTP_TYPE_F32) {
|
||||
if (octx->src[0]->type != HTP_TYPE_F32 &&
|
||||
octx->src[0]->type != HTP_TYPE_F16 &&
|
||||
octx->src[0]->type != HTP_TYPE_Q8_0) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
@@ -167,52 +225,28 @@ int op_get_rows(struct htp_ops_context * octx) {
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
const uint32_t nb00 = octx->src[0]->nb[0];
|
||||
const uint32_t nb0 = octx->dst->nb[0];
|
||||
|
||||
const bool can_use_dma = (nb00 == sizeof(float)) && (nb0 == sizeof(float));
|
||||
const bool use_dma = can_use_dma && (ne00 >= 2048);
|
||||
|
||||
struct get_rows_context grctx;
|
||||
grctx.octx = octx;
|
||||
grctx.get_rows_div_ne10 = init_fastdiv_values(octx->src[1]->ne[0]);
|
||||
grctx.get_rows_div_ne10_ne11 = init_fastdiv_values(octx->src[1]->ne[0] * octx->src[1]->ne[1]);
|
||||
grctx.kparams = kparams;
|
||||
grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
|
||||
|
||||
if (use_dma) {
|
||||
grctx.chunks_per_row = 1;
|
||||
grctx.chunk_size = ne00;
|
||||
grctx.total_tasks = nr;
|
||||
grctx.get_rows_div_chunks_per_row = init_fastdiv_values(1);
|
||||
const uint32_t ne00 = octx->src[0]->ne[0];
|
||||
htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, kparams->n_threads);
|
||||
|
||||
const uint32_t n_threads = MIN(nr, octx->n_threads);
|
||||
grctx.tasks_per_thread = (nr + n_threads - 1) / n_threads;
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32_dma, &grctx, n_threads);
|
||||
work_queue_func_t q_func = NULL;
|
||||
if (kparams->use_dma) {
|
||||
q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);
|
||||
} else {
|
||||
uint32_t chunks_per_row = 1;
|
||||
uint32_t chunk_size = ne00;
|
||||
uint32_t total_tasks = nr;
|
||||
|
||||
if (nr < octx->n_threads) {
|
||||
const uint32_t min_chunk_size = 1024;
|
||||
uint32_t max_chunks = ne00 / min_chunk_size;
|
||||
if (max_chunks == 0) {
|
||||
max_chunks = 1;
|
||||
}
|
||||
chunks_per_row = MIN((octx->n_threads + nr - 1) / nr, max_chunks);
|
||||
chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row;
|
||||
total_tasks = nr * chunks_per_row;
|
||||
switch (octx->src[0]->type) {
|
||||
case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break;
|
||||
case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;
|
||||
case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;
|
||||
default: return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
grctx.chunks_per_row = chunks_per_row;
|
||||
grctx.chunk_size = chunk_size;
|
||||
grctx.total_tasks = total_tasks;
|
||||
grctx.get_rows_div_chunks_per_row = init_fastdiv_values(chunks_per_row);
|
||||
|
||||
const uint32_t n_threads = MIN(total_tasks, octx->n_threads);
|
||||
grctx.tasks_per_thread = (total_tasks + n_threads - 1) / n_threads;
|
||||
|
||||
worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32_hvx, &grctx, n_threads);
|
||||
}
|
||||
|
||||
work_queue_run(octx->ctx->work_queue, q_func, &grctx, kparams->n_threads);
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
#ifndef HTP_GET_ROWS_OPS_H
|
||||
#define HTP_GET_ROWS_OPS_H
|
||||
|
||||
#include "hex-fastdiv.h"
|
||||
|
||||
struct htp_get_rows_kernel_params {
|
||||
int32_t n_threads;
|
||||
int32_t use_dma;
|
||||
int32_t chunks_per_row;
|
||||
int32_t chunk_size;
|
||||
int32_t total_tasks;
|
||||
int32_t tasks_per_thread;
|
||||
int32_t vtcm_size;
|
||||
|
||||
// Fastdiv helpers
|
||||
struct fastdiv_values div_ne10;
|
||||
struct fastdiv_values div_ne10_ne11;
|
||||
struct fastdiv_values div_chunks_per_row;
|
||||
struct fastdiv_values div_ne02;
|
||||
struct fastdiv_values div_ne03;
|
||||
};
|
||||
|
||||
struct htp_get_rows_vtcm_layout {
|
||||
size_t total_bytes;
|
||||
size_t off_src0;
|
||||
size_t off_dst;
|
||||
|
||||
size_t src0_bytes_per_thread;
|
||||
size_t dst_bytes_per_thread;
|
||||
|
||||
size_t src0_spad_half_size;
|
||||
size_t dst_spad_half_size;
|
||||
};
|
||||
|
||||
static inline void htp_get_rows_vtcm_layout_build(
|
||||
struct htp_get_rows_vtcm_layout * vtcm_layout,
|
||||
int type,
|
||||
uint32_t ne00,
|
||||
uint32_t n_threads) {
|
||||
|
||||
uint32_t src0_row_size = 0;
|
||||
switch (type) {
|
||||
case 0: // HTP_TYPE_F32
|
||||
src0_row_size = ne00 * 4;
|
||||
break;
|
||||
case 1: // HTP_TYPE_F16
|
||||
src0_row_size = ne00 * 2;
|
||||
break;
|
||||
case 8: // HTP_TYPE_Q8_0
|
||||
src0_row_size = (ne00 / 32) * 34;
|
||||
break;
|
||||
default:
|
||||
src0_row_size = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;
|
||||
size_t dst_row_size_aligned = (ne00 * sizeof(float) + 255) & ~255;
|
||||
|
||||
vtcm_layout->src0_spad_half_size = src0_row_size_aligned;
|
||||
vtcm_layout->dst_spad_half_size = dst_row_size_aligned;
|
||||
|
||||
vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
|
||||
vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2;
|
||||
|
||||
vtcm_layout->off_src0 = 0;
|
||||
vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;
|
||||
vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
|
||||
}
|
||||
|
||||
#if defined(__cplusplus)
|
||||
static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob");
|
||||
#else
|
||||
_Static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob");
|
||||
#endif
|
||||
|
||||
#endif // HTP_GET_ROWS_OPS_H
|
||||
@@ -39,17 +39,22 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) {
|
||||
|
||||
#define HEX_L2_LINE_SIZE 128
|
||||
#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration)
|
||||
#define HEX_L2_FLUSH_IL_THRESHOLD 1024 // inline flush threshold
|
||||
#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024)
|
||||
#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)
|
||||
|
||||
static inline void hex_l2flush(void * addr, size_t size) {
|
||||
const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1);
|
||||
const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1);
|
||||
for (uint32_t i = s; i < e; i += HEX_L2_BLOCK_SIZE) {
|
||||
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0);
|
||||
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1);
|
||||
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2);
|
||||
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3);
|
||||
const uint32_t eb = s + ((e - s) & ~(HEX_L2_BLOCK_SIZE - 1));
|
||||
for (uint32_t i = s; i < eb; i += HEX_L2_BLOCK_SIZE) {
|
||||
Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 0));
|
||||
Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 1));
|
||||
Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 2));
|
||||
Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 3));
|
||||
}
|
||||
for (uint32_t i = eb; i < e; i += HEX_L2_LINE_SIZE) {
|
||||
Q6_dccleaninva_A((void *) i);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -117,8 +117,7 @@ struct htp_context {
|
||||
|
||||
int op_matmul(struct htp_ops_context * octx);
|
||||
int op_matmul_id(struct htp_ops_context * octx);
|
||||
int op_matmul_qkv(struct htp_ops_context * octx);
|
||||
int op_matmul_ffn(struct htp_ops_context * octx);
|
||||
int op_matmul_nx(struct htp_ops_context * octx);
|
||||
int op_binary(struct htp_ops_context * octx);
|
||||
int op_unary(struct htp_ops_context * octx);
|
||||
int op_sum_rows(struct htp_ops_context * octx);
|
||||
@@ -141,5 +140,6 @@ int op_solve_tri(struct htp_ops_context * octx);
|
||||
int op_gated_delta_net(struct htp_ops_context * octx);
|
||||
int op_pad(struct htp_ops_context * octx);
|
||||
int op_im2col(struct htp_ops_context * octx);
|
||||
int op_allreduce(struct htp_ops_context * octx);
|
||||
|
||||
#endif /* HTP_CTX_H */
|
||||
|
||||
@@ -43,13 +43,6 @@ enum htp_data_type {
|
||||
|
||||
|
||||
|
||||
// Mask to enable various stages of the Ops.
|
||||
// Used for debugging and profiling.
|
||||
enum htp_op_stage {
|
||||
HTP_OPSTAGE_QUEUE = (1 << 0), // Enable Queueing (ie calls into NPU)
|
||||
HTP_OPSTAGE_COMPUTE = (1 << 1), // Enable Compute
|
||||
};
|
||||
|
||||
// Do not reorder first 4 (used as an index)
|
||||
enum htp_op_code {
|
||||
HTP_OP_MUL = 0,
|
||||
@@ -58,8 +51,7 @@ enum htp_op_code {
|
||||
HTP_OP_DIV = 3,
|
||||
HTP_OP_MUL_MAT,
|
||||
HTP_OP_MUL_MAT_ID,
|
||||
HTP_OP_MUL_MAT_QKV,
|
||||
HTP_OP_MUL_MAT_FFN,
|
||||
HTP_OP_MUL_MAT_NX,
|
||||
HTP_OP_MUL_MAT_ADD,
|
||||
HTP_OP_RMS_NORM,
|
||||
HTP_OP_RMS_NORM_MUL,
|
||||
@@ -99,12 +91,15 @@ enum htp_op_code {
|
||||
HTP_OP_CONCAT,
|
||||
HTP_OP_CLAMP,
|
||||
HTP_OP_IM2COL,
|
||||
HTP_OP_FENCE,
|
||||
HTP_OP_ALLREDUCE,
|
||||
HTP_OP_ALLREDUCE_ADD,
|
||||
|
||||
HTP_OP_INVALID
|
||||
};
|
||||
|
||||
#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS
|
||||
#define HTP_OP_MAX_INPUTS 6 // aka GGML_MAX_SRCS
|
||||
#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS
|
||||
#define HTP_OP_MAX_OUTPUTS 4
|
||||
#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS
|
||||
#define HTP_OP_MAX_KERN_PARAMS 32
|
||||
@@ -112,13 +107,16 @@ enum htp_op_code {
|
||||
#define HTP_OP_MAX_BUFS 16
|
||||
#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16)
|
||||
|
||||
#define HTP_FENCE_TIMEOUT (1000000000ULL)
|
||||
|
||||
#define HTP_OP_MAX_VMEM_DEFAULT (3355443200u)
|
||||
|
||||
#define HTP_MMAP_MAX_VMEM (2147483648u)
|
||||
|
||||
enum htp_tensor_flags {
|
||||
HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights)
|
||||
HTP_TENSOR_DIRTY = (1U << 1) // Tensor buffer is dirty and needs to be flushed
|
||||
HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute)
|
||||
HTP_TENSOR_REPACK = (1U << 1), // Tensor is in repacked tiled format
|
||||
HTP_TENSOR_FENCE = (1U << 2) // Tensor is synchronization fence (explicitly managed)
|
||||
};
|
||||
|
||||
// Tensor descriptor
|
||||
@@ -175,6 +173,7 @@ enum htp_trace_event_id {
|
||||
HTP_TRACE_EVT_L2FLUSH = 1,
|
||||
HTP_TRACE_EVT_INIT = 2,
|
||||
HTP_TRACE_EVT_BUFF = 3,
|
||||
HTP_TRACE_EVT_FENCE = 4,
|
||||
|
||||
HTP_TRACE_EVT_HVX_COMP = 20,
|
||||
HTP_TRACE_EVT_HVX_A_QUANT = 21,
|
||||
@@ -215,6 +214,7 @@ struct htp_opbatch_req {
|
||||
uint32_t n_ops; // Number of ops
|
||||
uint32_t n_traces; // Number of trace descriptors per thread
|
||||
uint32_t pad; // unused
|
||||
uint64_t seq; // Sequence number
|
||||
// struct htp_buf_desc bufs[]; -- dspqueue buf 0
|
||||
// struct htp_tensor tensors[]; -- dspqueue buf 0
|
||||
// struct htp_op_desc ops[]; -- dspqueue buf 0
|
||||
@@ -231,6 +231,7 @@ struct htp_opbatch_rsp {
|
||||
uint32_t pad; // align to 8 bytes
|
||||
uint64_t cycles_start; // Start cycle counter
|
||||
uint64_t cycles_stop; // Stop cycle counter
|
||||
uint64_t seq; // Sequence number
|
||||
// struct htp_prof_desc profs[]; -- dspqueue buf 0
|
||||
};
|
||||
|
||||
|
||||
@@ -79,7 +79,14 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co
|
||||
|
||||
for (uint32_t i = 0; i < n; i++) {
|
||||
const struct htp_tensor * t = tensors[i];
|
||||
if (!t) continue;
|
||||
if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (t->size <= HEX_L2_FLUSH_IL_THRESHOLD) {
|
||||
hex_l2flush((void *) (uintptr_t) t->data, t->size);
|
||||
continue;
|
||||
}
|
||||
|
||||
uint32_t t_start = t->data;
|
||||
uint32_t t_end = t_start + t->size;
|
||||
@@ -242,7 +249,7 @@ void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * co
|
||||
|
||||
for (uint32_t i = 0; i < n; i++) {
|
||||
const struct htp_tensor * t = tensors[i];
|
||||
if (t && (t->flags & HTP_TENSOR_COMPUTE) && is_tensor_dirty(ctx, t)) {
|
||||
if (t && !(t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE)) && is_tensor_dirty(ctx, t)) {
|
||||
dirty_tensors[n_dirty++] = t;
|
||||
total_dirty += t->size;
|
||||
}
|
||||
|
||||
@@ -13,6 +13,15 @@ static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) {
|
||||
return (uint32_t *) &t->flags;
|
||||
}
|
||||
|
||||
static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
|
||||
switch (type) {
|
||||
case HTP_TYPE_F32: return ne00 * 4;
|
||||
case HTP_TYPE_F16: return ne00 * 2;
|
||||
case HTP_TYPE_Q8_0: return (ne00 / 32) * 34;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
struct htp_context;
|
||||
void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
|
||||
void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
|
||||
|
||||
@@ -17,9 +17,9 @@
|
||||
|
||||
#define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \
|
||||
do { \
|
||||
dst_type * restrict vdst = (dst_type *) dst; \
|
||||
src0_type * restrict vsrc0 = (src0_type *) src0; \
|
||||
src1_type * restrict vsrc1 = (src1_type *) src1; \
|
||||
dst_type * vdst = (dst_type *) dst; \
|
||||
src0_type * vsrc0 = (src0_type *) src0; \
|
||||
src1_type * vsrc1 = (src1_type *) src1; \
|
||||
\
|
||||
const uint32_t epv = 128 / (elem_size); \
|
||||
const uint32_t nvec = n / epv; \
|
||||
@@ -57,40 +57,40 @@
|
||||
|
||||
// Generic macro to define alignment permutations for an op
|
||||
#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \
|
||||
static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
|
||||
static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
|
||||
assert((uintptr_t) dst % 128 == 0); \
|
||||
assert((uintptr_t) src0 % 128 == 0); \
|
||||
assert((uintptr_t) src1 % 128 == 0); \
|
||||
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
|
||||
} \
|
||||
static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
|
||||
static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
|
||||
assert((uintptr_t) dst % 128 == 0); \
|
||||
assert((uintptr_t) src0 % 128 == 0); \
|
||||
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
|
||||
} \
|
||||
static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
|
||||
static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
|
||||
assert((uintptr_t) dst % 128 == 0); \
|
||||
assert((uintptr_t) src1 % 128 == 0); \
|
||||
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
|
||||
} \
|
||||
static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
|
||||
static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
|
||||
assert((uintptr_t) dst % 128 == 0); \
|
||||
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
|
||||
} \
|
||||
static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
|
||||
static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
|
||||
assert((uintptr_t) src0 % 128 == 0); \
|
||||
assert((uintptr_t) src1 % 128 == 0); \
|
||||
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
|
||||
} \
|
||||
static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
|
||||
static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
|
||||
assert((uintptr_t) src0 % 128 == 0); \
|
||||
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
|
||||
} \
|
||||
static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
|
||||
static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
|
||||
assert((uintptr_t) src1 % 128 == 0); \
|
||||
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
|
||||
} \
|
||||
static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
|
||||
static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
|
||||
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
|
||||
} \
|
||||
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
#ifndef HVX_QUANT_H
|
||||
#define HVX_QUANT_H
|
||||
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "hvx-arith.h"
|
||||
#include "hvx-base.h"
|
||||
#include "hvx-reduce.h"
|
||||
#include "hvx-repl.h"
|
||||
#include "hvx-utils.h"
|
||||
|
||||
#ifndef GGML_COMMON_DECL_C
|
||||
#define GGML_COMMON_DECL_C
|
||||
#endif
|
||||
#include "ggml-common.h"
|
||||
#include "ggml-impl.h"
|
||||
|
||||
static inline void hvx_quantize_row_q8_0_f32(void * restrict dst_ptr, const float * restrict src_ptr, int n) {
|
||||
const int nb = n / QK8_0;
|
||||
block_q8_0 * dst = (block_q8_0 *) dst_ptr;
|
||||
HVX_Vector zero = Q6_V_vzero();
|
||||
|
||||
int i = 0;
|
||||
for (; i + 3 < nb; i += 4) {
|
||||
HVX_Vector * vx = (HVX_Vector *) (src_ptr + i * QK8_0);
|
||||
|
||||
HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));
|
||||
HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));
|
||||
HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));
|
||||
HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));
|
||||
|
||||
HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);
|
||||
HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);
|
||||
HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);
|
||||
HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);
|
||||
|
||||
HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero);
|
||||
HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero);
|
||||
HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);
|
||||
HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);
|
||||
|
||||
HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));
|
||||
HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));
|
||||
|
||||
HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));
|
||||
HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));
|
||||
|
||||
HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
|
||||
HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
|
||||
HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);
|
||||
HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);
|
||||
|
||||
HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);
|
||||
HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);
|
||||
vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));
|
||||
vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));
|
||||
|
||||
HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);
|
||||
HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);
|
||||
HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16);
|
||||
|
||||
hvx_vec_store_u(&dst[i + 0].d, 2, vd01_hf);
|
||||
hvx_vec_store_u(dst[i + 0].qs, 32, vx_i8);
|
||||
|
||||
hvx_vec_store_u(&dst[i + 1].d, 2, Q6_V_vror_VR(vd01_hf, 64));
|
||||
hvx_vec_store_u(dst[i + 1].qs, 32, Q6_V_vror_VR(vx_i8, 32));
|
||||
|
||||
hvx_vec_store_u(&dst[i + 2].d, 2, vd23_hf);
|
||||
hvx_vec_store_u(dst[i + 2].qs, 32, Q6_V_vror_VR(vx_i8, 64));
|
||||
|
||||
hvx_vec_store_u(&dst[i + 3].d, 2, Q6_V_vror_VR(vd23_hf, 64));
|
||||
hvx_vec_store_u(dst[i + 3].qs, 32, Q6_V_vror_VR(vx_i8, 96));
|
||||
}
|
||||
|
||||
for (; i < nb; i++) {
|
||||
const float * block_src = src_ptr + i * QK8_0;
|
||||
HVX_Vector vx = *(const HVX_UVector *) block_src;
|
||||
HVX_Vector v_abs = hvx_vec_abs_f32(vx);
|
||||
HVX_Vector v_max = hvx_vec_reduce_max_f32(v_abs);
|
||||
float amax = hvx_vec_get_f32(v_max);
|
||||
|
||||
const float d = amax / 127.0f;
|
||||
const float id = d ? (1.0f / d) : 0.0f;
|
||||
dst[i].d = GGML_FP32_TO_FP16(d);
|
||||
|
||||
HVX_Vector vid = hvx_vec_splat_f32(id);
|
||||
HVX_Vector v_scaled = hvx_vec_mul_f32_f32(vx, vid);
|
||||
HVX_Vector v_scaled_qf = Q6_Vqf32_vsub_VsfVsf(v_scaled, zero);
|
||||
HVX_Vector v_scaled_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(zero, v_scaled_qf)));
|
||||
HVX_Vector v_i16 = hvx_vec_i16_from_hf_rnd_sat(v_scaled_hf);
|
||||
HVX_Vector v_i8 = Q6_Vb_vpack_VhVh_sat(zero, v_i16);
|
||||
|
||||
hvx_vec_store_u(dst[i].qs, 32, v_i8);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void hvx_dequantize_row_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) {
|
||||
const int nb = n / QK8_0;
|
||||
const block_q8_0 * src = (const block_q8_0 *) src_ptr;
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
|
||||
HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
|
||||
HVX_Vector vd = Q6_V_lo_W(vp_f32);
|
||||
|
||||
HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs;
|
||||
|
||||
HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8);
|
||||
HVX_Vector v_i16 = Q6_V_lo_W(p16);
|
||||
HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16);
|
||||
HVX_Vector v_i32 = Q6_V_lo_W(p32);
|
||||
|
||||
HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
|
||||
HVX_Vector res = hvx_vec_mul_f32_f32(v_f32, vd);
|
||||
|
||||
float * block_dst = dst_ptr + i * QK8_0;
|
||||
hvx_vmem(block_dst) = res;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void hvx_dequantize_row_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, int n) {
|
||||
const int nb = n / QK8_0;
|
||||
const block_q8_0 * src = (const block_q8_0 *) src_ptr;
|
||||
|
||||
for (int i = nb - 1; i >= 0; i--) {
|
||||
HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
|
||||
HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
|
||||
HVX_Vector vd = Q6_V_lo_W(vp_f32);
|
||||
|
||||
HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs;
|
||||
|
||||
HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8);
|
||||
HVX_Vector v_i16 = Q6_V_lo_W(p16);
|
||||
HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16);
|
||||
HVX_Vector v_i32 = Q6_V_lo_W(p32);
|
||||
|
||||
HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
|
||||
HVX_Vector res_f32 = hvx_vec_mul_f32_f32(v_f32, vd);
|
||||
|
||||
HVX_Vector res_f16 = hvx_vec_f32_to_f16(res_f32, Q6_V_vzero());
|
||||
|
||||
__fp16 * block_dst = dst_ptr + i * QK8_0;
|
||||
hvx_vec_store_u(block_dst, QK8_0 * sizeof(__fp16), res_f16);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void hvx_dequantize_row_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) {
|
||||
const int nb = n / 32;
|
||||
const _Float16 * src = (const _Float16 *) src_ptr;
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * 32);
|
||||
HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16);
|
||||
HVX_Vector res = Q6_V_lo_W(vp_f32);
|
||||
|
||||
float * block_dst = dst_ptr + i * 32;
|
||||
hvx_vmem(block_dst) = res;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
#endif // HVX_QUANT_H
|
||||
@@ -18,6 +18,7 @@
|
||||
#include <qurt_memory.h>
|
||||
#include <remote.h>
|
||||
#include <string.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
#include "hex-utils.h"
|
||||
#include "hex-dma.h"
|
||||
@@ -32,6 +33,7 @@
|
||||
#include "htp_iface.h"
|
||||
#include "work-queue.h"
|
||||
#include "hex-profile.h"
|
||||
#include "allreduce-ops.h"
|
||||
|
||||
#define HMX_QUEUE_CAPACITY 16
|
||||
#define HMX_QUEUE_STACK_SIZE 16384
|
||||
@@ -46,6 +48,36 @@ struct htp_handle {
|
||||
struct htp_context * ctx;
|
||||
};
|
||||
|
||||
static inline void * htp_mmap(uint32_t fd, uint32_t size) {
|
||||
void * va = (void *)-1;
|
||||
for (int retry = 0; retry < 2; retry++) {
|
||||
#if __HVX_ARCH__ > 73
|
||||
va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
|
||||
#else
|
||||
if (size > HTP_MMAP_MAX_VMEM) {
|
||||
FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size);
|
||||
abort();
|
||||
}
|
||||
va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
|
||||
#endif
|
||||
if (va != (void *)-1 && va != NULL) {
|
||||
return va;
|
||||
}
|
||||
if (retry == 0) {
|
||||
FARF(HIGH, "mmap failed first try (va %p fd %u size %u), retrying...", va, fd, size);
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline void htp_munmap(void * va, uint32_t size) {
|
||||
#if __HVX_ARCH__ > 73
|
||||
HAP_munmap2(va, size);
|
||||
#else
|
||||
HAP_munmap(va, size);
|
||||
#endif
|
||||
}
|
||||
|
||||
AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) {
|
||||
(void) uri;
|
||||
struct htp_handle * h = calloc(1, sizeof(*h));
|
||||
@@ -127,11 +159,7 @@ AEEResult htp_iface_close(remote_handle64 handle) {
|
||||
// release the mmaps (if any)
|
||||
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
|
||||
if (ctx->mmap[i].size) {
|
||||
#if __HVX_ARCH__ > 73
|
||||
HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size);
|
||||
#else
|
||||
HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);
|
||||
#endif
|
||||
htp_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);
|
||||
ctx->mmap[i].size = 0;
|
||||
ctx->mmap[i].base = NULL;
|
||||
ctx->mmap[i].fd = -1;
|
||||
@@ -175,18 +203,9 @@ AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) {
|
||||
struct htp_mmap *m = &ctx->mmap[i];
|
||||
if (!m->size) {
|
||||
FARF(HIGH, "mmap : fd %u size %u", fd, size);
|
||||
#if __HVX_ARCH__ > 73
|
||||
void *va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
|
||||
#else
|
||||
if (size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB
|
||||
FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size);
|
||||
abort(); // can't do much else at this point
|
||||
}
|
||||
|
||||
void *va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0);
|
||||
#endif
|
||||
if (va == (void*)-1) {
|
||||
FARF(ERROR, "mmap failed : va %p fd %u size %u", va, fd, (uint32_t) size);
|
||||
void *va = htp_mmap(fd, size);
|
||||
if (va == NULL) {
|
||||
FARF(ERROR, "mmap failed : fd %u size %u", fd, (uint32_t) size);
|
||||
return AEE_EFAILED;
|
||||
}
|
||||
|
||||
@@ -212,11 +231,7 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
|
||||
struct htp_mmap *m = &ctx->mmap[i];
|
||||
if (fd < 0 || m->fd == fd) {
|
||||
FARF(HIGH, "unmmap : base %p fd %u size %u", (void*) m->base, m->fd, (uint32_t) m->size);
|
||||
#if __HVX_ARCH__ > 73
|
||||
HAP_munmap2((void *) m->base, m->size);
|
||||
#else
|
||||
HAP_munmap((void *) m->base, m->size);
|
||||
#endif
|
||||
htp_munmap((void *) m->base, m->size);
|
||||
m->size = 0;
|
||||
m->base = NULL;
|
||||
m->fd = -1;
|
||||
@@ -228,7 +243,7 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
|
||||
|
||||
static void vtcm_acquire(struct htp_context * ctx) {
|
||||
if (!ctx->vtcm_valid) {
|
||||
int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 1000000u);
|
||||
int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u);
|
||||
if (err != 0) {
|
||||
FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err);
|
||||
abort();
|
||||
@@ -692,8 +707,45 @@ static inline void profile_stop(uint32_t mode, struct profile_data * d) {
|
||||
}
|
||||
}
|
||||
|
||||
static int op_fence(struct htp_ops_context * octx) {
|
||||
struct htp_context *ctx = octx->ctx;
|
||||
struct htp_thread_trace * tr = &ctx->trace[0];
|
||||
const uint32_t seq = (uint32_t) octx->op_params[0];
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
|
||||
|
||||
const struct htp_tensor * sync = octx->src[0];
|
||||
atomic_uint * sync_fence = (atomic_uint *) sync->data;
|
||||
uint64_t spins = 0;
|
||||
while (1) {
|
||||
Q6_dccleaninva_A((void *) sync_fence);
|
||||
asm volatile ("syncht" : : : "memory");
|
||||
uint32_t val = atomic_load(&sync_fence[0]);
|
||||
if ((int32_t)(val - seq) >= 0) {
|
||||
break;
|
||||
}
|
||||
if (++spins > HTP_FENCE_TIMEOUT) {
|
||||
FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq %u\n", sync_fence, spins, seq);
|
||||
break;
|
||||
}
|
||||
hex_pause();
|
||||
}
|
||||
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
|
||||
|
||||
FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq %u\n", sync_fence, spins, seq);
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
static int execute_op(struct htp_ops_context * octx) {
|
||||
switch (octx->op) {
|
||||
case HTP_OP_FENCE:
|
||||
return op_fence(octx);
|
||||
|
||||
case HTP_OP_ALLREDUCE:
|
||||
case HTP_OP_ALLREDUCE_ADD:
|
||||
return op_allreduce(octx);
|
||||
|
||||
case HTP_OP_MUL_MAT:
|
||||
case HTP_OP_MUL_MAT_ADD:
|
||||
return op_matmul(octx);
|
||||
@@ -701,11 +753,8 @@ static int execute_op(struct htp_ops_context * octx) {
|
||||
case HTP_OP_MUL_MAT_ID:
|
||||
return op_matmul_id(octx);
|
||||
|
||||
case HTP_OP_MUL_MAT_QKV:
|
||||
return op_matmul_qkv(octx);
|
||||
|
||||
case HTP_OP_MUL_MAT_FFN:
|
||||
return op_matmul_ffn(octx);
|
||||
case HTP_OP_MUL_MAT_NX:
|
||||
return op_matmul_nx(octx);
|
||||
|
||||
case HTP_OP_MUL:
|
||||
case HTP_OP_ADD:
|
||||
@@ -818,12 +867,8 @@ static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct
|
||||
|
||||
static inline void drop_mmap(struct htp_context *ctx, struct htp_mmap *m) {
|
||||
if (m->size) {
|
||||
FARF(HIGH, "unmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
|
||||
#if __HVX_ARCH__ > 73
|
||||
HAP_munmap2((void *) m->base, m->size);
|
||||
#else
|
||||
HAP_munmap((void *) m->base, m->size);
|
||||
#endif
|
||||
FARF(ALWAYS, "unmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
|
||||
htp_munmap((void *) m->base, m->size);
|
||||
m->size = 0;
|
||||
m->base = 0;
|
||||
m->fd = -1;
|
||||
@@ -837,18 +882,9 @@ static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) {
|
||||
for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) {
|
||||
struct htp_mmap *m = &ctx->mmap[i];
|
||||
if (!m->size) {
|
||||
#if __HVX_ARCH__ > 73
|
||||
void *va = HAP_mmap2(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0);
|
||||
#else
|
||||
if (b->size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB
|
||||
FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) b->size);
|
||||
abort(); // can't do much else at this point
|
||||
}
|
||||
|
||||
void *va = HAP_mmap(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0);
|
||||
#endif
|
||||
if (va == (void*)-1) {
|
||||
FARF(ERROR, "mmap failed : va %p fd %u size %u", va, b->fd, (uint32_t) b->size);
|
||||
void *va = htp_mmap(b->fd, b->size);
|
||||
if (va == NULL) {
|
||||
FARF(ERROR, "mmap failed : fd %u size %u", b->fd, (uint32_t) b->size);
|
||||
abort(); // can't do much else at this point
|
||||
}
|
||||
|
||||
@@ -856,10 +892,13 @@ static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) {
|
||||
m->fd = b->fd;
|
||||
m->size = b->size;
|
||||
|
||||
FARF(HIGH, "mmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
|
||||
FARF(ALWAYS, "mmap : fd %u base %p size %u", m->fd, (void*) m->base, (uint32_t) m->size);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
FARF(ERROR, "mmap failed : exceeded mapping capacity limit of %u", HTP_MAX_MMAPS);
|
||||
abort();
|
||||
}
|
||||
|
||||
static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t n_bufs) {
|
||||
@@ -1081,6 +1120,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
|
||||
rsp.usecs = batch_prof.usecs;
|
||||
rsp.cycles_start = batch_prof.cycles_start;
|
||||
rsp.cycles_stop = batch_prof.cycles_stop;
|
||||
rsp.seq = req->seq;
|
||||
|
||||
if (ctx->profiler == HTP_PROF_TRACE) {
|
||||
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -88,6 +88,7 @@ struct htp_mm_kernel_params {
|
||||
int32_t vtcm_src2_size; // src2 scratchpad size in VTCM (fused only)
|
||||
int32_t vtcm_src3_size; // src3 scratchpad size in VTCM (fused only)
|
||||
int32_t vtcm_dst_size; // dst scratchpad size in VTCM
|
||||
int32_t n_weights; // Number of weights for fused NX
|
||||
|
||||
// Precomputed division values
|
||||
struct fastdiv_values div_ne12_ne1;
|
||||
@@ -463,8 +464,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
|
||||
size_t src2_row_size,
|
||||
uint32_t n_prefetch,
|
||||
bool is_matmul_id,
|
||||
bool is_fused_qkv,
|
||||
bool is_fused_ffn
|
||||
bool is_fused_nx
|
||||
) {
|
||||
size_t src0_sz = 0;
|
||||
size_t src1_sz = 0;
|
||||
@@ -476,44 +476,33 @@ static inline void htp_mm_hvx_vtcm_layout_build(
|
||||
wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL ||
|
||||
wtype == HTP_TYPE_MXFP4);
|
||||
|
||||
if (is_fused_qkv || is_fused_ffn) {
|
||||
if (is_fused_nx) {
|
||||
const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);
|
||||
const size_t quant_scratch_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)) * n_threads;
|
||||
|
||||
size_t src0_sz_per_thread = 0;
|
||||
size_t src2_sz_per_thread = 0;
|
||||
size_t src3_sz_per_thread = 0;
|
||||
size_t weight_sz_per_thread = 0;
|
||||
|
||||
if (is_repack) {
|
||||
uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype);
|
||||
uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32;
|
||||
uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
|
||||
|
||||
src0_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
|
||||
src2_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
|
||||
if (is_fused_qkv) {
|
||||
src3_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
|
||||
}
|
||||
weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
|
||||
} else {
|
||||
src0_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
|
||||
src2_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
|
||||
if (is_fused_qkv) {
|
||||
src3_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
|
||||
}
|
||||
weight_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
|
||||
}
|
||||
|
||||
size_t flat_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
|
||||
size_t tiled_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
|
||||
size_t flat_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
|
||||
size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
|
||||
|
||||
if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
|
||||
src1_sz = hex_round_up(flat_src1_row_size * src1_nrows, 128);
|
||||
} else {
|
||||
src1_sz = hex_round_up(tiled_src1_row_size * src1_nrows, 128);
|
||||
}
|
||||
size_t act_sz = (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT)
|
||||
? hex_round_up(flat_act_row_size * src1_nrows, 128)
|
||||
: hex_round_up(tiled_act_row_size * src1_nrows, 128);
|
||||
|
||||
src0_sz = src0_sz_per_thread * n_threads;
|
||||
src2_sz = src2_sz_per_thread * n_threads;
|
||||
src3_sz = src3_sz_per_thread * n_threads;
|
||||
src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer
|
||||
src1_sz = act_sz; // quantized activation buffer
|
||||
src2_sz = 0;
|
||||
src3_sz = 0;
|
||||
dst_sz = quant_scratch_size;
|
||||
} else if (is_matmul_id) {
|
||||
const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128);
|
||||
@@ -616,8 +605,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
|
||||
}
|
||||
|
||||
size_t off = 0;
|
||||
VTCM_LAYOUT_ALLOC(off, off_src1, src1_sz);
|
||||
VTCM_LAYOUT_ALLOC(off, off_src0, src0_sz);
|
||||
VTCM_LAYOUT_ALLOC(off, off_src1, src1_sz);
|
||||
VTCM_LAYOUT_ALLOC(off, off_src2, src2_sz);
|
||||
VTCM_LAYOUT_ALLOC(off, off_src3, src3_sz);
|
||||
VTCM_LAYOUT_ALLOC(off, off_dst, dst_sz);
|
||||
|
||||
@@ -8,14 +8,20 @@
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "hex-dma.h"
|
||||
#include "dma-queue.h"
|
||||
#include "work-queue.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "hex-utils.h"
|
||||
#include "hvx-copy.h"
|
||||
#include "hvx-quant.h"
|
||||
|
||||
#define GGML_COMMON_DECL_C
|
||||
#include "ggml-common.h"
|
||||
|
||||
#include "htp-ctx.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-tensor.h"
|
||||
#include "htp/set-rows-ops.h"
|
||||
|
||||
#define set_rows_preamble \
|
||||
const uint32_t ne00 = octx->src[0]->ne[0]; \
|
||||
@@ -47,116 +53,142 @@
|
||||
\
|
||||
const uint32_t nr = ne01;
|
||||
|
||||
struct htp_set_rows_context {
|
||||
struct set_rows_context {
|
||||
struct htp_ops_context * octx;
|
||||
struct fastdiv_values div_ne12;
|
||||
struct fastdiv_values div_ne11;
|
||||
uint32_t src0_nrows_per_thread;
|
||||
const struct htp_set_rows_kernel_params * kparams;
|
||||
struct htp_set_rows_vtcm_layout vtcm_layout;
|
||||
uint8_t * vtcm_base;
|
||||
};
|
||||
|
||||
static void set_rows_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) {
|
||||
struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data;
|
||||
struct htp_ops_context * octx = srctx->octx;
|
||||
|
||||
set_rows_preamble;
|
||||
|
||||
uint64_t qt = HAP_perf_get_qtimer_count();
|
||||
|
||||
// parallelize by rows of src0
|
||||
const uint32_t dr = srctx->src0_nrows_per_thread;
|
||||
const uint32_t ir0 = dr * ith;
|
||||
if (ir0 >= nr) {
|
||||
return;
|
||||
}
|
||||
const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr;
|
||||
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
for (uint32_t i03 = 0; i03 < ne03; ++i03) {
|
||||
for (uint32_t i02 = 0; i02 < ne02; ++i02) {
|
||||
for (uint32_t i = ir0; i < ir1; ++i) {
|
||||
const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12);
|
||||
const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11);
|
||||
const uint32_t i10 = i;
|
||||
|
||||
const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
|
||||
|
||||
uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
|
||||
if (i1 >= ne1) {
|
||||
// ignore invalid indices
|
||||
continue;
|
||||
}
|
||||
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + i02*nb02 + i03*nb03;
|
||||
const uintptr_t dst_ptr = octx->dst->data + i1*nb1 + i02*nb2 + i03*nb3;
|
||||
|
||||
// copy row
|
||||
hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, ne00);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
|
||||
FARF(HIGH, "set-rows-f32-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
|
||||
ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
|
||||
#define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \
|
||||
static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
|
||||
struct set_rows_context * srctx = (struct set_rows_context *)data; \
|
||||
struct htp_ops_context * octx = srctx->octx; \
|
||||
const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \
|
||||
set_rows_preamble; \
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
|
||||
const uint32_t dr = kparams->tasks_per_thread; \
|
||||
const uint32_t ir0 = dr * ith; \
|
||||
if (ir0 >= kparams->total_tasks) { \
|
||||
return; \
|
||||
} \
|
||||
const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
|
||||
dma_queue * dma_queue = octx->ctx->dma[ith]; \
|
||||
const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \
|
||||
uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \
|
||||
uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \
|
||||
const uint32_t src0_row_size = ne00 * sizeof(float); \
|
||||
const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \
|
||||
const uint32_t nrows_per_thread = ir1 - ir0; \
|
||||
const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \
|
||||
uint32_t pi_step = 0; \
|
||||
uint32_t pi02 = 0; \
|
||||
uint32_t pi03 = 0; \
|
||||
for (uint32_t step = 0, spad_idx = 0; step < total_steps && spad_idx < 2; ++step, spad_idx++) { \
|
||||
uint32_t i = ir0 + pi_step; \
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + pi02*nb02 + pi03*nb03; \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)octx->dst->data, \
|
||||
vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \
|
||||
dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \
|
||||
(const void *)src0_ptr), \
|
||||
vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \
|
||||
pi_step++; \
|
||||
if (pi_step == nrows_per_thread) { \
|
||||
pi_step = 0; \
|
||||
pi02++; \
|
||||
if (pi02 == ne02) { \
|
||||
pi02 = 0; \
|
||||
pi03++; \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
uint32_t ci_step = 0; \
|
||||
uint32_t ci02 = 0; \
|
||||
uint32_t ci03 = 0; \
|
||||
uint32_t ci11_base = 0; \
|
||||
uint32_t ci12_base = 0; \
|
||||
for (uint32_t step = 0; step < total_steps; ++step) { \
|
||||
void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \
|
||||
void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \
|
||||
uint32_t i = ir0 + ci_step; \
|
||||
const uintptr_t src1_addr = octx->src[1]->data + i*nb10 + ci11_base*nb11 + ci12_base*nb12; \
|
||||
const IDX_TYPE i1 = *(const IDX_TYPE *)src1_addr; \
|
||||
const bool valid_i1 = ((uint64_t)i1 < (uint64_t)ne1); \
|
||||
const uint32_t target_i1 = (uint32_t)i1; \
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, step); \
|
||||
if (valid_i1) { \
|
||||
COMPUTE_EXPR; \
|
||||
} \
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, step); \
|
||||
if (valid_i1) { \
|
||||
const uintptr_t dst_ptr = octx->dst->data + target_i1*nb1 + ci02*nb2 + ci03*nb3; \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \
|
||||
dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 1); \
|
||||
} else { \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)octx->dst->data, (const void *)dst_spad), \
|
||||
dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \
|
||||
} \
|
||||
const uint32_t next_step = step + 2; \
|
||||
if (next_step < total_steps) { \
|
||||
uint32_t ni = ir0 + pi_step; \
|
||||
const uintptr_t psrc0_ptr = octx->src[0]->data + ni*nb01 + pi02*nb02 + pi03*nb03; \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \
|
||||
vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \
|
||||
pi_step++; \
|
||||
if (pi_step == nrows_per_thread) { \
|
||||
pi_step = 0; \
|
||||
pi02++; \
|
||||
if (pi02 == ne02) { \
|
||||
pi02 = 0; \
|
||||
pi03++; \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
ci_step++; \
|
||||
if (ci_step == nrows_per_thread) { \
|
||||
ci_step = 0; \
|
||||
ci02++; \
|
||||
ci11_base++; \
|
||||
if (ci11_base == ne11) { \
|
||||
ci11_base = 0; \
|
||||
} \
|
||||
if (ci02 == ne02) { \
|
||||
ci02 = 0; \
|
||||
ci03++; \
|
||||
ci12_base++; \
|
||||
if (ci12_base == ne12) { \
|
||||
ci12_base = 0; \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
dma_queue_flush(dma_queue); \
|
||||
}
|
||||
|
||||
static void set_rows_thread_f16_f32(unsigned int nth, unsigned int ith, void *data) {
|
||||
struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data;
|
||||
struct htp_ops_context * octx = srctx->octx;
|
||||
SET_ROWS_THREAD_DMA_FN(f32, int32_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
|
||||
SET_ROWS_THREAD_DMA_FN(f32, int64_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
|
||||
|
||||
set_rows_preamble;
|
||||
SET_ROWS_THREAD_DMA_FN(f16, int32_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
|
||||
SET_ROWS_THREAD_DMA_FN(f16, int64_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
|
||||
|
||||
uint64_t qt = HAP_perf_get_qtimer_count();
|
||||
|
||||
// parallelize by rows of src0
|
||||
const uint32_t dr = srctx->src0_nrows_per_thread;
|
||||
const uint32_t ir0 = dr * ith;
|
||||
if (ir0 >= nr) {
|
||||
return;
|
||||
}
|
||||
const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr;
|
||||
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
for (uint32_t i03 = 0; i03 < ne03; ++i03) {
|
||||
for (uint32_t i02 = 0; i02 < ne02; ++i02) {
|
||||
for (uint32_t i = ir0; i < ir1; ++i) {
|
||||
const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12);
|
||||
const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11);
|
||||
const uint32_t i10 = i;
|
||||
|
||||
const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
|
||||
|
||||
uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
|
||||
if (i1 >= ne1) {
|
||||
// ignore invalid indices
|
||||
continue;
|
||||
}
|
||||
|
||||
const uint8_t* src0_ptr = (const uint8_t *) octx->src[0]->data + i*nb01 + i02*nb02 + i03*nb03;
|
||||
uint8_t* dst_ptr = (uint8_t *) octx->dst->data + i1*nb1 + i02*nb2 + i03*nb3;
|
||||
|
||||
hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
|
||||
FARF(HIGH, "set-rows-f16-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
|
||||
ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
|
||||
}
|
||||
SET_ROWS_THREAD_DMA_FN(q8_0, int32_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); })
|
||||
SET_ROWS_THREAD_DMA_FN(q8_0, int64_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); })
|
||||
|
||||
int op_set_rows(struct htp_ops_context * octx) {
|
||||
const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params *)octx->kernel_params;
|
||||
set_rows_preamble;
|
||||
|
||||
const uint32_t n_threads = MIN(nr, octx->n_threads);
|
||||
|
||||
if (octx->src[0]->type != HTP_TYPE_F32) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16) {
|
||||
if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_Q8_0) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
@@ -164,27 +196,27 @@ int op_set_rows(struct htp_ops_context * octx) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
// l2fetch the src1 (indices) tensor in the main thread
|
||||
hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]);
|
||||
|
||||
struct htp_set_rows_context srctx;
|
||||
struct set_rows_context srctx;
|
||||
srctx.octx = octx;
|
||||
srctx.div_ne12 = init_fastdiv_values(ne12);
|
||||
srctx.div_ne11 = init_fastdiv_values(ne11);
|
||||
srctx.kparams = kparams;
|
||||
|
||||
srctx.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads;
|
||||
htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, kparams->n_threads);
|
||||
srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
|
||||
|
||||
switch(octx->dst->type) {
|
||||
case HTP_TYPE_F32:
|
||||
worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f32_f32, &srctx, n_threads);
|
||||
break;
|
||||
case HTP_TYPE_F16:
|
||||
worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f16_f32, &srctx, n_threads);
|
||||
break;
|
||||
default:
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
work_queue_func_t q_func = NULL;
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
switch (octx->dst->type) {
|
||||
case HTP_TYPE_F32: q_func = is_i32 ? set_rows_thread_dma_f32_int32_t : set_rows_thread_dma_f32_int64_t; break;
|
||||
case HTP_TYPE_F16: q_func = is_i32 ? set_rows_thread_dma_f16_int32_t : set_rows_thread_dma_f16_int64_t; break;
|
||||
case HTP_TYPE_Q8_0: q_func = is_i32 ? set_rows_thread_dma_q8_0_int32_t : set_rows_thread_dma_q8_0_int64_t; break;
|
||||
default: return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
work_queue_run(octx->ctx->work_queue, q_func, &srctx, kparams->n_threads);
|
||||
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
#ifndef HTP_SET_ROWS_OPS_H
|
||||
#define HTP_SET_ROWS_OPS_H
|
||||
|
||||
#include "hex-fastdiv.h"
|
||||
|
||||
struct htp_set_rows_kernel_params {
|
||||
int32_t n_threads;
|
||||
int32_t total_tasks;
|
||||
int32_t tasks_per_thread;
|
||||
int32_t vtcm_size;
|
||||
|
||||
// Fastdiv helpers
|
||||
struct fastdiv_values div_ne11;
|
||||
struct fastdiv_values div_ne12;
|
||||
struct fastdiv_values div_tasks_per_thread;
|
||||
struct fastdiv_values div_ne02;
|
||||
};
|
||||
|
||||
struct htp_set_rows_vtcm_layout {
|
||||
size_t total_bytes;
|
||||
size_t off_src0;
|
||||
size_t off_dst;
|
||||
|
||||
size_t src0_bytes_per_thread;
|
||||
size_t dst_bytes_per_thread;
|
||||
|
||||
size_t src0_spad_half_size;
|
||||
size_t dst_spad_half_size;
|
||||
};
|
||||
|
||||
static inline void htp_set_rows_vtcm_layout_build(
|
||||
struct htp_set_rows_vtcm_layout * vtcm_layout,
|
||||
int dst_type,
|
||||
uint32_t ne00,
|
||||
uint32_t n_threads) {
|
||||
|
||||
size_t src0_row_size = ne00 * 4;
|
||||
size_t dst_row_size = 0;
|
||||
switch (dst_type) {
|
||||
case 0: // HTP_TYPE_F32
|
||||
dst_row_size = ne00 * 4;
|
||||
break;
|
||||
case 1: // HTP_TYPE_F16
|
||||
dst_row_size = ne00 * 2;
|
||||
break;
|
||||
case 8: // HTP_TYPE_Q8_0
|
||||
dst_row_size = (ne00 / 32) * 34;
|
||||
break;
|
||||
default:
|
||||
dst_row_size = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;
|
||||
size_t dst_row_size_aligned = (dst_row_size + 255) & ~255;
|
||||
|
||||
vtcm_layout->src0_spad_half_size = src0_row_size_aligned;
|
||||
vtcm_layout->dst_spad_half_size = dst_row_size_aligned;
|
||||
|
||||
vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
|
||||
vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2;
|
||||
|
||||
vtcm_layout->off_src0 = 0;
|
||||
vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;
|
||||
vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
|
||||
}
|
||||
|
||||
#if defined(__cplusplus)
|
||||
static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob");
|
||||
#else
|
||||
_Static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob");
|
||||
#endif
|
||||
|
||||
#endif // HTP_SET_ROWS_OPS_H
|
||||
Reference in New Issue
Block a user