diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index 363be41ba..95b6127a4 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -255,7 +255,7 @@ Writing high-performance operators for Hexagon requires following specific guide ### Tracing Instrumentation - All kernels must include trace events for performance profiling and timeline visualization in Perfetto - ([`hex-profile.h`](../../../ggml/src/ggml-hexagon/htp/hex-profile.h)). + ([`htp-trace.h`](../../../ggml/src/ggml-hexagon/htp/htp-trace.h)). - Surround compute sections with `htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)` and `htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)`. - Use specific event types for major phases: diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 9b2600c66..185aa4d46 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -242,10 +242,11 @@ static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbat char evt_str[256] = "----"; if (opt_profile == 3) { - snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u", + static_assert(HTP_OP_MAX_NTHREADS == 10, "current implementation assumes 10 trace counters"); + snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u", rsp.n_traces[0], rsp.n_traces[1], rsp.n_traces[2], rsp.n_traces[3], rsp.n_traces[4], rsp.n_traces[5], rsp.n_traces[6], rsp.n_traces[7], - rsp.n_traces[8], rsp.n_traces[9], rsp.n_traces[10]); + rsp.n_traces[8], rsp.n_traces[9]); } GGML_LOG_DEBUG("ggml-hex: %s profile-op OPBATCH|----|n-ops %u|%s|----|----|usec %u cycles %llu start %llu mhz %.1f\n", @@ -253,16 +254,11 @@ static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbat } static void ggml_hexagon_dump_trace_events(const char * sess_name, const htp_opbatch_rsp & rsp, - const htp_trace_desc * trace_events, uint32_t n_traces) { + const htp_trace_desc * trace_events, uint32_t n_threads, uint32_t n_traces) { if (opt_profile == 3 && trace_events) { - uint32_t valid_cnt[HTP_MAX_NTHREADS + 1] = {0}; - for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) { - uint32_t count = rsp.n_traces[t]; - valid_cnt[t] = count > n_traces ? n_traces : count; - } - - for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) { - for (uint32_t idx = 0; idx < valid_cnt[t]; idx++) { + for (uint32_t t = 0; t < n_threads; t++) { + uint32_t count = std::min(rsp.n_traces[t], n_traces); + for (uint32_t idx = 0; idx < count; idx++) { const auto & e = trace_events[t * n_traces + idx]; bool is_stop = (e.info & 0x8000) != 0; uint16_t info = e.info & 0x7FFF; @@ -393,7 +389,7 @@ struct ggml_hexagon_session { const char * htp_session_name(const struct ggml_hexagon_session * sess) { return sess->c_name(); } size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess) { return sess->vtcm_size; } -uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess) { return sess->n_threads; } +uint32_t htp_session_n_hvx(const struct ggml_hexagon_session * sess) { return sess->n_hvx; } uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess) { return sess->n_hmx; } uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess) { return sess->mdev.count; } @@ -1547,7 +1543,7 @@ struct ggml_hexagon_opqueue { size_t tr_size = 0; if (opt_profile == 3) { - tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc); + tr_size = sess->n_threads * opt_optrace * sizeof(htp_trace_desc); } shm_blk_size = sizeof(htp_buf_desc) * n_bufs + @@ -1602,7 +1598,7 @@ struct ggml_hexagon_opqueue { size_t tr_size = 0; if (opt_profile == 3) { req.n_traces = opt_optrace; - tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc); + tr_size = shm_buf->sess->n_threads * req.n_traces * sizeof(htp_trace_desc); } else { req.n_traces = 0; } @@ -1657,7 +1653,7 @@ struct ggml_hexagon_opqueue { uint32_t n_traces = 0; if (opt_profile == 3) { n_traces = opt_optrace; - tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc); + tr_size = shm_buf->sess->n_threads * n_traces * sizeof(htp_trace_desc); } const size_t m_size = b_size + t_size + o_size + p_size + tr_size; @@ -1686,7 +1682,7 @@ struct ggml_hexagon_opqueue { ggml_hexagon_dump_op_prof(shm_buf->sess->c_name(), ops[i], pd[i]); } - ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, n_traces); + ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, shm_buf->sess->n_threads, n_traces); } } }; @@ -2162,11 +2158,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n unsigned long long hw_vtcm_size = 0; int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size); if (hw_err == 0) { - const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads); - this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads; - this->n_hvx = this->n_threads; - this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; - this->vtcm_size = (uint64_t)hw_vtcm_size; + this->n_threads = (std::min)((uint32_t) HTP_OP_MAX_NTHREADS, (uint32_t) hw_n_threads); + const uint32_t max_n_hvx = (std::min)({this->n_threads - 1, (uint32_t) hw_n_hvx, (uint32_t) HTP_OP_MAX_NHVX}); + this->n_hvx = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_hvx) : max_n_hvx; + this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; + this->vtcm_size = (uint64_t)hw_vtcm_size; if (opt_vtcm_size > 0) { if (opt_vtcm_size > hw_vtcm_size) { GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size); @@ -2178,11 +2174,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n (unsigned long long)(this->vtcm_size / (1024 * 1024))); } else { GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err); - const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS); - this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads; - this->n_hvx = this->n_threads; - this->n_hmx = (opt_nhmx != 0) ? 1 : 0; - this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024; + this->n_threads = (std::min)(8u, (uint32_t) HTP_OP_MAX_NTHREADS); + const uint32_t max_n_hvx = (std::min)(this->n_threads - 1, (uint32_t) HTP_OP_MAX_NHVX); + this->n_hvx = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_hvx) : (std::min)(4u, max_n_hvx); + this->n_hmx = (opt_nhmx != 0) ? 1 : 0; + this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024; } } @@ -2251,7 +2247,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); // Start dspqueue/opbatch processing - err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem, + err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_hvx, opt_nhmx, opt_vtcm_size, this->max_vmem, (uint32_t) opt_power_mode); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.h index 815a021b8..4e9e54415 100644 --- a/ggml/src/ggml-hexagon/htp-common.h +++ b/ggml/src/ggml-hexagon/htp-common.h @@ -97,7 +97,7 @@ struct ggml_hexagon_session; const char * htp_session_name(const struct ggml_hexagon_session * sess); size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess); -uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess); +uint32_t htp_session_n_hvx(const struct ggml_hexagon_session * sess); uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess); uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess); diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp index 7a3bdd46b..eb0862669 100644 --- a/ggml/src/ggml-hexagon/htp-fusion.cpp +++ b/ggml/src/ggml-hexagon/htp-fusion.cpp @@ -239,11 +239,11 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) { if (kparams.m_chunk < orig_kparams->m_chunk || kparams.n_chunk < orig_kparams->n_chunk || - kparams.n_act_threads < orig_kparams->n_act_threads) { - HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n", + kparams.n_act_hvx < orig_kparams->n_act_hvx) { + HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, hvx %d->%d)\n", htp_session_name(sess), orig_kparams->m_chunk, kparams.m_chunk, orig_kparams->n_chunk, kparams.n_chunk, - orig_kparams->n_act_threads, kparams.n_act_threads); + orig_kparams->n_act_hvx, kparams.n_act_hvx); return false; } } @@ -546,8 +546,8 @@ static bool fuse_binary_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const uint32_t total_elems = (uint32_t) ggml_nelements(src0); const uint32_t elem_size = (uint32_t) ggml_type_size(src0->type); - const uint32_t max_threads = htp_session_n_threads(sess); - if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_threads, n_srcs, false, vtcm_size)) { + const uint32_t max_hvx = htp_session_n_hvx(sess); + if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_hvx, n_srcs, false, vtcm_size)) { return false; } } diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index 49b673e85..76923d26b 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -154,7 +154,7 @@ bool htp_kparams_solve_allreduce( kparams->n_ranks = (int32_t) n_ranks; kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; - const uint32_t max_threads = htp_session_n_threads(sess); + const uint32_t max_hvx = htp_session_n_hvx(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); const uint32_t nelem = (uint32_t) ggml_nelements(dst); @@ -208,27 +208,27 @@ bool htp_kparams_solve_allreduce( if (use_1d) { const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; - const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nelem / 128)); - kparams->n_threads = n_threads; - const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); + const uint32_t n_hvx = (std::min)(max_hvx, (std::max)(1u, rank_nelem / 128)); + kparams->n_hvx = n_hvx; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_hvx, has_add, is_row_bcast); uint32_t block_elems = 65536; - if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) { - block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128); + if (block_elems > rank_nelem / n_hvx && rank_nelem / n_hvx > 128) { + block_elems = hex_round_up(rank_nelem / (n_hvx * 2), 128); } block_elems = (std::max)(128u, block_elems); kparams->block_elems = block_elems; - kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + kparams->vtcm_size_per_hvx = 2 * block_elems * elem_size; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; while ((size_t) kparams->vtcm_size > vtcm_size && block_elems > 128) { const size_t max_bytes_per_buf = vtcm_size / (n_vtcm_buffers * 2); block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); if (block_elems < 128) break; kparams->block_elems = block_elems; - kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + kparams->vtcm_size_per_hvx = 2 * block_elems * elem_size; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; } if (vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { @@ -237,33 +237,33 @@ bool htp_kparams_solve_allreduce( return false; } - kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems); + kparams->elems_per_hvx = hex_round_up((rank_nelem + n_hvx - 1) / n_hvx, block_elems); kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D; return true; } else { const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; - const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nrows)); - kparams->n_threads = n_threads; - const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); + const uint32_t n_hvx = (std::min)(max_hvx, (std::max)(1u, rank_nrows)); + kparams->n_hvx = n_hvx; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_hvx, has_add, is_row_bcast); const uint32_t row_bytes = ne0 * elem_size; const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); kparams->row_size_aligned = row_size_aligned; - const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads; - uint32_t block_rows = (std::min)(128u, nrows_per_thread); + const uint32_t nrows_per_hvx = (rank_nrows + n_hvx - 1) / n_hvx; + uint32_t block_rows = (std::min)(128u, nrows_per_hvx); block_rows = (std::max)(1u, block_rows); kparams->block_elems = block_rows; - kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + kparams->vtcm_size_per_hvx = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; while ((size_t) kparams->vtcm_size > vtcm_size && block_rows > 1) { const size_t max_rows_per_buf = vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned); block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); kparams->block_elems = block_rows; - kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + kparams->vtcm_size_per_hvx = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_hvx; if (max_rows_per_buf == 0) break; } @@ -273,7 +273,7 @@ bool htp_kparams_solve_allreduce( return false; } - kparams->elems_per_thread = nrows_per_thread; + kparams->elems_per_hvx = nrows_per_hvx; kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D; return true; } @@ -313,12 +313,12 @@ static bool htp_flash_attn_is_hmx_eligible( return true; } - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t n_head = q->ne[2]; const uint32_t n_kv_heads = k->ne[2]; const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1; const uint32_t S = k->ne[1]; - const uint32_t total_threads = n_threads * htp_session_mdev_count(sess); + const uint32_t total_threads = n_hvx * htp_session_mdev_count(sess); if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) { return true; @@ -326,7 +326,7 @@ static bool htp_flash_attn_is_hmx_eligible( const bool is_tile_aligned = (G > 0 && (32 % G == 0)); if (!is_tile_aligned) { - if (n_threads >= 6) { + if (n_hvx >= 6) { return false; } return S >= 1024; @@ -336,9 +336,9 @@ static bool htp_flash_attn_is_hmx_eligible( if (DK == 256) { s_cross = 4096; } else if (DK <= 64) { - s_cross = (n_threads >= 8) ? 2048 : ((n_threads >= 6) ? 768 : 512); + s_cross = (n_hvx >= 8) ? 2048 : ((n_hvx >= 6) ? 768 : 512); } else { - s_cross = (n_threads >= 8) ? 1024 : 512; + s_cross = (n_hvx >= 8) ? 1024 : 512; } return S >= s_cross; @@ -382,7 +382,7 @@ static bool htp_kparams_solve_flash_attn( return false; } - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); const uint32_t neq0 = q->ne[0]; @@ -434,7 +434,7 @@ static bool htp_kparams_solve_flash_attn( const uint32_t DK_pad = hex_round_up(DK, 64); const uint32_t DV_pad = hex_round_up(DV, 64); size_t Br = 0, Bc = 0; - int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size); + int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_hvx, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size); if (ret == 0) { kparams->kernel_type = HTP_FA_KERNEL_HMX; if (logit_softcap == 0.0f) { @@ -447,10 +447,10 @@ static bool htp_kparams_solve_flash_attn( kparams->Br = Br; kparams->Bc = Bc; kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc; - kparams->n_threads = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1; + kparams->n_hvx = (kparams->n_kv_blocks >= 3 && n_hvx >= 2) ? n_hvx : 1; kparams->u.hmx.g_br = hex_align_up(G * Br, 32); - kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? 1 : 0; - kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head); + kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_hvx >= 2) ? 1 : 0; + kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_hvx, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head); kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0; kparams->u.hmx.div_G = init_fastdiv_values(G); if (mask) { @@ -465,13 +465,13 @@ static bool htp_kparams_solve_flash_attn( kparams->Br = 1; kparams->Bc = 64; kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128); const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128); const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128); - kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_threads); + kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_hvx); if ((size_t) kparams->vtcm_size > vtcm_size) { HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n", kparams->vtcm_size, vtcm_size); @@ -554,7 +554,7 @@ static bool htp_kparams_solve_gated_delta_net( const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]); const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]); const uint32_t total_rows = (uint32_t) (H * n_seqs); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), total_rows); const size_t vtcm_size = htp_session_vtcm_size(sess); const bool can_use_hmx = (opt_gdn_select >= 2) && @@ -568,7 +568,7 @@ static bool htp_kparams_solve_gated_delta_net( struct htp_gdn_vtcm_layout hvx_layout; uint32_t n_heads_batch = 1; - if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_threads, true, &n_heads_batch)) { + if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_hvx, true, &n_heads_batch)) { kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED; kparams->pipeline = hmx_layout.pipeline ? 1 : 0; kparams->chunk_size = HTP_GDN_CHUNK_SIZE; @@ -576,7 +576,7 @@ static bool htp_kparams_solve_gated_delta_net( kparams->n_heads_batch = (uint16_t) n_heads_batch; kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes; } else { - htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_threads); + htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_hvx); if (hvx_layout.total_bytes > vtcm_size) { return false; } @@ -586,7 +586,7 @@ static bool htp_kparams_solve_gated_delta_net( kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes; } - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->S_v = (uint32_t) S_v; kparams->H = (uint32_t) H; kparams->n_tokens = (uint32_t) n_tokens; @@ -677,7 +677,7 @@ static bool htp_kparams_solve_hmx_mm( ) { const bool is_tiled = ggml_hexagon_tensor_will_tile(src0); const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0; - const int n_threads = (int) htp_session_n_threads(sess); + const int n_hvx = (int) htp_session_n_hvx(sess); const uint32_t mdev_count = htp_session_mdev_count(sess); const int ne10 = act->ne[0]; @@ -699,10 +699,10 @@ static bool htp_kparams_solve_hmx_mm( size_t n_chunk = 0; size_t vtcm_size = 0; bool use_grouped = false; - int act_threads_selected = 0; + int act_hvx_selected = 0; if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) { - if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_hvx, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_hvx_selected, &vtcm_size)) { use_grouped = true; } } @@ -719,7 +719,7 @@ static bool htp_kparams_solve_hmx_mm( m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32); } const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver; - if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_hvx, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_hvx_selected, &vtcm_size)) { return false; } } @@ -728,13 +728,13 @@ static bool htp_kparams_solve_hmx_mm( kparams->pipeline = pipeline ? 1 : 0; kparams->m_chunk = m_chunk; kparams->n_chunk = n_chunk; - kparams->n_threads = n_threads; - kparams->n_act_threads = act_threads_selected; + kparams->n_hvx = n_hvx; + kparams->n_act_hvx = act_hvx_selected; kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0; kparams->aligned_tile_size = aligned_tile_size; kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); kparams->vtcm_size = vtcm_size; - kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected); + kparams->div_n_act_hvx = init_fastdiv_values(act_hvx_selected); kparams->div_ne00_padded = init_fastdiv_values(ne00_padded); kparams->vtcm_bias_size = (int32_t) src2_size; kparams->kernel_type = (is_batched && !is_matmul_id) ? HTP_MM_KERNEL_HMX_BATCHED : HTP_MM_KERNEL_HMX_2D; @@ -764,9 +764,9 @@ static void htp_kparams_solve_hvx_mm( return; } - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); kparams->n_hmx = 0; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; GGML_UNUSED(ne02); GGML_UNUSED(ne03); @@ -780,7 +780,7 @@ static void htp_kparams_solve_hvx_mm( const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0); if (is_matmul_id) { - kparams->kernel_type = (src1_nrows < (int) n_threads) + kparams->kernel_type = (src1_nrows < (int) n_hvx) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); @@ -789,7 +789,7 @@ static void htp_kparams_solve_hvx_mm( uint32_t best_n_prefetch = 2; for (uint32_t d = max_prefetch; d >= 2; d /= 2) { htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, 0, src0->nb[1], kparams->act_row_size, 0, d, true, false ); if (L.total_bytes <= vtcm_budget) { @@ -808,7 +808,7 @@ static void htp_kparams_solve_hvx_mm( } else { if (k_align && opt_mm_select >= 1) { kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); - kparams->kernel_type = (src1_nrows < (int) n_threads) + kparams->kernel_type = (src1_nrows < (int) n_hvx) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; struct htp_mm_hvx_vtcm_layout L; @@ -816,7 +816,7 @@ static void htp_kparams_solve_hvx_mm( uint32_t best_n_prefetch = 2; for (uint32_t d = max_prefetch; d >= 2; d /= 2) { htp_mm_hvx_vtcm_layout_build( - &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false ); if (L.total_bytes <= vtcm_budget) { @@ -827,7 +827,7 @@ static void htp_kparams_solve_hvx_mm( uint32_t m_chunk = 0; if (htp_mm_hvx_solve_vtcm_params( - kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + kparams->kernel_type, wtype, ne10, src1_nrows, n_hvx, dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget, &L, &m_chunk)) { kparams->n_prefetch = best_n_prefetch; @@ -857,7 +857,7 @@ static void htp_kparams_solve_hvx_mm( struct htp_mm_hvx_vtcm_layout L; uint32_t m_chunk = 0; if (htp_mm_hvx_solve_vtcm_params( - ktype, wtype, ne10, src1_nrows, n_threads, + ktype, wtype, ne10, src1_nrows, n_hvx, dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget, &L, &m_chunk)) { kparams->kernel_type = ktype; @@ -1093,12 +1093,12 @@ bool htp_kparams_solve_mmnx( int32_t n_weights, struct htp_mm_kernel_params * kparams ) { - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t n_hmx = htp_session_n_hmx(sess); const size_t vtcm_budget = htp_session_vtcm_size(sess); memset(kparams, 0, sizeof(*kparams)); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; @@ -1140,7 +1140,7 @@ bool htp_kparams_solve_mmnx( for (uint32_t d = max_prefetch; d >= 2; d /= 2) { struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads, + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_hvx, 0, src0_row_size, act_row_size, 0, d, false, true ); if (L.total_bytes <= vtcm_budget) { @@ -1151,7 +1151,7 @@ bool htp_kparams_solve_mmnx( struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( - &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads, + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_hvx, 0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true ); @@ -1244,7 +1244,7 @@ static bool htp_kparams_solve_binary( } } - const uint32_t max_threads_sess = htp_session_n_threads(sess); + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); const size_t elem_size = ggml_type_size(src0->type); @@ -1278,14 +1278,14 @@ static bool htp_kparams_solve_binary( const uint32_t total_elems = (uint32_t) ggml_nelements(src0); const uint32_t max_chunk_elems = 32768 / elem_size; const uint32_t min_chunk_elems = 256; - const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads_sess) - 1) / (2 * max_threads_sess), 32); + const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_hvx_sess) - 1) / (2 * max_hvx_sess), 32); const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems)); const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size; - const uint32_t n_threads = (total_chunks >= 2) ? (std::min)(max_threads_sess, total_chunks) : 1; + const uint32_t n_hvx = (total_chunks >= 2) ? (std::min)(max_hvx_sess, total_chunks) : 1; kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->n_srcs = 2; kparams->rows_per_buffer = 1; kparams->src0_row_size_aligned = src0_row_size_aligned; @@ -1346,11 +1346,11 @@ static bool htp_kparams_solve_binary( } const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); kparams->kernel_type = kernel_type; - kparams->n_threads = (std::min)(max_threads_sess, max_threads); + kparams->n_hvx = (std::min)(max_hvx_sess, max_hvx); kparams->n_srcs = 2; kparams->src0_row_size_aligned = src0_row_size_aligned; kparams->src1_row_size_aligned = src1_row_size_aligned; @@ -1403,7 +1403,7 @@ static bool htp_kparams_solve_binary( if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) { const uint32_t total_elems = (uint32_t) ggml_nelements(src0); return htp_binary_solve_chunked(&L, kparams, total_elems, (uint32_t) elem_size, - max_threads_sess, 2, is_scalar_broadcast, vtcm_size); + max_hvx_sess, 2, is_scalar_broadcast, vtcm_size); } return false; } @@ -1415,7 +1415,7 @@ static bool htp_kparams_solve_binary( } // collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector -static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_threads) { +static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_hvx) { if (n_tensors == 0 || !tensors[0]) { return; } @@ -1497,9 +1497,9 @@ static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, s break; } } - if (nrows < n_threads && rows_contig && row_size_curr >= 4096) { + if (nrows < n_hvx && rows_contig && row_size_curr >= 4096) { const size_t vec_elems = 128 / elem_size; - int64_t k = std::max(row_size_curr / 16384, (n_threads * 4 + nrows - 1) / nrows); + int64_t k = std::max(row_size_curr / 16384, (n_hvx * 4 + nrows - 1) / nrows); k = std::min(k, row_size_curr / 2048); while (k > 1 && t0->ne[0] % (k * vec_elems) != 0) { k--; @@ -1576,12 +1576,12 @@ static bool htp_kparams_solve_unary_impl( const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); const size_t vtcm_size = htp_session_vtcm_size(sess); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); const bool broadcast_weight = has_weight && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); @@ -1590,13 +1590,13 @@ static bool htp_kparams_solve_unary_impl( } kparams->broadcast_weight = broadcast_weight; - const bool src0_contig = ggml_is_contiguous_1(src0); - const bool src1_contig = (has_weight && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true; - const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - const uint32_t min_block_rows = (uint32_t) (std::max(1, 2048 / dst_data_row_size)); - uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1]; - if (src0_nrows_per_thread >= 2 * min_block_rows) { - max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_thread / 4)); + const bool src0_contig = ggml_is_contiguous_1(src0); + const bool src1_contig = (has_weight && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true; + const uint32_t src0_nrows_per_hvx = (src0_nrows + n_hvx - 1) / n_hvx; + const uint32_t min_block_rows = (uint32_t) (std::max(1, 2048 / dst_data_row_size)); + uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1]; + if (src0_nrows_per_hvx >= 2 * min_block_rows) { + max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_hvx / 4)); } kparams->src0_contig = src0_contig ? 1 : 0; @@ -1604,16 +1604,16 @@ static bool htp_kparams_solve_unary_impl( struct htp_unary_vtcm_layout L; uint32_t col_tile = 0; - uint32_t vtcm_row_per_thread = 0; + uint32_t vtcm_row_per_hvx = 0; htp_unary_solve_layout(&L, op, src0->ne[0], dst->ne[0], has_weight ? src1->ne[0] : 0, - max_block, broadcast_weight, n_threads, vtcm_size, elem_size, - &col_tile, &vtcm_row_per_thread); + max_block, broadcast_weight, n_hvx, vtcm_size, elem_size, + &col_tile, &vtcm_row_per_hvx); kparams->col_tile = col_tile; kparams->vtcm_size = L.total_bytes; - kparams->block = col_tile ? 0 : vtcm_row_per_thread; + kparams->block = col_tile ? 0 : vtcm_row_per_hvx; const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1; kparams->tiles_per_row = tiles_per_row; @@ -1622,7 +1622,7 @@ static bool htp_kparams_solve_unary_impl( kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]); kparams->div_tpr = init_fastdiv_values(tiles_per_row); - return (col_tile > 0 || vtcm_row_per_thread > 0) && L.total_bytes <= vtcm_size; + return (col_tile > 0 || vtcm_row_per_hvx > 0) && L.total_bytes <= vtcm_size; } static bool htp_kparams_solve_unary( @@ -1710,22 +1710,22 @@ bool htp_kparams_solve_repeat_get_rows( } const uint32_t total_tasks = nr; - const uint32_t min_tasks_per_thread = (uint32_t) (std::max(1, (128 + dst->nb[1] - 1) / dst->nb[1])); - const uint32_t max_threads = (std::max)(1u, total_tasks / min_tasks_per_thread); + const uint32_t min_tasks_per_hvx = (uint32_t) (std::max(1, (128 + dst->nb[1] - 1) / dst->nb[1])); + const uint32_t max_hvx = (std::max)(1u, total_tasks / min_tasks_per_hvx); const size_t vtcm_size = htp_session_vtcm_size(sess); - kparams->n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); struct htp_get_rows_vtcm_layout vtcm_layout = {}; - while (kparams->n_threads > 0) { - htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_threads); + while (kparams->n_hvx > 0) { + htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_hvx); if (vtcm_layout.total_bytes <= vtcm_size) { break; } - --kparams->n_threads; + --kparams->n_hvx; } - if (kparams->n_threads == 0 && total_tasks > 0) { + if (kparams->n_hvx == 0 && total_tasks > 0) { htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1); } @@ -1740,7 +1740,7 @@ bool htp_kparams_solve_repeat_get_rows( if (total_tasks == 0) { return true; } - return kparams->n_threads > 0 && (size_t) kparams->vtcm_size <= vtcm_size; + return kparams->n_hvx > 0 && (size_t) kparams->vtcm_size <= vtcm_size; } static bool htp_kparams_solve_get_rows( @@ -1777,11 +1777,11 @@ static bool htp_kparams_solve_set_rows( const uint32_t nr = src0->ne[1]; - kparams->n_threads = (std::min)(htp_session_n_threads(sess), nr); + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), nr); kparams->total_tasks = nr; struct htp_set_rows_vtcm_layout vtcm_layout; - htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads); + htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_hvx); kparams->vtcm_size = vtcm_layout.total_bytes; return vtcm_layout.total_bytes <= htp_session_vtcm_size(sess); @@ -1819,9 +1819,9 @@ static bool htp_kparams_solve_softmax( const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const size_t dst_data_row_size = op->ne[0] * ggml_type_size(op->type); - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); float scale = 1.0f; float max_bias = 0.0f; @@ -1851,12 +1851,12 @@ static bool htp_kparams_solve_softmax( const uint32_t ne10 = src1 ? src1->ne[0] : 1; struct htp_softmax_vtcm_layout layout; - htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads); + htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_hvx); if (layout.total_bytes > htp_session_vtcm_size(sess)) { return false; } - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; if (!kparams->use_src1) { @@ -1962,16 +1962,16 @@ static bool htp_kparams_solve_rope( return false; } - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), src0_nrows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), src0_nrows); const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0; struct htp_rope_vtcm_layout layout; - htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors); + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_hvx, n_freq_factors); if (layout.total_bytes > htp_session_vtcm_size(sess)) { return false; } - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); @@ -2005,26 +2005,26 @@ static bool htp_kparams_solve_sum_rows( const uint32_t ne00 = src0->ne[0]; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t max_threads_sess = htp_session_n_threads(sess); + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); if (op->op == GGML_OP_SUM) { const uint32_t total_elems = (uint32_t) ggml_nelements(src0); - const uint32_t n_threads = (total_elems >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1; - const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads; + const uint32_t n_hvx = (total_elems >= 1024) ? max_hvx_sess : 1; + const uint32_t raw_chunk = (total_elems + n_hvx - 1) / n_hvx; - kparams->n_threads = n_threads; - kparams->elems_per_thread = hex_round_up(raw_chunk, 32); + kparams->n_hvx = n_hvx; + kparams->elems_per_hvx = hex_round_up(raw_chunk, 32); } else if (op->op == GGML_OP_ARGMAX && src0_nrows <= 1) { - const uint32_t n_threads = (ne00 >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1; - const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads; + const uint32_t n_hvx = (ne00 >= 1024) ? max_hvx_sess : 1; + const uint32_t raw_chunk = (ne00 + n_hvx - 1) / n_hvx; - kparams->n_threads = n_threads; - kparams->elems_per_thread = hex_round_up(raw_chunk, 32); + kparams->n_hvx = n_hvx; + kparams->elems_per_hvx = hex_round_up(raw_chunk, 32); } else { const uint32_t min_rpt = (src0->nb[1] < 1024) ? 32 : 1; - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rpt); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rpt); - kparams->n_threads = (std::min)(max_threads_sess, max_threads); + kparams->n_hvx = (std::min)(max_hvx_sess, max_hvx); } return true; @@ -2065,28 +2065,28 @@ bool htp_kparams_solve_unary_mul_glu( const size_t dst_row_size = dst->ne[0] * elem_size; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); - const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, src0_nrows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); const size_t vtcm_size = htp_session_vtcm_size(sess); const size_t row_size_aligned = hex_round_up(dst_row_size, 128); - uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned)); - if (vtcm_row_per_thread < 2) { + uint32_t vtcm_row_per_hvx = (uint32_t) (vtcm_size / (n_hvx * 3 * row_size_aligned)); + if (vtcm_row_per_hvx < 2) { return false; } - const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - if (src0_nrows_per_thread >= 2 && vtcm_row_per_thread > src0_nrows_per_thread) { - uint32_t target_rpt = (src0_nrows_per_thread >= 4) ? (src0_nrows_per_thread / 2) & ~1u : 2; - vtcm_row_per_thread = (std::max)(2u, (std::min)(vtcm_row_per_thread, target_rpt)); + const uint32_t src0_nrows_per_hvx = (src0_nrows + n_hvx - 1) / n_hvx; + if (src0_nrows_per_hvx >= 2 && vtcm_row_per_hvx > src0_nrows_per_hvx) { + uint32_t target_rpt = (src0_nrows_per_hvx >= 4) ? (src0_nrows_per_hvx / 2) & ~1u : 2; + vtcm_row_per_hvx = (std::max)(2u, (std::min)(vtcm_row_per_hvx, target_rpt)); } struct htp_glu_vtcm_layout L; - htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread); + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, vtcm_row_per_hvx); - kparams->n_threads = n_threads; - kparams->vtcm_row_per_thread = L.vtcm_row_per_thread; + kparams->n_hvx = n_hvx; + kparams->vtcm_row_per_hvx = L.vtcm_row_per_hvx; kparams->vtcm_size = (uint32_t) L.total_bytes; return true; @@ -2119,11 +2119,11 @@ static bool htp_kparams_solve_repeat( const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; const size_t dst_row_size = dst->ne[0] * ggml_type_size(dst->type); - const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); - const uint32_t max_threads = (std::max)(1u, total_dst_rows / min_rows_per_thread); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t min_rows_per_hvx = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_hvx = (std::max)(1u, total_dst_rows / min_rows_per_hvx); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->nr0 = dst->ne[0] / src0->ne[0]; kparams->nr1 = dst->ne[1] / src0->ne[1]; kparams->nr2 = dst->ne[2] / src0->ne[2]; @@ -2186,31 +2186,31 @@ static bool htp_kparams_solve_ssm_conv( const uint32_t n_s = (uint32_t) n_s_64; const uint32_t ncs = (uint32_t) src0->ne[0]; - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), (d_inner + 31) / 32); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), (d_inner + 31) / 32); const size_t vtcm_size = htp_session_vtcm_size(sess); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->d_conv = d_conv; kparams->d_inner = d_inner; kparams->n_t = n_t; kparams->n_s = n_s; - const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads; - const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32); + const uint32_t raw_rpt = (d_inner + n_hvx - 1) / n_hvx; + const uint32_t d_inner_per_hvx = hex_round_up(raw_rpt, 32); if (n_t == 1) { - kparams->d_inner_tile = d_inner_per_thread; + kparams->d_inner_tile = d_inner_per_hvx; } else { - const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; - const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); - const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes; + const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_hvx * d_conv * sizeof(float), 128) + 128; + const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_hvx * sizeof(float), 128); + const uint32_t vtcm_src1_per_hvx = src1_raw_bytes + src1_T_bytes; - const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_threads : (1024 * 1024)); + const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_hvx : (1024 * 1024)); const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128); - const size_t fixed_bytes = vtcm_src1_per_thread + src0_block_T; + const size_t fixed_bytes = vtcm_src1_per_hvx + src0_block_T; const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024); - uint32_t target_max_tile = hex_round_up((d_inner_per_thread + 3) / 4, 32); + uint32_t target_max_tile = hex_round_up((d_inner_per_hvx + 3) / 4, 32); target_max_tile = (std::max)(target_max_tile, 32u); target_max_tile = (std::min)(target_max_tile, 128u); @@ -2223,14 +2223,14 @@ static bool htp_kparams_solve_ssm_conv( if (d_inner_tile > target_max_tile) { d_inner_tile = target_max_tile; } - if (d_inner_tile > d_inner_per_thread) { - d_inner_tile = d_inner_per_thread; + if (d_inner_tile > d_inner_per_hvx) { + d_inner_tile = d_inner_per_hvx; } kparams->d_inner_tile = d_inner_tile; } struct htp_ssm_conv_vtcm_layout L; - htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, d_conv, n_t, kparams->d_inner_tile, n_threads); + htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_hvx, d_conv, n_t, kparams->d_inner_tile, n_hvx); kparams->vtcm_size = (uint32_t) L.total_bytes; kparams->div_d_inner_tile = init_fastdiv_values(kparams->d_inner_tile); @@ -2256,7 +2256,7 @@ static bool htp_kparams_solve_sort( const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const uint32_t ne00 = src0->ne[0]; const uint32_t k = dst->ne[0]; - const uint32_t n_threads_max = htp_session_n_threads(sess) > 0 ? htp_session_n_threads(sess) : 4; + const uint32_t n_hvx_max = htp_session_n_hvx(sess) > 0 ? htp_session_n_hvx(sess) : 4; const size_t vtcm_budget = htp_session_vtcm_size(sess) > 0 ? htp_session_vtcm_size(sess) : (8 * 1024 * 1024); int32_t order = GGML_SORT_ORDER_DESC; @@ -2265,11 +2265,11 @@ static bool htp_kparams_solve_sort( } struct htp_sort_vtcm_layout layout; - if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k)) { + if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_hvx_max, vtcm_budget, is_top_k)) { return false; } - kparams->n_threads = layout.n_threads; + kparams->n_hvx = layout.n_hvx; kparams->order = (uint32_t) order; kparams->is_top_k = is_top_k ? 1 : 0; kparams->chunk_elems = layout.chunk_elems; @@ -2347,11 +2347,11 @@ static bool htp_kparams_solve_pool_2d( const uint32_t planes = (uint32_t) (is_pool_1d ? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3] : (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), planes > 0 ? planes : 1); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), planes > 0 ? planes : 1); const uint32_t planes_per_chunk = (dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1; struct htp_pool_vtcm_layout layout; - if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_threads, htp_session_vtcm_size(sess))) { + if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_hvx, htp_session_vtcm_size(sess))) { return false; } @@ -2365,7 +2365,7 @@ static bool htp_kparams_solve_pool_2d( kparams->stride_y = (uint32_t) stride_y; kparams->pad_x = pad_x; kparams->pad_y = pad_y; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->planes = planes; kparams->planes_per_chunk = planes_per_chunk; kparams->pool_op = (uint32_t) params[0]; @@ -2471,18 +2471,18 @@ bool htp_kparams_solve_upscale( const float invscale1 = antialias ? (1.0f / filter_radius1) : 0.0f; const uint32_t total_slices = dst_z * dst_w; - uint32_t n_threads = htp_session_n_threads(sess); + uint32_t n_hvx = htp_session_n_hvx(sess); struct htp_upscale_vtcm_layout layout; uint32_t dst_nrows_per_chunk = 0; - while (n_threads > 0) { - if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_threads, + while (n_hvx > 0) { + if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_hvx, htp_session_vtcm_size(sess), &dst_nrows_per_chunk)) { break; } - n_threads--; + n_hvx--; } - if (n_threads == 0) { + if (n_hvx == 0) { return false; } @@ -2509,8 +2509,8 @@ bool htp_kparams_solve_upscale( const uint32_t chunks_per_slice = (dst_y + dst_nrows_per_chunk - 1) / dst_nrows_per_chunk; const uint32_t total_chunks = total_slices * chunks_per_slice; - n_threads = (std::min)(n_threads, total_chunks > 0 ? total_chunks : 1); - htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_threads); + n_hvx = (std::min)(n_hvx, total_chunks > 0 ? total_chunks : 1); + htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_hvx); const uint32_t slice_dst_bytes = dst_x * dst_y * sizeof(float); const uint32_t chunk_dst_bytes = dst_x * dst_nrows_per_chunk * sizeof(float); @@ -2565,7 +2565,7 @@ bool htp_kparams_solve_upscale( kparams->invscale0 = invscale0; kparams->invscale1 = invscale1; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->chunks_per_group = chunks_per_group; kparams->dst_nrows_per_chunk = dst_nrows_per_chunk; @@ -2609,7 +2609,7 @@ static bool htp_kparams_solve_concat( } const uint32_t type_size = ggml_type_size(dst->type); - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t mdev_count = htp_session_mdev_count(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); @@ -2643,10 +2643,10 @@ static bool htp_kparams_solve_concat( (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense, type_size, total_rows, - n_threads, mdev_count, vtcm_size)) { + n_hvx, mdev_count, vtcm_size)) { kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS; - kparams->n_threads = (uint8_t) n_threads; - kparams->vtcm_size = kparams->u.gather.slice * n_threads; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = kparams->u.gather.slice * n_hvx; return true; } } @@ -2654,7 +2654,7 @@ static bool htp_kparams_solve_concat( const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size); if (dma_strides_ok) { kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR; - kparams->n_threads = 1; + kparams->n_hvx = 1; return true; } @@ -2665,7 +2665,7 @@ static bool htp_kparams_solve_concat( if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok && (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) { - const uint32_t nth = n_threads > 0 ? n_threads : 8; + const uint32_t nth = n_hvx > 0 ? n_hvx : 8; struct htp_concat_transposed_vtcm_layout layout; htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, nth); @@ -2674,7 +2674,7 @@ static bool htp_kparams_solve_concat( } kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED; - kparams->n_threads = nth; + kparams->n_hvx = nth; kparams->vtcm_size = layout.total_bytes; kparams->u.transposed.div_ne2 = init_fastdiv_values((uint32_t) dst->ne[2]); return true; @@ -2777,14 +2777,14 @@ static bool htp_kparams_solve_cpy_transpose( m++; } - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); uint64_t vtcm = 0; for (uint32_t blk_bytes = HTP_COPY_TRANSPOSE_BLK_BYTES; ; blk_bytes /= 2) { if (!htp_copy_transpose_layout_build(&tp, blk_bytes)) { return false; } - vtcm = (uint64_t) n_threads * 2 * (tp.in_buf_size + tp.out_buf_size); + vtcm = (uint64_t) n_hvx * 2 * (tp.in_buf_size + tp.out_buf_size); if (vtcm <= vtcm_size) { break; } @@ -2800,7 +2800,7 @@ static bool htp_kparams_solve_cpy_transpose( tp.div_ne2 = init_fastdiv_values(tp.ne[2]); kparams->kernel_type = HTP_COPY_KERNEL_TRANSPOSE; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->vtcm_size = (uint32_t) vtcm; kparams->u.transpose = tp; return true; @@ -2872,7 +2872,7 @@ static bool htp_kparams_solve_cpy( const bool src_is_contiguous = ggml_is_contiguous(src0); const bool dst_is_contiguous = ggml_is_contiguous(dst); - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t mdev_count = htp_session_mdev_count(sess); const size_t vtcm_size = htp_session_vtcm_size(sess); @@ -2895,10 +2895,10 @@ static bool htp_kparams_solve_cpy( (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], 0, 0, false, src_type_size, total_rows, - n_threads, mdev_count, vtcm_size)) { + n_hvx, mdev_count, vtcm_size)) { kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS; - kparams->n_threads = (uint8_t) n_threads; - kparams->vtcm_size = kparams->u.gather.slice * n_threads; + kparams->n_hvx = (uint8_t) n_hvx; + kparams->vtcm_size = kparams->u.gather.slice * n_hvx; return true; } } @@ -2914,7 +2914,7 @@ static bool htp_kparams_solve_cpy( } kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE; - kparams->n_threads = n_threads > 0 ? (uint8_t) n_threads : 4; + kparams->n_hvx = n_hvx > 0 ? (uint8_t) n_hvx : 4; kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]); kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0])); kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0])); @@ -2936,7 +2936,7 @@ static bool htp_kparams_solve_cpy( return false; } - const uint32_t nth = n_threads > 0 ? n_threads : 4; + const uint32_t nth = n_hvx > 0 ? n_hvx : 4; struct htp_copy_convert_vtcm_layout layout; htp_copy_convert_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, nth); @@ -2946,7 +2946,7 @@ static bool htp_kparams_solve_cpy( kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT; kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); - kparams->n_threads = (uint8_t) nth; + kparams->n_hvx = (uint8_t) nth; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->u.convert.blk_rows = layout.blk_rows; kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); @@ -3046,8 +3046,8 @@ static bool htp_kparams_solve_im2col( is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1); } - const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8; - const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads; + const uint32_t max_hvx = (sess && htp_session_n_hvx(sess) > 0) ? htp_session_n_hvx(sess) : 8; + const uint32_t n_hvx = (total_rows < max_hvx) ? total_rows : max_hvx; const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0; const uint32_t row_size = (uint32_t) dst->nb[2]; const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; @@ -3060,12 +3060,12 @@ static bool htp_kparams_solve_im2col( const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_PATCHEMBED_DMA, KH, patch_stride)); if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->src_row_bytes = src_row_bytes; kparams->dst_row_bytes = dst_row_bytes; @@ -3113,7 +3113,7 @@ static bool htp_kparams_solve_im2col( // threads work on output-col blocks, so a single output row (1D) can use them all const uint32_t nbpr = (OW + owb - 1) / owb; const uint64_t nblocks = (uint64_t) total_rows * nbpr; - const uint32_t blk_threads = (uint32_t) (nblocks < max_threads ? nblocks : max_threads); + const uint32_t blk_threads = (uint32_t) (nblocks < max_hvx ? nblocks : max_hvx); struct htp_im2col_vtcm_layout layout; htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, blk_threads, @@ -3121,7 +3121,7 @@ static bool htp_kparams_solve_im2col( if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA; - kparams->n_threads = (uint8_t) blk_threads; + kparams->n_hvx = (uint8_t) blk_threads; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->src_row_bytes = src_row_bytes; kparams->dst_row_bytes = dst_row_bytes; @@ -3232,8 +3232,8 @@ static bool htp_kparams_solve_im2col_3d( p0 == 0 && p1 == 0 && p2 == 0 && d0 == 1 && d1 == 1 && d2 == 1); - const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8; - const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads; + const uint32_t max_hvx = (sess && htp_session_n_hvx(sess) > 0) ? htp_session_n_hvx(sess) : 8; + const uint32_t n_hvx = (total_rows < max_hvx) ? total_rows : max_hvx; const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0; const uint32_t row_size = (uint32_t) dst->nb[2]; const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; @@ -3246,11 +3246,11 @@ static bool htp_kparams_solve_im2col_3d( const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256); + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, 256); if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->src_row_bytes = src_row_bytes; kparams->dst_row_bytes = dst_row_bytes; @@ -3296,11 +3296,11 @@ static bool htp_kparams_solve_im2col_3d( const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256); + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_hvx, 256); if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->vtcm_size = (uint32_t) layout.total_bytes; kparams->src_row_bytes = src_row_bytes; kparams->dst_row_bytes = dst_row_bytes; @@ -3355,16 +3355,16 @@ static bool htp_kparams_solve_pad( const uint32_t type_size = (uint32_t) ggml_type_size(src0->type); const uint32_t total_dst_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); - const uint32_t max_threads = (std::max)(1u, total_dst_rows); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_dst_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); struct htp_pad_vtcm_layout layout; - htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_threads); + htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_hvx); const bool use_dma = (src0->nb[0] == type_size) && (src0->ne[0] >= 512) && (htp_session_vtcm_size(sess) >= layout.total_bytes); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_rows = total_dst_rows; kparams->use_dma = use_dma ? 1 : 0; kparams->elem_size = type_size; @@ -3407,15 +3407,15 @@ static bool htp_kparams_solve_cumsum( } const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); - const uint32_t max_threads = (std::max)(1u, total_rows); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); struct htp_cumsum_vtcm_layout layout; - htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_threads); + htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_hvx); const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_rows = total_rows; kparams->use_dma = use_dma ? 1 : 0; kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0; @@ -3425,17 +3425,17 @@ static bool htp_kparams_solve_cumsum( // Shrink until the layout fits: halve tile_cols down to 1024 columns, then drop threads. False below the VTCM floor. static bool htp_top_k_mask_search(struct htp_top_k_mask_vtcm_layout * L, const struct htp_top_k_mask_shape * S, uint32_t kernel_type, - size_t vtcm_size, uint32_t * n_threads, uint32_t * tile_cols) { + size_t vtcm_size, uint32_t * n_hvx, uint32_t * tile_cols) { const uint32_t min_cols = 1024; while (true) { - htp_top_k_mask_vtcm_layout_build(L, S, kernel_type, *n_threads, *tile_cols); + htp_top_k_mask_vtcm_layout_build(L, S, kernel_type, *n_hvx, *tile_cols); if (L->total_bytes <= vtcm_size) { return true; } if (*tile_cols > min_cols) { *tile_cols = (std::max)(min_cols, (uint32_t) hex_round_up(*tile_cols / 2, 64)); - } else if (*n_threads > 1) { - (*n_threads)--; + } else if (*n_hvx > 1) { + (*n_hvx)--; } else { return false; } @@ -3514,11 +3514,11 @@ static bool htp_kparams_solve_top_k_mask( } // PICKS for one row with a resident mask when its single row tile fits, ROWS otherwise - const uint32_t sess_threads = (std::max)(1u, htp_session_n_threads(sess)); + const uint32_t sess_threads = (std::max)(1u, htp_session_n_hvx(sess)); const size_t vtcm_size = htp_session_vtcm_size(sess); uint32_t kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS; - uint32_t n_threads = sess_threads; + uint32_t n_hvx = sess_threads; uint32_t tile_cols = 0; struct htp_top_k_mask_vtcm_layout L; @@ -3526,14 +3526,14 @@ static bool htp_kparams_solve_top_k_mask( if (total_rows == 1 && S.mask_resident && sess_threads > 1) { kernel_type = HTP_TOP_K_MASK_KERNEL_PICKS; tile_cols = hex_round_up(S.n_kv, 64); - htp_top_k_mask_vtcm_layout_build(&L, &S, kernel_type, n_threads, tile_cols); + htp_top_k_mask_vtcm_layout_build(&L, &S, kernel_type, n_hvx, tile_cols); found = L.total_bytes <= vtcm_size; } if (!found) { kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS; - n_threads = (std::max)(1u, (std::min)(sess_threads, total_rows)); + n_hvx = (std::max)(1u, (std::min)(sess_threads, total_rows)); tile_cols = hex_round_up(S.n_kv, 64); - found = htp_top_k_mask_search(&L, &S, kernel_type, vtcm_size, &n_threads, &tile_cols); + found = htp_top_k_mask_search(&L, &S, kernel_type, vtcm_size, &n_hvx, &tile_cols); } if (!found) { return false; @@ -3548,7 +3548,7 @@ static bool htp_kparams_solve_top_k_mask( } const auto units = [](uint64_t bytes) { return (uint16_t) (bytes / HTP_TOP_K_MASK_UNIT); }; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->kernel_type = (uint8_t) kernel_type; kparams->flags = (dst->type == GGML_TYPE_F16 ? HTP_TOP_K_MASK_F16 : 0) | (score ? HTP_TOP_K_MASK_HAS_SCORE : 0) | @@ -3581,8 +3581,8 @@ static bool htp_kparams_solve_top_k_mask( if (kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS) { // per thread slices of the static copies and of the flag clear, a multiple of the unit - const auto slice = [&](uint64_t n) { return units(hex_round_up((n + n_threads - 1) / n_threads, HTP_TOP_K_MASK_UNIT)); }; - kparams->u.picks.picks_per_thread = (uint32_t) hex_round_up((S.n_top + n_threads - 1) / n_threads, 32); + const auto slice = [&](uint64_t n) { return units(hex_round_up((n + n_hvx - 1) / n_hvx, HTP_TOP_K_MASK_UNIT)); }; + kparams->u.picks.picks_per_hvx = (uint32_t) hex_round_up((S.n_top + n_hvx - 1) / n_hvx, 32); kparams->u.picks.blocks_slice_u = slice(L.blocks_bytes); kparams->u.picks.top_slice_u = slice((uint64_t) S.n_top * 4); kparams->u.picks.extra_slice_u = slice((uint64_t) S.n_extra * 4); @@ -3633,12 +3633,12 @@ static bool htp_kparams_solve_fill( const uint32_t nb1 = (uint32_t) dst->nb[1]; const uint32_t total_rows = ne1 * ne2 * ne3; - const uint32_t max_threads = (std::max)(1u, total_rows); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); const uint32_t elem_size = (uint32_t) ggml_type_size(dst->type); const uint32_t rows_per_chunk = (nb1 > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(nb1, HEX_L2_LINE_SIZE)) : 1; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_rows = total_rows; kparams->elem_size = elem_size; kparams->kernel_type = ggml_is_contiguous(dst) ? HTP_FILL_KERNEL_CONTIGUOUS : HTP_FILL_KERNEL_STRIDED; @@ -3685,10 +3685,10 @@ static bool htp_kparams_solve_roll( } const uint32_t total_rows = ne1 * ne2 * ne3; - const uint32_t max_threads = (std::max)(1u, total_rows); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_rows); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_rows = total_rows; kparams->i0_src0 = i0_src0; kparams->n0 = ne0 - i0_src0; @@ -3731,8 +3731,8 @@ static bool htp_kparams_solve_diag( } const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]); - const uint32_t max_threads = (std::max)(1u, total_batches); - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t max_hvx = (std::max)(1u, total_batches); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), max_hvx); const uint32_t dst_batch_size = (uint32_t) (dst->ne[1] * dst->nb[1]); bool can_split = (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !ggml_is_permuted(dst); @@ -3753,11 +3753,11 @@ static bool htp_kparams_solve_diag( const uint32_t dst_row_size = (uint32_t) (dst->ne[0] * sizeof(float)); struct htp_diag_vtcm_layout layout; - htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads); + htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_hvx); const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes); - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_batches = total_batches; kparams->use_dma = use_dma ? 1 : 0; kparams->can_split = can_split ? 1 : 0; @@ -3788,19 +3788,19 @@ static bool htp_kparams_solve_solve_tri( return false; } - const uint32_t max_threads_sess = htp_session_n_threads(sess); + const uint32_t max_hvx_sess = htp_session_n_hvx(sess); const uint32_t k = (uint32_t) src1->ne[0]; const uint32_t col_block = 32; const uint32_t k_chunks = (k + col_block - 1) / col_block; const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]); - const bool batched = total_batches >= max_threads_sess; + const bool batched = total_batches >= max_hvx_sess; const uint32_t total_jobs = batched ? total_batches : (total_batches * k_chunks); - const uint32_t max_threads = (std::max)(1u, total_jobs); - const uint32_t n_threads = (std::min)(max_threads_sess, max_threads); + const uint32_t max_hvx = (std::max)(1u, total_jobs); + const uint32_t n_hvx = (std::min)(max_hvx_sess, max_hvx); const uint32_t batch_size = (uint32_t) dst->nb[2]; const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->total_batches = total_batches; kparams->total_jobs = total_jobs; kparams->k_chunks = k_chunks; @@ -3877,7 +3877,7 @@ static bool htp_kparams_conv2d_win( const uint32_t halo = moff / 32 + 1; const uint32_t n_rt = (Cout + 31) / 32; const uint64_t n_q = (uint64_t) (N * IHS - (IHS - OH) - 1) * SR + OW; - const uint32_t nth = kp->n_threads; + const uint32_t nth = kp->n_hvx; const size_t tile = 2048; if ((uint64_t) N * IHS * SR + moff + 64 >= (1ull << 31)) { @@ -3900,14 +3900,14 @@ static bool htp_kparams_conv2d_win( } // enough packing units for all threads const uint32_t kpw2 = es == 2 ? 64 : 32; - while (!fold && w_cs > 2 && n_rt * ((Cin + 31) / 32) * (32 / w_cs) < kp->n_threads) { + while (!fold && w_cs > 2 && n_rt * ((Cin + 31) / 32) * (32 / w_cs) < kp->n_hvx) { w_cs /= 2; } wraw = (std::min)(wraw, (size_t) 32 * ((w_cs * T + kpw2 - 1) / kpw2) * 128); wscr = 3 * wraw + 64 * 128 + 128; } else { w_cs = fold ? KW : (std::min)(T, 16u); - while (!fold && w_cs > 1 && n_rt * ((Cin + 31) / 32) * ((T + w_cs - 1) / w_cs) < kp->n_threads) { + while (!fold && w_cs > 1 && n_rt * ((Cin + 31) / 32) * ((T + w_cs - 1) / w_cs) < kp->n_hvx) { w_cs = (w_cs + 1) / 2; } wscr = 2 * (size_t) w_cs * 32 * 128; @@ -3959,7 +3959,7 @@ static bool htp_kparams_conv2d_win( } } - uint32_t n_threads = nth; + uint32_t n_hvx = nth; // Keep the fold schedule, where output traffic can dominate input packing. if (!fold && rt_block < n_rt && ct > 0 && nth > 0) { const uint32_t old_rt = rt_block; @@ -4022,7 +4022,7 @@ static bool htp_kparams_conv2d_win( rt_block = rb; ct = candidate; seg_b = sb; - n_threads = nt; + n_hvx = nt; stage = input_stage; thr = scratch; act = input; @@ -4047,7 +4047,7 @@ static bool htp_kparams_conv2d_win( const uint32_t n_cs = (32 + CS - 1) / CS; kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_WINDOW; - kp->n_threads = (uint8_t) n_threads; + kp->n_hvx = (uint8_t) n_hvx; kp->cwhn = cwhn ? 1 : 0; kp->w_kmajor = !w_rows ? 1 : 0; kp->n_row_tiles = n_rt; @@ -4165,7 +4165,7 @@ static bool htp_kparams_conv2d_tiled( const uint32_t n_owt = (g.OW + 31) / 32; const uint32_t n_ct = g.N * g.OH * n_owt; const uint32_t wk = (std::min)(k_pad, 512u); - const uint32_t max_threads = htp_session_n_threads(sess); + const uint32_t max_hvx = htp_session_n_hvx(sess); const size_t tile = 2048; uint32_t rt_block = n_rt; @@ -4184,7 +4184,7 @@ static bool htp_kparams_conv2d_tiled( // Keep the kernel resident to avoid regenerating the input for each rt block. // Balance patch work per thread and chunk overhead before reducing rt. for (uint32_t rb = n_rt; rb > 0 && ct == 0; rb = rb > 1 ? (rb + 1) / 2 : 0) { - for (uint32_t nt = max_threads; nt > 0; nt /= 2) { + for (uint32_t nt = max_hvx; nt > 0; nt /= 2) { for (uint32_t sg = (std::min)(8u, n_owt); sg > 0; --sg) { // Keep the unused final stride lane for full-vector loads during packing. const uint64_t wb = (uint64_t) 32 * sg * g.s0 + (uint64_t) (g.KW - 1) * g.d0; @@ -4219,11 +4219,11 @@ static bool htp_kparams_conv2d_tiled( if (candidate == 0 || (i > 0 && candidate == candidates[0])) { continue; } - const uint32_t per_thread = (candidate + nt - 1) / nt; - if (sg > per_thread) { + const uint32_t per_hvx = (candidate + nt - 1) / nt; + if (sg > per_hvx) { continue; } - const size_t work = (size_t) (n_ct / candidate) * per_thread + (n_ct % candidate + nt - 1) / nt; + const size_t work = (size_t) (n_ct / candidate) * per_hvx + (n_ct % candidate + nt - 1) / nt; const size_t chunks = n_ct / candidate + (n_ct % candidate != 0); const size_t cost = work * n_kt + c_chunk * chunks; if (cost < best_cost) { @@ -4267,7 +4267,7 @@ static bool htp_kparams_conv2d_tiled( } kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_TILED; - kp->n_threads = (uint8_t) nth; + kp->n_hvx = (uint8_t) nth; kp->cwhn = cwhn ? 1 : 0; kp->w_kmajor = kmajor ? 1 : 0; kp->n_row_tiles = n_rt; @@ -4303,7 +4303,7 @@ static bool htp_kparams_solve_conv2d_unchecked( const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); kparams->total_rows = total_rows; - kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), total_rows); + kparams->n_hvx = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_hvx(sess), total_rows); if (htp_kparams_conv2d_win(sess, dst, kparams)) { return true; @@ -4448,10 +4448,10 @@ static bool htp_kparams_solve_conv2d_dw( const uint32_t w_size = (uint32_t) (kw * kh * c * ggml_type_size(src0->type)); - uint32_t max_threads_sess = (uint32_t) htp_session_n_threads(sess); + uint32_t max_hvx_sess = (uint32_t) htp_session_n_hvx(sess); uint32_t oh_chunk = oh_u; - uint32_t n_threads = max_threads_sess; + uint32_t n_hvx = max_hvx_sess; struct htp_conv2d_dw_vtcm_layout layout; while (true) { @@ -4459,7 +4459,7 @@ static bool htp_kparams_solve_conv2d_dw( const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128; const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128; - htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_threads); + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_hvx); if (layout.total_bytes <= vtcm_budget && layout.total_bytes > 0) { break; @@ -4467,8 +4467,8 @@ static bool htp_kparams_solve_conv2d_dw( if (oh_chunk > 1) { oh_chunk = (oh_chunk + 1) / 2; - } else if (n_threads > 1) { - n_threads--; + } else if (n_hvx > 1) { + n_hvx--; } else { return false; } @@ -4476,18 +4476,18 @@ static bool htp_kparams_solve_conv2d_dw( const uint32_t n_chunks = (oh_u + oh_chunk - 1) / oh_chunk; const uint32_t total_tasks = cwhn ? (n * n_chunks) : (c * n * n_chunks); - n_threads = (std::min)(n_threads, total_tasks); - if (n_threads == 0) { - n_threads = 1; + n_hvx = (std::min)(n_hvx, total_tasks); + if (n_hvx == 0) { + n_hvx = 1; } const uint32_t max_ih_chunk = (std::min)(h, (oh_chunk - 1) * (uint32_t) s1 + (kh - 1) * (uint32_t) d1 + 1); const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128; const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128; - htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_threads); + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_hvx); kparams->kernel_type = kernel_type; - kparams->n_threads = (uint8_t) n_threads; + kparams->n_hvx = (uint8_t) n_hvx; kparams->cwhn = cwhn ? 1 : 0; kparams->is_f16 = (src0->type == GGML_TYPE_F16) ? 1 : 0; kparams->total_tasks = total_tasks; @@ -4566,7 +4566,7 @@ static bool htp_kparams_solve_conv3d( return false; } kparams->total_rows = (uint32_t) total_rows; - kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), (uint32_t) total_rows); + kparams->n_hvx = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_hvx(sess), (uint32_t) total_rows); return htp_kparams_conv2d_tiled(sess, op, kparams); } @@ -4702,8 +4702,8 @@ static bool htp_kparams_solve_dsv4_hc_pre_impl( if (kparams->total_jobs == 0) { return true; } - kparams->n_threads = (std::min)(htp_session_n_threads(sess), kparams->total_jobs); - kparams->jobs_per_thread = (kparams->total_jobs + kparams->n_threads - 1) / kparams->n_threads; + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), kparams->total_jobs); + kparams->jobs_per_hvx = (kparams->total_jobs + kparams->n_hvx - 1) / kparams->n_hvx; const size_t vtcm_budget = htp_session_vtcm_size(sess); const uint32_t n_rows = gated ? HTP_DSV4_HC_PRE_GATED_ROWS : HTP_DSV4_HC_PRE_PLAIN_ROWS; @@ -4711,7 +4711,7 @@ static bool htp_kparams_solve_dsv4_hc_pre_impl( while (block_chunks > 16) { const uint32_t stride = block_chunks * 128; const uint32_t buf_sz = n_rows * stride; - if (2 * buf_sz * kparams->n_threads <= vtcm_budget) { + if (2 * buf_sz * kparams->n_hvx <= vtcm_budget) { break; } block_chunks /= 2; @@ -4720,13 +4720,13 @@ static bool htp_kparams_solve_dsv4_hc_pre_impl( kparams->dma_block_chunks = block_chunks; kparams->dma_row_stride = block_chunks * 128; kparams->dma_buffer_size = n_rows * kparams->dma_row_stride; - kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_threads; + kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_hvx; if (kparams->vtcm_size > vtcm_budget) { return false; } kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks); - kparams->div_n_threads = init_fastdiv_values(kparams->n_threads); + kparams->div_n_hvx = init_fastdiv_values(kparams->n_hvx); return true; } @@ -4906,15 +4906,15 @@ static bool htp_kparams_solve_dsv4_hc_post_impl( if (kparams->total_jobs == 0) { return true; } - kparams->n_threads = (std::min)(htp_session_n_threads(sess), kparams->total_jobs); - kparams->jobs_per_thread = (kparams->total_jobs + kparams->n_threads - 1) / kparams->n_threads; + kparams->n_hvx = (std::min)(htp_session_n_hvx(sess), kparams->total_jobs); + kparams->jobs_per_hvx = (kparams->total_jobs + kparams->n_hvx - 1) / kparams->n_hvx; const size_t vtcm_budget = htp_session_vtcm_size(sess); uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks); while (block_chunks > 16) { const uint32_t stride = block_chunks * 128; const uint32_t buf_sz = HTP_DSV4_HC_POST_ROWS * stride; - if (2 * buf_sz * kparams->n_threads <= vtcm_budget) { + if (2 * buf_sz * kparams->n_hvx <= vtcm_budget) { break; } block_chunks /= 2; @@ -4923,13 +4923,13 @@ static bool htp_kparams_solve_dsv4_hc_post_impl( kparams->dma_block_chunks = block_chunks; kparams->dma_row_stride = block_chunks * 128; kparams->dma_buffer_size = HTP_DSV4_HC_POST_ROWS * kparams->dma_row_stride; - kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_threads; + kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_hvx; if (kparams->vtcm_size > vtcm_budget) { return false; } kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks); - kparams->div_n_threads = init_fastdiv_values(kparams->n_threads); + kparams->div_n_hvx = init_fastdiv_values(kparams->n_hvx); return true; } @@ -5072,12 +5072,12 @@ static bool htp_kparams_solve_lightning_indexer_impl( kparams->div_m_ne3 = init_fastdiv_values((uint32_t) m->ne[3]); if (opt_li_select >= 2 && htp_session_n_hmx(sess) > 0 && D % HMX_FP16_TILE_N_COLS == 0) { - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); const uint32_t n_stream = q->ne[3]; const uint32_t n_dot = D / HMX_FP16_TILE_N_COLS; auto vtcm_usage = [&](uint32_t n_tok, uint32_t kv_block, uint32_t n_qsets) { - return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_threads, k_row_size); + return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_hvx, k_row_size); }; uint32_t n_tok = (std::min)(n_tokens, (uint32_t) HTP_LID_HMX_TOKENS_MAX); @@ -5118,10 +5118,10 @@ static bool htp_kparams_solve_lightning_indexer_impl( if (vtcm_size <= htp_session_vtcm_size(sess)) { const uint32_t n_tt = (n_tokens + n_tok - 1) / n_tok; kparams->kernel_type = HTP_LID_KERNEL_HMX; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->n_tokens_tile = n_tok; kparams->n_kv_blocks = (n_rows + kv_block - 1) / kv_block; - kparams->blocks_per_thread = 0; + kparams->blocks_per_hvx = 0; kparams->kv_block = kv_block; kparams->n_qsets = n_qsets; kparams->n_qbuf = n_qbuf; @@ -5135,22 +5135,22 @@ static bool htp_kparams_solve_lightning_indexer_impl( } } - const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), n_kv_blocks); + const uint32_t n_hvx = (std::min)(htp_session_n_hvx(sess), n_kv_blocks); uint32_t n_tok = (std::min)((uint32_t) q->ne[2], (uint32_t) HTP_LID_TOKENS_MAX); - while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size) > htp_session_vtcm_size(sess)) { + while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_hvx, k_row_size) > htp_session_vtcm_size(sess)) { n_tok /= 2; } - const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size); + const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_hvx, k_row_size); if (vtcm_size > htp_session_vtcm_size(sess)) { return false; } kparams->kernel_type = HTP_LID_KERNEL_HVX; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->n_tokens_tile = n_tok; kparams->n_kv_blocks = n_kv_blocks; - kparams->blocks_per_thread = (n_kv_blocks + n_threads - 1) / n_threads; + kparams->blocks_per_hvx = (n_kv_blocks + n_hvx - 1) / n_hvx; kparams->kv_block = HTP_LID_KV_BLOCK; kparams->n_qsets = 1; kparams->n_qbuf = 1; @@ -5186,7 +5186,7 @@ static inline bool htp_kparams_invoke( } void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & node) { - const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hvx = htp_session_n_hvx(sess); if (htp_op_is_unary(node.opcode) && htp_unary_is_elementwise(node.opcode)) { if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) { @@ -5199,7 +5199,7 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & struct ggml_tensor src0_c = *node.inputs[0]; struct ggml_tensor dst_c = *node.outputs[0]; struct ggml_tensor * tensors[2] = { &src0_c, &dst_c }; - htp_collapse_dims(tensors, 2, elem_size, n_threads); + htp_collapse_dims(tensors, 2, elem_size, n_hvx); if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { node.inputs[0] = node.add_dummy(src0_c); node.outputs[0] = node.add_dummy(dst_c); @@ -5220,7 +5220,7 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & struct ggml_tensor src1_c = *node.inputs[1]; struct ggml_tensor dst_c = *node.outputs[0]; struct ggml_tensor * tensors[3] = { &src0_c, &src1_c, &dst_c }; - htp_collapse_dims(tensors, 3, elem_size, n_threads); + htp_collapse_dims(tensors, 3, elem_size, n_hvx); if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { node.inputs[0] = node.add_dummy(src0_c); node.inputs[1] = node.add_dummy(src1_c); diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp index a4efa8503..8dffedc4b 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.cpp +++ b/ggml/src/ggml-hexagon/htp-opnode.cpp @@ -157,7 +157,7 @@ static std::string format_kernel_params(const htp_opnode & node) { } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { path = "hvx-tiled"; } - snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", path, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { const auto * kparams = node.kparams(); const char * path = "unknown"; @@ -167,10 +167,11 @@ static std::string format_kernel_params(const htp_opnode & node) { } else if (type == HTP_FA_KERNEL_HVX) { path = "hvx"; } - snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", path, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (htp_op_is_unary(node.opcode)) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->col_tile ? "wide-row" : "row-block", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) { snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]); } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) { @@ -179,41 +180,43 @@ static std::string format_kernel_params(const htp_opnode & node) { snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]); } else if (node.opcode == HTP_OP_SSM_CONV) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->n_t == 1 ? "decode" : "prefill", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_SOFTMAX) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_type, (int) kparams->n_threads, (int) kparams->vtcm_size); + snprintf(str, max_size, "k%d nhvx %u vtcm %u", (int) kparams->kernel_type, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { const auto * kparams = node.kparams(); const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent"; - snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar", - (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s-%s nhvx %u vtcm %u", path, kparams->kda ? "kda" : "scalar", + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_DSV4_HC_PRE) { const auto * kparams = node.kparams(); const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_PRE_KERNEL_GATED ? "gated" : "plain"; - snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str, - (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_DSV4_HC_POST) { const auto * kparams = node.kparams(); const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity"; - snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str, - (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, + (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s tok %u bc %u nth %u vtcm %u", + snprintf(str, max_size, "%s nhvx %u tok %u bc %u vtcm %u", kparams->kernel_type == HTP_LID_KERNEL_HMX ? "hmx" : "hvx", - (unsigned int) kparams->n_tokens_tile, (unsigned int) kparams->kv_block, - (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + (unsigned int) kparams->n_hvx, (unsigned int) kparams->n_tokens_tile, + (unsigned int) kparams->kv_block, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "nhvx %u vtcm %u", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d", + snprintf(str, max_size, "%s nhvx %u nchk %u chk %u vtcm %u", node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort", - (int) kparams->n_threads, (int) kparams->n_chunks, - (int) kparams->chunk_elems, (int) kparams->vtcm_size); + (unsigned int) kparams->n_hvx, (unsigned int) kparams->n_chunks, + (unsigned int) kparams->chunk_elems, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_GET_ROWS) { const auto * kparams = node.kparams(); const char * ktype_str = "unknown"; @@ -222,16 +225,16 @@ static std::string format_kernel_params(const htp_opnode & node) { case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break; case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break; } - snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (htp_op_is_glu(node.opcode)) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "nth %d vtcm %u", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "nhvx %u vtcm %u", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_PAD) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "k%d nth %d vtcm %u", (int) kparams->kernel_type, (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "k%d nhvx %u vtcm %u", (int) kparams->kernel_type, (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_CUMSUM) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->use_dma ? "dma" : "hvx", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_TOP_K_MASK) { const auto * kparams = node.kparams(); const char * ktype_str = "?"; @@ -239,19 +242,19 @@ static std::string format_kernel_params(const htp_opnode & node) { case HTP_TOP_K_MASK_KERNEL_ROWS: ktype_str = "rows"; break; case HTP_TOP_K_MASK_KERNEL_PICKS: ktype_str = "picks"; break; } - snprintf(str, max_size, "%s nth %d tile %u x%u vtcm %u", ktype_str, (int) kparams->n_threads, (unsigned int) kparams->tile_cols, (unsigned int) kparams->n_col_tiles, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u tile %u x%u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->tile_cols, (unsigned int) kparams->n_col_tiles, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_FILL) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (int) kparams->n_threads); + snprintf(str, max_size, "%s nhvx %u", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (unsigned int) kparams->n_hvx); } else if (node.opcode == HTP_OP_ROLL) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "k%d nth %d", (int) kparams->kernel_type, (int) kparams->n_threads); + snprintf(str, max_size, "k%d nhvx %u", (int) kparams->kernel_type, (unsigned int) kparams->n_hvx); } else if (node.opcode == HTP_OP_DIAG) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u vtcm %u", kparams->use_dma ? "dma" : "hvx", (unsigned int) kparams->n_hvx, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_SOLVE_TRI) { const auto * kparams = node.kparams(); - snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads); + snprintf(str, max_size, "%s nhvx %u", kparams->batched ? "batched" : "chunked", (unsigned int) kparams->n_hvx); } else if (node.opcode == HTP_OP_UPSCALE) { const auto * kparams = node.kparams(); const char * ktype_str = "unknown"; @@ -264,7 +267,7 @@ static std::string format_kernel_params(const htp_opnode & node) { case HTP_UPSCALE_KERNEL_BILINEAR_AA: ktype_str = "bilinear-aa"; break; case HTP_UPSCALE_KERNEL_BICUBIC: ktype_str = "bicubic"; break; } - snprintf(str, max_size, "%s nth %d chk %d vtcm %u", ktype_str, (int) kparams->n_threads, (int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size); + snprintf(str, max_size, "%s nhvx %u chk %u vtcm %u", ktype_str, (unsigned int) kparams->n_hvx, (unsigned int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size); } else { snprintf(str, max_size, "----"); } diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c index 7fcc4e866..1b95917ae 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.c +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c @@ -15,7 +15,7 @@ #include "hvx-utils.h" #include "htp-tensor.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "allreduce-ops.h" #include "htp-fence.h" @@ -29,9 +29,9 @@ struct htp_allreduce_context { uint32_t row_size_aligned; uint32_t rank_elem_start; uint32_t rank_nelem; - uint32_t elems_per_thread; + uint32_t elems_per_hvx; uint32_t block_elems; - uint32_t vtcm_size_per_thread; + uint32_t vtcm_size_per_hvx; bool is_row_bcast; uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; uint8_t * dst_spad_base; @@ -39,7 +39,8 @@ struct htp_allreduce_context { }; #define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \ -static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ +static void allreduce_thread_dma_1d_##SUFFIX(const htp_thread_t * th, unsigned int nth, unsigned int ith, \ + void * data) { \ struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ struct htp_ops_context * octx = actx->octx; \ \ @@ -47,19 +48,19 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, const uint32_t n_dsts = actx->n_dsts; \ const uint32_t block_elems = actx->block_elems; \ \ - const uint32_t dr = actx->elems_per_thread; \ + const uint32_t dr = actx->elems_per_hvx; \ const uint32_t ir0 = actx->rank_elem_start + dr * ith; \ const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \ if (ir0 >= ir1) return; \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + struct htp_thread_trace * tr = th->trace; \ + dma_queue * dma_q = th->dma_q; \ \ - const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \ - uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \ - uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_thread_offset) : NULL; \ + const size_t vtcm_hvx_offset = ith * actx->vtcm_size_per_hvx; \ + uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_hvx_offset; \ + uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_hvx_offset) : NULL; \ \ - const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + const size_t spad_half = actx->vtcm_size_per_hvx / 2; \ uint32_t ir_prefetch = ir0; \ int spad_idx = 0; \ \ @@ -72,7 +73,7 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \ } \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + spad_idx * spad_half; \ const dma_addr_t s_ddr = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ } \ @@ -99,13 +100,13 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_pop(dma_q); \ } \ uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \ - uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \ - uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ + uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ uint8_t * r_spad = HAS_ADD ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ HVX_ADD_FN(d_spad, s0_spad, s1_spad, cur_elems); \ for (uint32_t s = 2; s < n_ranks; s++) { \ - uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ HVX_ADD_FN(d_spad, d_spad, ss_spad, cur_elems); \ } \ if (HAS_ADD) { \ @@ -120,7 +121,7 @@ static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \ size_t next_bytes = next_elems * sizeof(TYPE); \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ const dma_addr_t s_next = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), next_bytes, next_bytes, next_bytes, 1); \ } \ @@ -143,7 +144,7 @@ DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1) DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1) #define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \ -static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ +static void allreduce_thread_dma_2d_##SUFFIX(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ struct htp_ops_context * octx = actx->octx; \ \ @@ -154,19 +155,19 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, const uint32_t row_size_aligned = actx->row_size_aligned; \ const uint32_t row_bytes = ne0 * sizeof(TYPE); \ \ - const uint32_t dr = actx->elems_per_thread; \ + const uint32_t dr = actx->elems_per_hvx; \ const uint32_t r0 = actx->rank_elem_start + dr * ith; \ const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \ if (r0 >= r1) return; \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + struct htp_thread_trace * tr = th->trace; \ + dma_queue * dma_q = th->dma_q; \ \ - const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \ - uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \ - uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_thread_offset)) : NULL; \ + const size_t vtcm_hvx_offset = ith * actx->vtcm_size_per_hvx; \ + uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_hvx_offset; \ + uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_hvx_offset)) : NULL; \ \ - const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + const size_t spad_half = actx->vtcm_size_per_hvx / 2; \ uint32_t r_prefetch = r0; \ int spad_idx = 0; \ \ @@ -178,7 +179,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \ } \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + spad_idx * spad_half; \ const dma_addr_t s_ddr = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \ } \ @@ -204,8 +205,8 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, dma_queue_pop(dma_q); \ } \ uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \ - uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \ - uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ + uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ for (uint32_t row = 0; row < cur_rows; row++) { \ @@ -214,7 +215,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, const uint8_t * s1_row = s1_spad + row * row_size_aligned; \ HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \ for (uint32_t s = 2; s < n_ranks; s++) { \ - const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half + row * row_size_aligned; \ + const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half + row * row_size_aligned; \ HVX_ADD_FN(d_row, d_row, ss_row, ne0); \ } \ if (HAS_ADD) { \ @@ -230,7 +231,7 @@ static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, if (r_prefetch < r1) { \ uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \ for (uint32_t s = 0; s < n_ranks; s++) { \ - uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_hvx_offset + comp_spad_idx * spad_half; \ const dma_addr_t s_next = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \ } \ @@ -259,19 +260,19 @@ static int validate_allreduce( const struct htp_allreduce_kernel_params * kparams, uint32_t n_ranks ) { - if (kparams->n_threads == 0 || (uint32_t) kparams->n_threads > octx->ctx->n_threads) { + if (kparams->n_hvx == 0 || (uint32_t) kparams->n_hvx > octx->ctx->n_hvx) { return HTP_STATUS_INVAL_PARAMS; } - octx->n_threads = (uint32_t) kparams->n_threads; + octx->n_hvx = (uint32_t) kparams->n_hvx; - if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) { + if (kparams->vtcm_size_per_hvx <= 0 || kparams->vtcm_size <= 0) { return HTP_STATUS_INVAL_PARAMS; } const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count( - n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0); - const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread; + n_ranks, octx->n_hvx, has_add, kparams->is_row_bcast != 0); + const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_hvx; if (vtcm_size != (size_t) kparams->vtcm_size) { return HTP_STATUS_INVAL_PARAMS; } @@ -360,10 +361,10 @@ int op_allreduce(struct htp_ops_context * octx) { // 2. Multi-threaded Reduction across assigned rank chunk if (nelem > 0) { - const uint32_t n_threads = (uint32_t) kparams->n_threads; + const uint32_t n_hvx = (uint32_t) kparams->n_hvx; const uint32_t block_elems = (uint32_t) kparams->block_elems; - const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread; - const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread; + const uint32_t elems_per_hvx = (uint32_t) kparams->elems_per_hvx; + const uint32_t vtcm_size_per_hvx = (uint32_t) kparams->vtcm_size_per_hvx; struct htp_allreduce_context actx; actx.octx = octx; @@ -375,9 +376,9 @@ int op_allreduce(struct htp_ops_context * octx) { actx.row_size_aligned = (uint32_t) kparams->row_size_aligned; actx.rank_elem_start = (uint32_t) kparams->rank_elem_start; actx.rank_nelem = (uint32_t) kparams->rank_nelem; - actx.elems_per_thread = elems_per_thread; + actx.elems_per_hvx = elems_per_hvx; actx.block_elems = block_elems; - actx.vtcm_size_per_thread = vtcm_size_per_thread; + actx.vtcm_size_per_hvx = vtcm_size_per_hvx; actx.is_row_bcast = (kparams->is_row_bcast != 0); work_queue_func_t reduce_fun = NULL; @@ -409,13 +410,13 @@ int op_allreduce(struct htp_ops_context * octx) { uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; for (uint32_t s = 0; s < n_ranks; s++) { actx.src_spad_base[s] = vtcm_ptr; - vtcm_ptr += n_threads * vtcm_size_per_thread; + vtcm_ptr += n_hvx * vtcm_size_per_hvx; } actx.dst_spad_base = vtcm_ptr; - vtcm_ptr += n_threads * vtcm_size_per_thread; + vtcm_ptr += n_hvx * vtcm_size_per_hvx; if (has_add) { actx.res_spad_base = vtcm_ptr; - vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread; + vtcm_ptr += (actx.is_row_bcast ? 1 : n_hvx) * vtcm_size_per_hvx; } if (has_add && actx.is_row_bcast) { @@ -426,7 +427,7 @@ int op_allreduce(struct htp_ops_context * octx) { dma_queue_pop(dma_q); } - work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, reduce_fun, &actx, n_hvx); } // 4. Exit Barrier: Synchronize all ranks after writing diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h index 1264c1cbb..1fb486cb6 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h @@ -24,21 +24,21 @@ enum htp_allreduce_mode { static inline size_t htp_allreduce_vtcm_buffer_count( uint32_t n_ranks, - uint32_t n_threads, + uint32_t n_hvx, bool has_add, bool is_row_bcast ) { - return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0); + return (size_t) (n_ranks + 1) * n_hvx + (has_add ? (is_row_bcast ? 1 : n_hvx) : 0); } struct htp_allreduce_kernel_params { uint8_t kernel_type; int32_t rank; int32_t n_ranks; - int32_t n_threads; + int32_t n_hvx; int32_t block_elems; // 1D: block_elems, 2D: block_rows - int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread - int32_t vtcm_size_per_thread; + int32_t elems_per_hvx; // 1D: nelem_per_hvx, 2D: nrows_per_hvx + int32_t vtcm_size_per_hvx; int32_t vtcm_size; int32_t ne0; int32_t ne1; diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index 9351a73f2..af2605954 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -463,10 +463,10 @@ struct htp_sort_chunk_ctx { size_t phase1_slot_size; float * merge_values; int32_t * merge_indices; - uint32_t real_count[HTP_MAX_NTHREADS]; + uint32_t real_count[HTP_OP_MAX_NHVX]; }; -static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { +static void htp_sort_chunk_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_sort_chunk_ctx * cctx = (struct htp_sort_chunk_ctx *) data; struct htp_ops_context * octx = cctx->octx; const struct htp_sort_kernel_params * kparams = cctx->kparams; @@ -487,7 +487,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { float * merge_values = cctx->merge_values; int32_t * merge_indices = cctx->merge_indices; - struct htp_thread_trace * tr = &octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const dma_addr_t src_addr = src0->data + (size_t) chunk_base * sizeof(float); if (kparams->is_top_k) { @@ -503,7 +503,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { } if (local_k == 1 && ne00 >= 128*1024) { - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); float max_value; @@ -515,7 +515,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { } if (local_k > 1 && local_k <= 64 && chunk_elems > 1024) { - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); top_k_scan(values_buf, real_count, local_k, &merge_values[i * local_k], &merge_indices[i * local_k]); @@ -526,7 +526,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { return; } - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); if (chunk_elems > real_count) { @@ -542,7 +542,7 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); } else { if (real_count > 0) { - dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + dma_sync_read(th->dma_q, values_buf, src_addr, real_count * sizeof(float)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); init_indices_ramp(indices_buf, real_count, chunk_base); @@ -562,7 +562,7 @@ struct htp_argsort_merge_l1_ctx { bool asc; }; -static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data) { +static void htp_argsort_merge_l1_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_argsort_merge_l1_ctx * mctx = (struct htp_argsort_merge_l1_ctx *) data; struct htp_sort_chunk_ctx * cctx = mctx->cctx; const struct htp_sort_kernel_params * kparams = cctx->kparams; @@ -585,7 +585,7 @@ static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data float * out_val = cctx->merge_values + out_offset; int32_t * out_idx = cctx->merge_indices + out_offset; - struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); @@ -596,7 +596,7 @@ struct htp_argsort_merge_l2_ctx { bool asc; }; -static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data) { +static void htp_argsort_merge_l2_job(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct htp_argsort_merge_l2_ctx * mctx = (struct htp_argsort_merge_l2_ctx *) data; struct htp_sort_chunk_ctx * cctx = mctx->cctx; const struct htp_sort_kernel_params * kparams = cctx->kparams; @@ -619,7 +619,7 @@ static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data float * out_val = (float *) slot_base; int32_t * out_idx = (int32_t *) (slot_base + 4 * chunk_elems * sizeof(float)); - struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); @@ -632,7 +632,7 @@ struct htp_sort_multi_row_ctx { size_t phase1_slot_size; uint32_t row_start; uint32_t row_end; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; }; struct htp_tensor_row_coord { @@ -721,7 +721,7 @@ static inline void compute_row_sort( } } -static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data) { +static void htp_sort_multi_row_job(const htp_thread_t * th, unsigned int n, unsigned int ith, void * data) { struct htp_sort_multi_row_ctx * mctx = (struct htp_sort_multi_row_ctx *) data; struct htp_ops_context * octx = mctx->octx; const struct htp_sort_kernel_params * kparams = mctx->kparams; @@ -742,21 +742,21 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data uint32_t row_start = mctx->row_start; uint32_t row_end = mctx->row_end; - uint32_t r_start = row_start + ith * mctx->nrows_per_thread; - uint32_t r_end = MIN(r_start + mctx->nrows_per_thread, row_end); + uint32_t r_start = row_start + ith * mctx->nrows_per_hvx; + uint32_t r_end = MIN(r_start + mctx->nrows_per_hvx, row_end); if (r_start >= r_end) return; size_t slot_size = mctx->phase1_slot_size; size_t spad_offset = ith * slot_size * (size_t) kparams->n_slots; - uint8_t * thread_spad = mctx->phase1_base + spad_offset; + uint8_t * hvx_spad = mctx->phase1_base + spad_offset; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - float * values_buf = (float *) thread_spad; - int32_t * indices_buf = (int32_t *) (thread_spad + chunk_elems * sizeof(float)); + float * values_buf = (float *) hvx_spad; + int32_t * indices_buf = (int32_t *) (hvx_spad + chunk_elems * sizeof(float)); - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; bool is_top_k = kparams->is_top_k != 0; uint32_t k = dst->ne[0]; @@ -768,7 +768,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data if (kparams->n_slots == 2) { struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); for (uint32_t r = r_start, spad_idx = 0; r < r_end && spad_idx < 2; r++, spad_idx++) { - uint8_t * cur_spad = thread_spad + spad_idx * slot_size; + uint8_t * cur_spad = hvx_spad + spad_idx * slot_size; float * cur_values = (float *) cur_spad; int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float)); @@ -791,7 +791,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data for (uint32_t r = r_start; r < r_end; r++) { uint32_t cur_slot = (r - r_start) & 1; - uint8_t * cur_spad = thread_spad + cur_slot * slot_size; + uint8_t * cur_spad = hvx_spad + cur_slot * slot_size; float * cur_values = (float *) cur_spad; int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float)); @@ -876,7 +876,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { const struct htp_sort_kernel_params * kparams = (const struct htp_sort_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -886,7 +886,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { (uint32_t) kparams->chunk_elems, (uint32_t) kparams->n_chunks, (uint32_t) kparams->merge_elems, - (uint32_t) kparams->n_threads, + (uint32_t) kparams->n_hvx, (uint32_t) kparams->n_slots); if (octx->ctx->vtcm_size < layout.total_bytes) { @@ -909,7 +909,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { cctx.merge_indices = merge_indices; memset(cctx.real_count, 0, sizeof(cctx.real_count)); - work_queue_run(octx->ctx->work_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks); + work_queue_run(octx->ctx->hvx_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks); if (is_top_k) { uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; @@ -952,10 +952,10 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { bool asc = kparams->order == GGML_SORT_ORDER_ASC; if (kparams->n_chunks == 8) { struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 4); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l1_job, &l1ctx, 4); struct htp_argsort_merge_l2_ctx l2ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l2_job, &l2ctx, 2); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l2_job, &l2ctx, 2); size_t slot_size = layout.phase1_slot_size; uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; @@ -976,7 +976,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); } else if (kparams->n_chunks == 4) { struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc }; - work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 2); + work_queue_run(octx->ctx->hvx_queue, htp_argsort_merge_l1_job, &l1ctx, 2); uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; uint32_t n01 = cctx.real_count[0] + cctx.real_count[1]; @@ -1031,13 +1031,13 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { if (rows_per_chunk == 0) { rows_per_chunk = 1; } - uint32_t dr = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + uint32_t dr = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); if (rows_per_chunk > 1) { dr = hex_round_up(dr, rows_per_chunk); } - mctx.nrows_per_thread = dr; + mctx.nrows_per_hvx = dr; - work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, htp_sort_multi_row_job, &mctx, octx->n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.h b/ggml/src/ggml-hexagon/htp/argsort-ops.h index f74189fba..d1cb8087b 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.h +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.h @@ -10,7 +10,7 @@ #include "htp-vtcm.h" struct htp_sort_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1) uint32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT uint32_t chunk_elems; @@ -31,7 +31,7 @@ struct htp_sort_vtcm_layout { uint32_t chunk_elems; uint32_t n_chunks; uint32_t merge_elems; - uint32_t n_threads; + uint32_t n_hvx; uint32_t n_slots; }; @@ -41,14 +41,14 @@ static inline void htp_sort_vtcm_layout_build( uint32_t chunk_elems, uint32_t n_chunks, uint32_t merge_elems, - uint32_t n_threads, + uint32_t n_hvx, uint32_t n_slots) { memset(L, 0, sizeof(*L)); L->chunk_elems = chunk_elems; L->n_chunks = n_chunks; L->merge_elems = merge_elems; - L->n_threads = n_threads; + L->n_hvx = n_hvx; L->n_slots = n_slots; const size_t values_size = ((chunk_elems * sizeof(float)) + 127) & ~127; @@ -57,7 +57,7 @@ static inline void htp_sort_vtcm_layout_build( size_t off = 0; if (total_rows > 1) { - VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_threads); + VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_hvx); } else { VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_chunks * n_slots); const size_t merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127; @@ -73,14 +73,14 @@ static inline bool htp_sort_solve_layout( uint32_t ne00, uint32_t total_rows, uint32_t k, - uint32_t n_threads_max, + uint32_t n_hvx_max, size_t vtcm_budget, bool is_top_k) { memset(layout, 0, sizeof(*layout)); if (total_rows > 1) { - uint32_t n_threads = total_rows < n_threads_max ? total_rows : n_threads_max; + uint32_t n_hvx = total_rows < n_hvx_max ? total_rows : n_hvx_max; uint32_t n_vec = (ne00 + 31) / 32; uint32_t n_vec_pow2 = 1; @@ -96,13 +96,13 @@ static inline bool htp_sort_solve_layout( n_slots = 1; } - size_t spad_per_thread = spad_per_slot * n_slots; + size_t spad_per_hvx = spad_per_slot * n_slots; - while (n_threads > 1 && (spad_per_thread * n_threads) > vtcm_budget) { - n_threads--; + while (n_hvx > 1 && (spad_per_hvx * n_hvx) > vtcm_budget) { + n_hvx--; } - htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_threads, n_slots); + htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_hvx, n_slots); return layout->total_bytes <= vtcm_budget; } @@ -112,7 +112,7 @@ static inline bool htp_sort_solve_layout( uint32_t n_chunks = 1; if (ne00 > 1024) { - while (n_chunks * 2 <= n_threads_max && n_chunks * 2 <= n_vec_pow2) { + while (n_chunks * 2 <= n_hvx_max && n_chunks * 2 <= n_vec_pow2) { n_chunks *= 2; } } diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index 48f461e46..a5951556b 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -14,7 +14,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "binary-ops.h" #include "htp-ctx.h" #include "htp-ops.h" @@ -39,7 +39,7 @@ struct htp_binary_context { struct fastdiv_values src1_dim3_div; // ne13 uint32_t block_max; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_rows; uint32_t row_start; uint32_t n_srcs; @@ -260,7 +260,7 @@ static void compute_add_id_f32( } // 1. Scalar src1 in VTCM via DMA (ne10 == 1) -static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_scalar(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; @@ -270,21 +270,21 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data const size_t row_stride = (size_t) ne11 * elem_size; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; const void * s1_table = VTCM_LAYOUT_PTR(const void, bctx->vtcm_base, layout->off_src1); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -309,7 +309,7 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_scalar_t compute = (compute_scalar_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -358,29 +358,29 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data } // 2. Vector Same Shape (ne1x == ne0x) or Simple Broadcast -static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_same_shape(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t src1_spad_half = layout->src1_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t n_srcs = bctx->n_srcs; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -415,7 +415,7 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_same_shape_t compute = (compute_same_shape_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -472,27 +472,27 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, } // 3. Row Broadcast (ne11 == 1, ne12 == 1, single row src1) -static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_row_broadcast(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -517,7 +517,7 @@ static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int it spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_row_bcast_t compute = (compute_row_bcast_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -555,22 +555,22 @@ static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int it } // 4. Vector Complex (ne10 == ne00, complex broadcast) -static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_complex(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); @@ -580,7 +580,7 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride; const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -603,7 +603,7 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_complex_t compute = (compute_complex_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -646,19 +646,19 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voi } // 5. Element Repeat (ne10 != ne00) -static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_element_repeat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); @@ -671,7 +671,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -694,7 +694,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; compute_repeat_t compute = (compute_repeat_t) bctx->compute; for (uint32_t ir = start_row; ir < end_row; ) { @@ -737,7 +737,7 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voi } // 6. ADD_ID (src1 gathered via src2 indices) -static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_add_id_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; @@ -760,19 +760,19 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; const uint32_t row_size_bytes = bctx->row_size_bytes; - const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_hvx, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); const int32_t * vtcm_src2 = VTCM_LAYOUT_PTR(const int32_t, bctx->vtcm_base, layout->off_src2); size_t src0_spad_half = layout->src0_spad_half_size; size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -795,7 +795,7 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); @@ -870,7 +870,7 @@ struct binary_chunked_context { uint32_t elem_start; uint32_t nelem; uint32_t chunk_size; - uint32_t chunks_per_thread; + uint32_t chunks_per_hvx; uint32_t total_chunks; uint32_t n_srcs; bool is_scalar; @@ -881,15 +881,15 @@ struct binary_chunked_context { compute_binary_scalar_chunked_f16_t compute_scalar_f16; }; -static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_chunked(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct binary_chunked_context * ctx = (struct binary_chunked_context *) data; struct htp_ops_context * octx = ctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - const uint32_t start_chunk = ctx->chunks_per_thread * ith; - const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_thread, ctx->total_chunks); + const uint32_t start_chunk = ctx->chunks_per_hvx * ith; + const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_hvx, ctx->total_chunks); if (start_chunk >= end_chunk) { return; } @@ -903,11 +903,11 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat ith, nth, start_chunk, end_chunk, ctx->total_chunks, ctx->nelem); const struct htp_binary_vtcm_layout * layout = &ctx->vtcm_layout; - uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); - uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread)); - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_hvx); + uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_hvx)); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_hvx); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t n_srcs = ctx->n_srcs; uint32_t prefetch_chunk = start_chunk; int spad_idx = 0; @@ -939,7 +939,7 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t c = start_chunk; c < end_chunk; c++) { const uint32_t c_start = ctx->elem_start + c * chunk_size; @@ -1091,7 +1091,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -1103,8 +1103,8 @@ static int execute_op_binary(struct htp_ops_context * octx) { const uint32_t chunk_size = kparams->chunk_size; const uint32_t total_chunks = (nelem + chunk_size - 1) / chunk_size; - const uint32_t n_threads = (total_chunks >= 2) ? MIN(octx->n_threads, total_chunks) : 1; - const uint32_t chunks_per_thread = (total_chunks + n_threads - 1) / n_threads; + const uint32_t n_hvx = (total_chunks >= 2) ? MIN(octx->n_hvx, total_chunks) : 1; + const uint32_t chunks_per_hvx = (total_chunks + n_hvx - 1) / n_hvx; float scalar_f32 = 0.0f; _Float16 scalar_f16 = 0; @@ -1128,7 +1128,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { .elem_start = elem_start, .nelem = nelem, .chunk_size = chunk_size, - .chunks_per_thread = chunks_per_thread, + .chunks_per_hvx = chunks_per_hvx, .total_chunks = total_chunks, .n_srcs = n_srcs, .is_scalar = is_scalar, @@ -1139,7 +1139,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { .compute_scalar_f16 = compute_scalar_f16, }; - work_queue_run(octx->ctx->work_queue, binary_thread_chunked, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, binary_thread_chunked, &cctx, n_hvx); return HTP_STATUS_OK; } @@ -1159,11 +1159,11 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; const size_t src1_row_size_aligned = kparams->src1_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; @@ -1176,7 +1176,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - worker_callback_t worker_func = NULL; + work_queue_func_t worker_func = NULL; void * compute_func = NULL; switch (kparams->kernel_type) { @@ -1336,7 +1336,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { } bctx.octx = octx; - bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + bctx.nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); bctx.total_rows = nrows; bctx.row_start = row_start; bctx.n_srcs = n_srcs; @@ -1375,7 +1375,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { } } - work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker_func, &bctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h index 8c5e683e8..6f849396c 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.h +++ b/ggml/src/ggml-hexagon/htp/binary-ops.h @@ -30,7 +30,7 @@ struct htp_binary_kernel_params { uint8_t is_scalar; uint8_t pad[2]; - uint32_t n_threads; + uint32_t n_hvx; uint32_t rows_per_buffer; uint32_t src0_row_size_aligned; @@ -86,9 +86,9 @@ struct htp_binary_vtcm_layout { size_t off_src2; size_t off_dst; - size_t src0_bytes_per_thread; - size_t src1_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t src1_spad_half_size; @@ -106,8 +106,8 @@ static inline void htp_binary_vtcm_layout_build( ) { memset(L, 0, sizeof(*L)); - const uint32_t n_threads = kparams->n_threads; - if (n_threads == 0) { + const uint32_t n_hvx = kparams->n_hvx; + if (n_hvx == 0) { return; } @@ -119,9 +119,9 @@ static inline void htp_binary_vtcm_layout_build( return; } - L->src0_bytes_per_thread = 2 * chunk_bytes; - L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes); - L->dst_bytes_per_thread = 2 * chunk_bytes; + L->src0_bytes_per_hvx = 2 * chunk_bytes; + L->src1_bytes_per_hvx = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes); + L->dst_bytes_per_hvx = 2 * chunk_bytes; L->src0_spad_half_size = chunk_bytes; L->src1_spad_half_size = kparams->is_scalar ? 0 : chunk_bytes; @@ -130,9 +130,9 @@ static inline void htp_binary_vtcm_layout_build( L->rows_per_buffer = 1; L->src1_size = 0; - const size_t src0_total = n_threads * L->src0_bytes_per_thread; - const size_t src1_total = kparams->is_scalar ? 128 : (n_threads * L->src1_bytes_per_thread); - const size_t dst_total = n_threads * L->dst_bytes_per_thread; + const size_t src0_total = n_hvx * L->src0_bytes_per_hvx; + const size_t src1_total = kparams->is_scalar ? 128 : (n_hvx * L->src1_bytes_per_hvx); + const size_t dst_total = n_hvx * L->dst_bytes_per_hvx; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); @@ -158,7 +158,7 @@ static inline void htp_binary_vtcm_layout_build( const size_t rows_per_buffer = (kparams->rows_per_buffer > 0) ? kparams->rows_per_buffer - : (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total); + : (vtcm_size - static_vtcm_size) / (n_hvx * spad_row_total); if (rows_per_buffer == 0) { return; } @@ -167,24 +167,24 @@ static inline void htp_binary_vtcm_layout_build( L->src1_size = kparams->src1_size; L->src2_size = kparams->src2_size; - L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned; - L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned; - L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + L->src0_bytes_per_hvx = rows_per_buffer * 2 * kparams->src0_row_size_aligned; + L->dst_bytes_per_hvx = rows_per_buffer * 2 * kparams->dst_row_size_aligned; + L->src1_bytes_per_hvx = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ? rows_per_buffer * 2 * (n_srcs - 1) * kparams->src1_row_size_aligned : 0; - L->src0_spad_half_size = L->src0_bytes_per_thread / 2; + L->src0_spad_half_size = L->src0_bytes_per_hvx / 2; L->src1_spad_half_size = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ? rows_per_buffer * kparams->src1_row_size_aligned : 0; - L->dst_spad_half_size = L->dst_bytes_per_thread / 2; + L->dst_spad_half_size = L->dst_bytes_per_hvx / 2; - const size_t src0_total = n_threads * L->src0_bytes_per_thread; + const size_t src0_total = n_hvx * L->src0_bytes_per_hvx; const size_t src1_total = (kparams->src1_size > 0) ? kparams->src1_size - : n_threads * L->src1_bytes_per_thread; + : n_hvx * L->src1_bytes_per_hvx; const size_t src2_total = kparams->src2_size; - const size_t dst_total = n_threads * L->dst_bytes_per_thread; + const size_t dst_total = n_hvx * L->dst_bytes_per_hvx; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); @@ -200,15 +200,15 @@ static inline bool htp_binary_solve_chunked( struct htp_binary_kernel_params * kparams, uint32_t total_elems, uint32_t elem_size, - uint32_t max_threads, + uint32_t max_hvx, uint32_t n_srcs, bool is_scalar, size_t vtcm_size ) { const size_t n_bufs = is_scalar ? 2 : (size_t) n_srcs + 1; const size_t scalar_bytes = is_scalar ? 128 : 0; - const size_t budget_per_buf = (max_threads > 0 && vtcm_size > scalar_bytes) - ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_threads * n_bufs), 128) + const size_t budget_per_buf = (max_hvx > 0 && vtcm_size > scalar_bytes) + ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_hvx * n_bufs), 128) : 0; const uint32_t max_chunk_bytes = (budget_per_buf > 0 && budget_per_buf < 32768) ? (uint32_t) budget_per_buf : 32768u; const uint32_t max_chunk_elems = max_chunk_bytes / elem_size; @@ -217,16 +217,16 @@ static inline bool htp_binary_solve_chunked( } const uint32_t min_chunk_elems = 256 < max_chunk_elems ? 256 : max_chunk_elems; - const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads) - 1) / (2 * max_threads), 32); + const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_hvx) - 1) / (2 * max_hvx), 32); const uint32_t chunk_size = (target_chunk_elems < min_chunk_elems) ? min_chunk_elems : (target_chunk_elems > max_chunk_elems) ? max_chunk_elems : target_chunk_elems; const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size; - const uint32_t n_threads = (total_chunks >= 2) ? (max_threads < total_chunks ? max_threads : total_chunks) : 1; + const uint32_t n_hvx = (total_chunks >= 2) ? (max_hvx < total_chunks ? max_hvx : total_chunks) : 1; kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; - kparams->n_threads = n_threads; + kparams->n_hvx = n_hvx; kparams->n_srcs = (uint8_t) n_srcs; kparams->rows_per_buffer = 1; kparams->src1_row_size_aligned = is_scalar ? 0 : kparams->src0_row_size_aligned; diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index 8f3e199c2..a374a8fd6 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -3,7 +3,7 @@ #include "hex-common.h" #include "dma-copy.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hexagon_protos.h" #include "hexagon_types.h" #include "htp-ctx.h" @@ -21,11 +21,11 @@ struct htp_concat_context { struct htp_ops_context * octx; uint8_t * spad0_base; uint8_t * spad1_base; - uint32_t spad0_size_per_thread; - uint32_t spad1_size_per_thread; + uint32_t spad0_size_per_hvx; + uint32_t spad1_size_per_hvx; uint32_t row_start; uint32_t nrows; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t nplanes; struct fastdiv_values div_ne2; }; @@ -36,7 +36,7 @@ static inline dma_addr_t concat_plane_addr(const struct htp_tensor * t, uint32_t return t->data + i2 * t->nb[2] + i3 * t->nb[3]; } -static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * data) { +static void concat_2d_f32_transposed(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_concat_context * cctx = (struct htp_concat_context *) data; struct htp_ops_context * octx = cctx->octx; @@ -48,12 +48,12 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src1_ne0 = src1->ne[0]; // dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column - const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread; - const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows); + const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_hvx; + const uint32_t row_end = MIN(row_begin + cctx->nrows_per_hvx, cctx->row_start + cctx->nrows); if (row_begin >= row_end) return; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct fastdiv_values * div_ne2 = &cctx->div_ne2; const uint32_t ne2 = dst->ne[2]; @@ -72,8 +72,8 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(float); // two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other - uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread; - uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread; + uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_hvx; + uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_hvx; const uint32_t spad0_slot = block_rows * spad0_stride; const uint32_t spad1_slot = block_cols_padded * spad1_stride; @@ -153,7 +153,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * dma_queue_flush(dma_q); } -static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * data) { +static void concat_2d_f16_transposed(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_concat_context * cctx = (struct htp_concat_context *) data; struct htp_ops_context * octx = cctx->octx; @@ -165,12 +165,12 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src1_ne0 = src1->ne[0]; // dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column - const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread; - const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows); + const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_hvx; + const uint32_t row_end = MIN(row_begin + cctx->nrows_per_hvx, cctx->row_start + cctx->nrows); if (row_begin >= row_end) return; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct fastdiv_values * div_ne2 = &cctx->div_ne2; const uint32_t ne2 = dst->ne[2]; @@ -189,8 +189,8 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(__fp16); // two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other - uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread; - uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread; + uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_hvx; + uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_hvx; const uint32_t spad0_slot = block_rows * spad0_stride; const uint32_t spad1_slot = block_cols_padded * spad1_stride; @@ -311,7 +311,7 @@ static int concat_regular(struct htp_ops_context * octx, int dim, uint32_t type_ } static int concat_transposed(struct htp_ops_context * octx, const struct htp_concat_kernel_params * kparams, uint32_t type_size) { - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -334,10 +334,10 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_concat_transposed_vtcm_layout layout; - htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads); + htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_hvx); uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; @@ -345,16 +345,16 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con cctx.octx = octx; cctx.spad0_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad0); cctx.spad1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad1); - cctx.spad0_size_per_thread = layout.src0_spad_size_per_thread; - cctx.spad1_size_per_thread = layout.src1_spad_size_per_thread; + cctx.spad0_size_per_hvx = layout.src0_spad_size_per_hvx; + cctx.spad1_size_per_hvx = layout.src1_spad_size_per_hvx; cctx.row_start = row_start; cctx.nrows = nrows; cctx.nplanes = dst->ne[2] * dst->ne[3]; cctx.div_ne2 = kparams->u.transposed.div_ne2; - cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + cctx.nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed; - work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker_func, &cctx, n_hvx); return HTP_STATUS_OK; } @@ -377,7 +377,7 @@ int op_concat(struct htp_ops_context * octx) { break; case HTP_CONCAT_KERNEL_GATHER_ROWS: - status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads); + status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_hvx); break; default: diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h index 1f0744434..580fe0e15 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.h +++ b/ggml/src/ggml-hexagon/htp/concat-ops.h @@ -21,7 +21,7 @@ struct htp_concat_transposed_params { struct htp_concat_kernel_params { uint8_t kernel_type; uint8_t dim; - uint8_t n_threads; + uint8_t n_hvx; uint8_t pad; uint32_t vtcm_size; @@ -39,8 +39,8 @@ _Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kerne #endif struct htp_concat_transposed_vtcm_layout { - uint32_t src0_spad_size_per_thread; - uint32_t src1_spad_size_per_thread; + uint32_t src0_spad_size_per_hvx; + uint32_t src1_spad_size_per_hvx; size_t off_spad0; size_t off_spad1; uint32_t total_bytes; @@ -51,7 +51,7 @@ static inline void htp_concat_transposed_vtcm_layout_build( uint32_t src0_ne0, uint32_t src1_ne0, uint32_t type_size, - uint32_t n_threads) { + uint32_t n_hvx) { // src1_ne0 is processed in chunks of block_cols tokens, so VTCM does not grow with the batch uint32_t block_rows = 128; @@ -62,12 +62,12 @@ static inline void htp_concat_transposed_vtcm_layout_build( uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + block_cols_padded * type_size; // double-buffered: two slots per thread - L->src0_spad_size_per_thread = 2 * block_rows * spad0_row_bytes; - L->src1_spad_size_per_thread = 2 * block_cols_padded * spad1_stride; + L->src0_spad_size_per_hvx = 2 * block_rows * spad0_row_bytes; + L->src1_spad_size_per_hvx = 2 * block_cols_padded * spad1_stride; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread); - VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_threads * L->src1_spad_size_per_thread); + VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_hvx * L->src0_spad_size_per_hvx); + VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_hvx * L->src1_spad_size_per_hvx); L->total_bytes = (uint32_t) off; } diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.c b/ggml/src/ggml-hexagon/htp/conv2d-ops.c index 6f35cb19b..67e8ec67b 100644 --- a/ggml/src/ggml-hexagon/htp/conv2d-ops.c +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.c @@ -12,7 +12,7 @@ #include "dma-queue.h" #include "hex-common.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -100,7 +100,7 @@ static void conv2d_mdev_rows(const struct htp_ops_context * octx, uint32_t total } } -static void conv2d_hmx_worker(void * data) { +static void conv2d_hmx_worker(const htp_thread_t * th, void * data) { struct htp_conv2d_hmx_task * task = (struct htp_conv2d_hmx_task *) data; const size_t dot_stride = task->n_dot_tiles * HMX_FP16_TILE_N_ELMS; @@ -205,7 +205,7 @@ static inline uint32_t conv2d_hmx_chunk_range(const struct htp_conv2d_hmx_contex return cs; } -static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_wpack_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; struct htp_ops_context * octx = hc->octx; const struct htp_conv2d_kernel_params * kp = hc->kp; @@ -218,7 +218,7 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d const bool is_f16 = kernel->type == HTP_TYPE_F16; const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size; const size_t w_buf_size = (size_t) WK * 32 * sizeof(__fp16); const size_t w_raw_size = (size_t) WK * 32 * sizeof(float); @@ -232,7 +232,7 @@ static void conv2d_hmx_wpack_thread(unsigned int nth, unsigned int ith, void * d const HVX_Vector zero = Q6_V_vzero(); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t rt = ith; rt < hc->n_rt; rt += nth) { const uint32_t co0 = (hc->rt0 + rt) * 32; @@ -704,7 +704,7 @@ static inline bool conv2d_wpack_kmajor_next_task(struct conv2d_wpack_kmajor_iter } // Kernel stored as [k][cout] (cout contiguous): DMA k rows of 64 cout, transpose into two row tiles -static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_wpack_kmajor_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; struct htp_ops_context * octx = hc->octx; const struct htp_conv2d_kernel_params * kp = hc->kp; @@ -721,7 +721,7 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); const uint32_t n_cb = (hc->n_rt + 1) / 2; - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; const size_t w_raw_size = (size_t) WK * 64 * sizeof(float); uint8_t * wraws[2] = { hc->thr_base + (size_t) ith * kp->thr_size, hc->thr_base + (size_t) ith * kp->thr_size + w_raw_size }; @@ -729,7 +729,7 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v HVX_Vector * scratch = vin + 32; HVX_Vector * sink = scratch + 32; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; struct conv2d_wpack_kmajor_iter it = { .cb = 0, @@ -778,7 +778,7 @@ static void conv2d_hmx_wpack_kmajor_thread(unsigned int nth, unsigned int ith, v } } -static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t ith) { +static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, const htp_thread_t * th, uint32_t ith) { struct htp_ops_context * octx = hc->octx; const struct htp_conv2d_kernel_params * kp = hc->kp; const struct htp_tensor * kernel = octx->src[0]; @@ -790,7 +790,7 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i const uint32_t d0 = (uint32_t) hc->g->d0; const uint32_t n_vec = kp->tiled.n_k_tiles * (HMX_FP16_TILE_N_ELMS / 64); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size; float * stages[2] = { (float *) thr, @@ -808,7 +808,7 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i uint32_t t0, t1; const uint32_t cs = conv2d_hmx_chunk_range(hc, hc->pack_chunk, hc->pack_tpt, ith, &t0, &t1); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; if (t0 >= t1) { return; @@ -866,7 +866,7 @@ static void conv2d_hmx_pack(const struct htp_conv2d_hmx_context * hc, uint32_t i } } -static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t ith) { +static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, const htp_thread_t * th, uint32_t ith) { struct htp_ops_context * octx = hc->octx; const struct htp_conv2d_kernel_params * kp = hc->kp; const struct htp_tensor * dst = octx->dst; @@ -877,7 +877,7 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t const uint32_t n_owt = kp->tiled.n_ow_tiles; const uint32_t OS = kp->tiled.seg_tiles * 32; - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = hc->thr_base + (size_t) ith * kp->thr_size + (size_t) (kp->tiled.n_stage > 1 ? 2 : 1) * kp->stage_size; float * ostage[2] = { (float *) thr, @@ -889,7 +889,7 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t const uint32_t cs = conv2d_hmx_chunk_range(hc, hc->unpack_chunk, hc->unpack_tpt, ith, &t0, &t1); const uint32_t nct = MIN(cs + kp->ct_chunk, hc->t_end) - cs; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; if (t0 >= t1) { return; @@ -945,11 +945,11 @@ static void conv2d_hmx_unpack(const struct htp_conv2d_hmx_context * hc, uint32_t } } -static void conv2d_hmx_phase_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_phase_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_hmx_context * hc = (const struct htp_conv2d_hmx_context *) data; if (hc->unpack_chunk >= 0) { - conv2d_hmx_unpack(hc, ith); + conv2d_hmx_unpack(hc, th, ith); } // weights of this rt block are packed before its first chunk if (hc->pack_chunk == 0) { @@ -960,7 +960,7 @@ static void conv2d_hmx_phase_thread(unsigned int nth, unsigned int ith, void * d } } if (hc->pack_chunk >= 0) { - conv2d_hmx_pack(hc, ith); + conv2d_hmx_pack(hc, th, ith); } } @@ -978,7 +978,7 @@ static void conv2d_hmx_run_phase(struct htp_conv2d_hmx_context * hc, int32_t pac hc->unpack_chunk = unpack; hc->pack_tpt = conv2d_hmx_tpt(hc, pack, nth); hc->unpack_tpt = conv2d_hmx_tpt(hc, unpack, nth); - work_queue_run(hc->octx->ctx->work_queue, conv2d_hmx_phase_thread, hc, nth); + work_queue_run(hc->octx->ctx->hvx_queue, conv2d_hmx_phase_thread, hc, nth); } static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv2d_kernel_params * kp) { @@ -997,7 +997,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv const size_t out_size = (size_t) kp->rt_block * kp->ct_chunk * tile; struct htp_conv2d_vtcm_layout layout; - htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_threads); + htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_hvx); uint8_t * base = (uint8_t *) octx->ctx->vtcm_base; @@ -1015,7 +1015,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv .t_end = (row_base + n_rows) * kp->tiled.n_ow_tiles, }; - const uint32_t nth = kp->n_threads; + const uint32_t nth = kp->n_hvx; const int32_t n_chunks = (int32_t) ((hc.t_end - hc.t_base + kp->ct_chunk - 1) / kp->ct_chunk); struct htp_conv2d_hmx_task task; @@ -1023,7 +1023,7 @@ static int conv2d_hmx_tiled(struct htp_ops_context * octx, const struct htp_conv for (uint32_t rt0 = 0; rt0 < kp->n_row_tiles; rt0 += kp->rt_block) { hc.rt0 = rt0; hc.n_rt = MIN(kp->rt_block, kp->n_row_tiles - rt0); - work_queue_run(octx->ctx->work_queue, kp->w_kmajor ? conv2d_hmx_wpack_kmajor_thread : conv2d_hmx_wpack_thread, &hc, nth); + work_queue_run(octx->ctx->hvx_queue, kp->w_kmajor ? conv2d_hmx_wpack_kmajor_thread : conv2d_hmx_wpack_thread, &hc, nth); conv2d_hmx_run_phase(&hc, 0, -1, nth); @@ -1110,7 +1110,7 @@ static inline uint32_t conv2d_hmx_win_act_off(uint32_t u) { return ((u >> 5) << 11) | ((u & 30) << 6) | ((u & 1) << 1); } -static void conv2d_hmx_win_worker(void * data) { +static void conv2d_hmx_win_worker(const htp_thread_t * th, void * data) { const struct htp_conv2d_win_task * j = (const struct htp_conv2d_win_task *) data; const uint32_t * taps = j->taps; @@ -1327,7 +1327,7 @@ static inline void conv2d_hmx_win_wpack_kmajor_compute(const struct htp_conv2d_w } // Cout-contiguous kernel: DMA w[t][c][32 cout] rows, tile vector p = (row 2p, row 2p+1) interleaved per cout -static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) { +static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { struct htp_ops_context * octx = wc->octx; const struct htp_conv2d_kernel_params * kp = wc->kp; const struct htp_tensor * kernel = octx->src[0]; @@ -1342,13 +1342,13 @@ static void conv2d_hmx_win_wpack_kmajor(const struct htp_conv2d_win_context * wc const bool is_f16 = kernel->type == HTP_TYPE_F16; const uint32_t es = is_f16 ? sizeof(__fp16) : sizeof(float); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; const size_t w_raw_size = (size_t) TG * 32 * VLEN; uint8_t * raws[2] = { wc->thr_base + (size_t) ith * kp->thr_size, wc->thr_base + (size_t) ith * kp->thr_size + w_raw_size }; const HVX_Vector zero = Q6_V_vzero(); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_tg = (T + TG - 1) / TG; const uint32_t n_units = wc->n_rt * n_cc * n_tg; @@ -1491,7 +1491,7 @@ static void conv2d_hmx_win_wrows_pack(const struct htp_conv2d_win_context * wc, } } -static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) { +static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { struct htp_ops_context * octx = wc->octx; const struct htp_conv2d_kernel_params * kp = wc->kp; const struct htp_tensor * kernel = octx->src[0]; @@ -1501,7 +1501,7 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, const uint32_t kpw = kernel->type == HTP_TYPE_F16 ? 64 : 32; // k per vector const size_t wraw = (size_t) 32 * ((CS * T + kpw - 1) / kpw) * VLEN; - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * raws[2] = { wc->thr_base + (size_t) ith * kp->thr_size, wc->thr_base + (size_t) ith * kp->thr_size + wraw }; uint8_t * xr = raws[1] + wraw; HVX_Vector * vin = (HVX_Vector *) (xr + wraw); @@ -1510,7 +1510,7 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, const uint32_t n_cs = (32 + CS - 1) / CS; const uint32_t n_units = wc->n_rt * kp->window.n_cc * n_cs; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; // one DMA per unit: push(N), push(N + 1), pop(N), pack(N), push(N + 2), ... for (uint32_t k = 0, unit = ith; k < 2 && unit < n_units; k++, unit += nth) { @@ -1531,13 +1531,13 @@ static void conv2d_hmx_win_wpack_rows(const struct htp_conv2d_win_context * wc, } } -static void conv2d_hmx_win_wpack_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_win_wpack_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; if (wc->kp->w_kmajor) { - conv2d_hmx_win_wpack_kmajor(wc, nth, ith); + conv2d_hmx_win_wpack_kmajor(wc, th, nth, ith); } else { - conv2d_hmx_win_wpack_rows(wc, nth, ith); + conv2d_hmx_win_wpack_rows(wc, th, nth, ith); } } @@ -1983,7 +1983,7 @@ static void conv2d_hmx_win_croutons_fold_cwhn(const struct htp_conv2d_win_contex } } -static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) { +static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { struct htp_ops_context * octx = wc->octx; const struct htp_conv2d_kernel_params * kp = wc->kp; @@ -1994,7 +1994,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign const uint32_t b0 = MIN(ith * bpt, ncb); const uint32_t b1 = MIN(b0 + bpt, ncb); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = wc->thr_base + (size_t) ith * kp->thr_size; float * stages[2] = { (float *) thr, (float *) (thr + kp->stage_size) }; __fp16 * act = wc->act[wc->pack_chunk & 1]; @@ -2002,7 +2002,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign uint32_t (*stage_fn)(const struct htp_conv2d_win_context *, dma_queue *, float *, uint32_t, uint32_t) = kp->window.fold ? conv2d_hmx_win_stage_fold : conv2d_hmx_win_stage; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t SB = kp->window.seg_b; @@ -2041,7 +2041,7 @@ static void conv2d_hmx_win_pack(const struct htp_conv2d_win_context * wc, unsign } } -static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsigned int nth, unsigned int ith) { +static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, const htp_thread_t * th, unsigned int nth, unsigned int ith) { struct htp_ops_context * octx = wc->octx; const struct htp_conv2d_kernel_params * kp = wc->kp; const struct htp_tensor * dst = octx->dst; @@ -2060,12 +2060,12 @@ static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsi const uint32_t i0 = MIN(ith * tpt, nt); const uint32_t i1 = MIN(i0 + tpt, nt); - dma_queue * dmaq = octx->ctx->dma[ith]; + dma_queue * dmaq = th->dma_q; uint8_t * thr = wc->thr_base + (size_t) ith * kp->thr_size + 2 * (size_t) kp->stage_size; float * obuf = (float *) thr; __fp16 * out = wc->out[wc->unpack_chunk & 1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; // nd[bi] = DMAs pushed from obuf bi, popped before it is reused HTP_CONV2D_WIN_OBUF steps later uint32_t nd[HTP_CONV2D_WIN_OBUF] = { 0 }; @@ -2154,11 +2154,11 @@ static void conv2d_hmx_win_unpack(const struct htp_conv2d_win_context * wc, unsi } // per-thread callback for one pipeline phase -static void conv2d_hmx_win_phase_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_win_phase_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; if (wc->unpack_chunk >= 0) { - conv2d_hmx_win_unpack(wc, nth, ith); + conv2d_hmx_win_unpack(wc, th, nth, ith); } // weights of this rt block are packed before its first chunk if (wc->pack_chunk == 0) { @@ -2170,12 +2170,12 @@ static void conv2d_hmx_win_phase_thread(unsigned int nth, unsigned int ith, void } } if (wc->pack_chunk >= 0) { - conv2d_hmx_win_pack(wc, nth, ith); + conv2d_hmx_win_pack(wc, th, nth, ith); } } // zero out 2 act buffer regions in VTCM before packing -static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_hmx_win_zero_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_conv2d_win_context * wc = (const struct htp_conv2d_win_context *) data; const size_t n = 2 * wc->layout.act_size / sizeof(float); @@ -2183,7 +2183,7 @@ static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void const size_t i0 = MIN((size_t) ith * per, n); const size_t i1 = MIN(i0 + per, n); - struct htp_thread_trace * tr = &wc->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint16_t block = (uint16_t) (i0 / VLEN_FP32); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, block); if (i1 > i0) { @@ -2195,7 +2195,7 @@ static void conv2d_hmx_win_zero_thread(unsigned int nth, unsigned int ith, void static void conv2d_hmx_win_run_phase(struct htp_conv2d_win_context * wc, int32_t pack, int32_t unpack) { wc->pack_chunk = pack; wc->unpack_chunk = unpack; - work_queue_run(wc->octx->ctx->work_queue, conv2d_hmx_win_phase_thread, wc, wc->kp->n_threads); + work_queue_run(wc->octx->ctx->hvx_queue, conv2d_hmx_win_phase_thread, wc, wc->kp->n_hvx); } @@ -2252,7 +2252,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d const size_t out_size = (size_t) kp->rt_block * kp->ct_chunk * tile; struct htp_conv2d_vtcm_layout layout; - htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_threads); + htp_conv2d_vtcm_layout_build(&layout, kp->rt_block, w_size, act_size, out_size, kp->thr_size, kp->n_hvx); uint8_t * base = (uint8_t *) octx->ctx->vtcm_base; @@ -2275,7 +2275,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d const uint32_t n_ch = octx->src[1]->ne[2] * (kp->window.fold ? KW : 1); if (!kp->cwhn && n_ch <= 2 * CONV2D_WIN_SCAT_MAX) { wctx.n_scat = (n_ch + 1) / 2; - work_queue_run(octx->ctx->work_queue, conv2d_hmx_win_zero_thread, &wctx, kp->n_threads); + work_queue_run(octx->ctx->hvx_queue, conv2d_hmx_win_zero_thread, &wctx, kp->n_hvx); } const int32_t n_chunks = (int32_t) ((wctx.t_end - wctx.t_base + kp->ct_chunk - 1) / kp->ct_chunk); @@ -2307,7 +2307,7 @@ static int conv2d_hmx_win(struct htp_ops_context * octx, const struct htp_conv2d for (uint32_t rt0 = 0; rt0 < kp->n_row_tiles; rt0 += kp->rt_block) { wctx.rt0 = rt0; wctx.n_rt = MIN(kp->rt_block, kp->n_row_tiles - rt0); - work_queue_run(octx->ctx->work_queue, conv2d_hmx_win_wpack_thread, &wctx, kp->n_threads); + work_queue_run(octx->ctx->hvx_queue, conv2d_hmx_win_wpack_thread, &wctx, kp->n_hvx); conv2d_hmx_win_run_phase(&wctx, 0, -1); @@ -3103,10 +3103,10 @@ struct conv2d_dw_dma_context { conv2d_dw_compute_fn_t compute_fn; uint32_t task_start; uint32_t task_count; - uint32_t tasks_per_thread; + uint32_t tasks_per_hvx; }; -static void conv2d_dw_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void conv2d_dw_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct conv2d_dw_dma_context * ctx = (const struct conv2d_dw_dma_context *) data; const struct htp_ops_context * octx = ctx->octx; const struct htp_conv2d_dw_kernel_params * kp = ctx->kp; @@ -3115,24 +3115,24 @@ static void conv2d_dw_dma_thread(unsigned int nth, unsigned int ith, void * data const struct htp_tensor * dst = octx->dst; conv2d_dw_compute_fn_t compute_fn = ctx->compute_fn; - const uint32_t first = ctx->task_start + ith * ctx->tasks_per_thread; - const uint32_t last = MIN(first + ctx->tasks_per_thread, ctx->task_start + ctx->task_count); + const uint32_t first = ctx->task_start + ith * ctx->tasks_per_hvx; + const uint32_t last = MIN(first + ctx->tasks_per_hvx, ctx->task_start + ctx->task_count); if (first >= last) { return; } (void) nth; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size; const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; const uint32_t pad_top_stride = kp->pad_top_stride; uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; - uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread; - uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread; + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) }; @@ -3213,7 +3213,7 @@ int op_conv_2d_dw(struct htp_ops_context * octx) { const uint32_t w_size = src0->size; struct htp_conv2d_dw_vtcm_layout layout; - htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, kp->n_threads); + htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, kp->n_hvx); uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; void * w_vtcm = VTCM_LAYOUT_PTR(void, vtcm_base, layout.off_w); @@ -3240,8 +3240,8 @@ int op_conv_2d_dw(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t nth = MIN(kp->n_threads, task_count); - const uint32_t tasks_per_thread = (task_count + nth - 1) / nth; + const uint32_t nth = MIN(kp->n_hvx, task_count); + const uint32_t tasks_per_hvx = (task_count + nth - 1) / nth; struct conv2d_dw_dma_context ctx = { .octx = octx, @@ -3251,9 +3251,9 @@ int op_conv_2d_dw(struct htp_ops_context * octx) { .compute_fn = conv2d_dw_kernels[kp->kernel_type][kp->is_f16 ? 1 : 0], .task_start = task_start, .task_count = task_count, - .tasks_per_thread = tasks_per_thread, + .tasks_per_hvx = tasks_per_hvx, }; - work_queue_run(octx->ctx->work_queue, conv2d_dw_dma_thread, &ctx, nth); + work_queue_run(octx->ctx->hvx_queue, conv2d_dw_dma_thread, &ctx, nth); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/conv2d-ops.h b/ggml/src/ggml-hexagon/htp/conv2d-ops.h index aefed269e..29dfa812f 100644 --- a/ggml/src/ggml-hexagon/htp/conv2d-ops.h +++ b/ggml/src/ggml-hexagon/htp/conv2d-ops.h @@ -60,7 +60,7 @@ struct htp_conv2d_window_params { struct htp_conv2d_kernel_params { uint8_t kernel_type; - uint8_t n_threads; + uint8_t n_hvx; uint8_t cwhn; uint8_t w_kmajor; @@ -98,7 +98,7 @@ enum htp_conv2d_dw_kernel_type { struct htp_conv2d_dw_kernel_params { uint8_t kernel_type; - uint8_t n_threads; + uint8_t n_hvx; uint8_t cwhn; uint8_t is_f16; @@ -140,8 +140,8 @@ struct htp_conv2d_dw_vtcm_layout { size_t off_dst; size_t src_spad_half_size; size_t dst_spad_half_size; - size_t src_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; }; static inline void htp_conv2d_dw_vtcm_layout_build( @@ -149,17 +149,17 @@ static inline void htp_conv2d_dw_vtcm_layout_build( uint32_t max_src_bytes, uint32_t max_dst_bytes, uint32_t w_size, - uint32_t n_threads + uint32_t n_hvx ) { L->src_spad_half_size = hex_round_up(max_src_bytes + 256, 128); L->dst_spad_half_size = hex_round_up(max_dst_bytes + 128, 128); - L->src_bytes_per_thread = 2 * L->src_spad_half_size; - L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_w, hex_round_up(w_size, 128)); - VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } @@ -181,7 +181,7 @@ static inline void htp_conv2d_vtcm_layout_build( size_t act_size, size_t out_size, size_t thr_size, - uint32_t n_threads + uint32_t n_hvx ) { const size_t sc = hex_round_up((size_t) rt_block * HTP_CONV2D_HMX_SCALES_SIZE, 2048); size_t off = 0; @@ -191,7 +191,7 @@ static inline void htp_conv2d_vtcm_layout_build( VTCM_LAYOUT_ALLOC(off, off_act[1], act_size); VTCM_LAYOUT_ALLOC(off, off_out[0], out_size); VTCM_LAYOUT_ALLOC(off, off_out[1], out_size); - VTCM_LAYOUT_ALLOC(off, off_thr, (size_t) n_threads * thr_size); + VTCM_LAYOUT_ALLOC(off, off_thr, (size_t) n_hvx * thr_size); L->act_size = act_size; L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index 59f406c7a..cdf4d8ec6 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -26,11 +26,11 @@ struct htp_copy_context { uint32_t row_start; uint32_t nrows; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t elem_start; uint32_t nelem; - uint32_t elem_per_thread; + uint32_t elem_per_hvx; struct htp_copy_convert_vtcm_layout layout; uint8_t * vtcm_src0; @@ -38,7 +38,7 @@ struct htp_copy_context { uint32_t tile_start; uint32_t ntiles; - uint32_t tiles_per_thread; + uint32_t tiles_per_hvx; }; #define cpy_preamble \ @@ -66,16 +66,17 @@ struct htp_copy_context { const uint32_t nb3 = dst->nb[3]; #define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ +static void cpy_thread_##NAME##_reshape(const htp_thread_t * th, unsigned int nth, unsigned int ith, \ + void * data) { \ struct htp_copy_context * ct = (struct htp_copy_context *) data; \ struct htp_ops_context * octx = ct->octx; \ cpy_preamble; \ - const uint32_t th_nelem = ct->elem_per_thread; \ + const uint32_t th_nelem = ct->elem_per_hvx; \ const uint32_t th_start = ct->elem_start + ith * th_nelem; \ const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \ if (th_start >= th_end) return; \ \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + dma_queue * dma_q = th->dma_q; \ \ const uint32_t ne01_ne00 = ne01 * ne00; \ const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \ @@ -215,19 +216,19 @@ static inline void cpy_transpose_push_in(dma_queue * q, const struct htp_copy_tr } // Pipelined tile worker: DMA in -> vgather transpose -> DMA out -static void cpy_thread_transpose(unsigned int nth, unsigned int ith, void * data) { +static void cpy_thread_transpose(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_copy_context * ct = (struct htp_copy_context *) data; struct htp_ops_context * octx = ct->octx; const struct htp_copy_transpose_params * tp = &ct->kparams->u.transpose; - const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_thread; - const uint32_t t1 = MIN(t0 + ct->tiles_per_thread, ct->tile_start + ct->ntiles); + const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_hvx; + const uint32_t t1 = MIN(t0 + ct->tiles_per_hvx, ct->tile_start + ct->ntiles); if (t0 >= t1) { return; } - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const dma_addr_t src = octx->src[0]->data; const dma_addr_t dst = octx->dst->data; @@ -332,18 +333,19 @@ static void cpy_thread_transpose(unsigned int nth, unsigned int ith, void * data } #define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \ -static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ +static void cpy_thread_##NAME##_sameshape(const htp_thread_t * th, unsigned int nth, \ + unsigned int ith, void * data) { \ struct htp_copy_context * ct = (struct htp_copy_context *) data; \ struct htp_ops_context * octx = ct->octx; \ cpy_preamble; \ \ - const uint32_t dr = ct->src0_nrows_per_thread; \ + const uint32_t dr = ct->src0_nrows_per_hvx; \ const uint32_t ir0 = ct->row_start + dr * ith; \ const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ if (ir0 >= ir1) return; \ \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + dma_queue * dma_q = th->dma_q; \ + struct htp_thread_trace * tr = th->trace; \ \ const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \ const struct htp_copy_convert_vtcm_layout * L = &ct->layout; \ @@ -352,8 +354,8 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo const uint32_t src0_row_stride = L->src0_row_stride; \ const uint32_t dst_row_stride = L->dst_row_stride; \ const uint32_t blk_rows = L->blk_rows; \ - uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_thread; \ - uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_thread; \ + uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_hvx; \ + uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_hvx; \ const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \ const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \ \ @@ -510,7 +512,7 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -532,17 +534,17 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; const struct htp_copy_convert_params * cvt = &kparams->u.convert; struct htp_copy_context ct; - htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_threads); + htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_hvx); ct.octx = octx; ct.kparams = kparams; ct.row_start = row_start; ct.nrows = nrows; - ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + ct.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; ct.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_src0); @@ -561,7 +563,7 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp return HTP_STATUS_NO_SUPPORT; } - work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + work_queue_run(octx->ctx->hvx_queue, copy_fun, &ct, n_hvx); return HTP_STATUS_OK; } @@ -569,7 +571,7 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -587,14 +589,14 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_copy_context ct; ct.octx = octx; ct.kparams = kparams; ct.elem_start = elem_start; ct.nelem = nelem; - ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div); + ct.elem_per_hvx = fastdiv(nelem + n_hvx - 1, &octx->n_hvx_div); work_queue_func_t copy_fun = NULL; switch (src0->type) { @@ -604,12 +606,12 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern default: return HTP_STATUS_NO_SUPPORT; } - work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + work_queue_run(octx->ctx->hvx_queue, copy_fun, &ct, n_hvx); return HTP_STATUS_OK; } static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -633,20 +635,20 @@ static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_ke return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_copy_context ct; - ct.octx = octx; - ct.kparams = kparams; - ct.tile_start = tile_start; - ct.ntiles = ntiles; - ct.tiles_per_thread = fastdiv(ntiles + n_threads - 1, &octx->n_threads_div); + ct.octx = octx; + ct.kparams = kparams; + ct.tile_start = tile_start; + ct.ntiles = ntiles; + ct.tiles_per_hvx = fastdiv(ntiles + n_hvx - 1, &octx->n_hvx_div); uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; ct.vtcm_src0 = vtcm_base; - ct.vtcm_dst = vtcm_base + (size_t) n_threads * 2 * tp->in_buf_size; + ct.vtcm_dst = vtcm_base + (size_t) n_hvx * 2 * tp->in_buf_size; - work_queue_run(octx->ctx->work_queue, cpy_thread_transpose, &ct, n_threads); + work_queue_run(octx->ctx->hvx_queue, cpy_thread_transpose, &ct, n_hvx); return HTP_STATUS_OK; } @@ -674,7 +676,7 @@ int op_cpy(struct htp_ops_context * octx) { status = cpy_transpose(octx, kparams); break; case HTP_COPY_KERNEL_GATHER_ROWS: - status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads); + status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_hvx); break; default: status = HTP_STATUS_NO_SUPPORT; diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h index 6bb96ca44..1ea50ca1d 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.h +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h @@ -60,7 +60,7 @@ struct htp_copy_kernel_params { uint8_t kernel_type; uint8_t src0_type_size; uint8_t dst_type_size; - uint8_t n_threads; + uint8_t n_hvx; uint32_t total_elems; uint32_t total_rows; @@ -88,8 +88,8 @@ struct htp_copy_convert_vtcm_layout { uint32_t blk_rows; uint32_t src0_buf_size; uint32_t dst_buf_size; - uint32_t spad0_size_per_thread; - uint32_t spad1_size_per_thread; + uint32_t spad0_size_per_hvx; + uint32_t spad1_size_per_hvx; }; static inline void htp_copy_convert_vtcm_layout_build( @@ -98,7 +98,7 @@ static inline void htp_copy_convert_vtcm_layout_build( uint32_t src_type_size, uint32_t dst_type_size, uint32_t blk_rows, - uint32_t n_threads) { + uint32_t n_hvx) { const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256); const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); @@ -108,12 +108,12 @@ static inline void htp_copy_convert_vtcm_layout_build( L->blk_rows = blk_rows; L->src0_buf_size = blk_rows * src0_row_stride; L->dst_buf_size = blk_rows * dst_row_stride; - L->spad0_size_per_thread = 2 * L->src0_buf_size; - L->spad1_size_per_thread = 2 * L->dst_buf_size; + L->spad0_size_per_hvx = 2 * L->src0_buf_size; + L->spad1_size_per_hvx = 2 * L->dst_buf_size; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_threads * L->spad0_size_per_thread); - VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_threads * L->spad1_size_per_thread); + VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_hvx * L->spad0_size_per_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_hvx * L->spad1_size_per_hvx); L->total_bytes = off; } @@ -124,7 +124,7 @@ static inline void htp_copy_convert_solve_layout( uint32_t ne01, uint32_t src_type_size, uint32_t dst_type_size, - uint32_t n_threads) { + uint32_t n_hvx) { const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256); const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); @@ -138,7 +138,7 @@ static inline void htp_copy_convert_solve_layout( blk_rows = 1; } - htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads); + htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_hvx); } #define HTP_COPY_TRANSPOSE_BLK_BYTES (64 * 1024) diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c index 36fc81276..292a05455 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c @@ -8,7 +8,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -48,7 +48,7 @@ struct htp_cumsum_context { size_t dst_row_size; size_t src_row_size_aligned; size_t dst_row_size_aligned; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint32_t total_rows; uint32_t row_start; }; @@ -57,7 +57,7 @@ struct htp_cumsum_context { struct htp_cumsum_context * cctx = (struct htp_cumsum_context *) data; \ struct htp_ops_context * octx = cctx->octx; \ htp_cumsum_tensors_preamble; \ - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; // --------------------------------------------------------------------------- // HVX prefix scan helpers @@ -118,11 +118,11 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri // Per thread worker: Double-buffered DMA // --------------------------------------------------------------------------- -static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { +static void cumsum_thread_f32_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_hvx, cctx->row_start + cctx->total_rows); if (ir0 >= ir1) { return; @@ -151,7 +151,7 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat src_row_size_aligned, src_row_size, src_row_size, 1); } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = ir0; ir < ir1; ir++) { float * dst_spad_row = (float *) dma_queue_pop(dma_q).src; @@ -184,16 +184,16 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat // Per thread worker: Direct HVX (no DMA) // --------------------------------------------------------------------------- -static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void cumsum_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_hvx, cctx->row_start + cctx->total_rows); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ir++) { @@ -234,11 +234,11 @@ int op_cumsum_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { return HTP_STATUS_NO_SUPPORT; @@ -246,12 +246,12 @@ int op_cumsum_f32(struct htp_ops_context * octx) { struct htp_cumsum_vtcm_layout layout = { 0 }; if (kparams->use_dma) { - htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads); + htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_hvx); - octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; - octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; - octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; - octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); octx->src0_spad.src = NULL; @@ -264,15 +264,15 @@ int op_cumsum_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_row_size_aligned = layout.src_row_size_aligned, .dst_row_size_aligned = layout.dst_row_size_aligned, - .rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_rows = nrows, .row_start = row_start, }; if (kparams->use_dma) { - work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, cumsum_thread_f32_dma, &cctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, cumsum_thread_f32, &cctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.h b/ggml/src/ggml-hexagon/htp/cumsum-ops.h index 6e1ac316f..48f4053ed 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.h +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.h @@ -11,30 +11,30 @@ struct htp_cumsum_vtcm_layout { size_t off_dst; size_t src_row_size_aligned; size_t dst_row_size_aligned; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; }; static inline void htp_cumsum_vtcm_layout_build( struct htp_cumsum_vtcm_layout * L, size_t src_row_size, size_t dst_row_size, - uint32_t n_threads + uint32_t n_hvx ) { L->src_row_size_aligned = hex_round_up(src_row_size, 128); L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); - L->src0_bytes_per_thread = 2 * L->src_row_size_aligned; - L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned; + L->src0_bytes_per_hvx = 2 * L->src_row_size_aligned; + L->dst_bytes_per_hvx = 2 * L->dst_row_size_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } struct htp_cumsum_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_rows; uint32_t use_dma; uint32_t vtcm_size; diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c index 7f03e8550..ea1a50316 100644 --- a/ggml/src/ggml-hexagon/htp/diag-ops.c +++ b/ggml/src/ggml-hexagon/htp/diag-ops.c @@ -6,7 +6,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -40,7 +40,7 @@ struct htp_diag_context { size_t dst_row_size; size_t src_batch_size_aligned; size_t dst_row_size_aligned; - uint32_t batches_per_thread; + uint32_t batches_per_hvx; uint32_t total_batches; uint32_t batch_start; }; @@ -60,12 +60,12 @@ static inline void hvx_diag_row_f32(const float * restrict src, float * restrict // Per thread worker: DMA src fetch, compute in VTCM, DMA dst writeback // --------------------------------------------------------------------------- -static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { +static void diag_thread_f32_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; - const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_hvx * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_hvx, dctx->batch_start + dctx->total_batches); if (ib0 >= ib1) { return; @@ -83,7 +83,7 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned); uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ib = ib0; ib < ib1; ib++) { const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02); @@ -124,16 +124,16 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) // Per thread worker: Direct HVX (no DMA) // --------------------------------------------------------------------------- -static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void diag_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_hvx * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_hvx, dctx->batch_start + dctx->total_batches); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); for (uint32_t ib = ib0; ib < ib1; ib++) { @@ -177,11 +177,11 @@ int op_diag_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nbatches))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nbatches))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { return HTP_STATUS_NO_SUPPORT; @@ -192,12 +192,12 @@ int op_diag_f32(struct htp_ops_context * octx) { struct htp_diag_vtcm_layout layout = { 0 }; if (kparams->use_dma) { - htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads); + htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_hvx); - octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; - octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; - octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; - octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); octx->src0_spad.src = NULL; @@ -211,15 +211,15 @@ int op_diag_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_batch_size_aligned = layout.src_batch_size_aligned, .dst_row_size_aligned = layout.dst_row_size_aligned, - .batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div), + .batches_per_hvx = fastdiv(nbatches + n_hvx - 1, &octx->n_hvx_div), .total_batches = nbatches, .batch_start = batch_start, }; if (kparams->use_dma) { - work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, diag_thread_f32_dma, &dctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, diag_thread_f32, &dctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.h b/ggml/src/ggml-hexagon/htp/diag-ops.h index 983bc0242..609a061c5 100644 --- a/ggml/src/ggml-hexagon/htp/diag-ops.h +++ b/ggml/src/ggml-hexagon/htp/diag-ops.h @@ -12,30 +12,30 @@ struct htp_diag_vtcm_layout { size_t off_dst; size_t src_batch_size_aligned; size_t dst_row_size_aligned; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; }; static inline void htp_diag_vtcm_layout_build( struct htp_diag_vtcm_layout * L, size_t src_batch_size, size_t dst_row_size, - uint32_t n_threads + uint32_t n_hvx ) { L->src_batch_size_aligned = hex_round_up(src_batch_size, 128); L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); - L->src0_bytes_per_thread = L->src_batch_size_aligned; - L->dst_bytes_per_thread = L->dst_row_size_aligned; + L->src0_bytes_per_hvx = L->src_batch_size_aligned; + L->dst_bytes_per_hvx = L->dst_row_size_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } struct htp_diag_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_batches; uint32_t use_dma; uint32_t can_split; diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h index 678fc61d7..28d06119b 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.h +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -9,7 +9,7 @@ #include #include "hex-utils.h" -#include "hex-profile.h" +#include "htp-trace.h" #ifdef __cplusplus extern "C" { diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c index 62f12c55d..b7123923c 100644 --- a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c +++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c @@ -9,7 +9,7 @@ #include "dma-queue.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-tensor.h" #include "hvx-utils.h" @@ -57,7 +57,7 @@ struct dsv4_hc_pre_context { uint32_t token_start; uint32_t n_chunks; uint32_t total_jobs; - uint32_t jobs_per_thread; + uint32_t jobs_per_hvx; float scale; uint8_t * vtcm; uint32_t dma_block_chunks; @@ -178,7 +178,7 @@ static __attribute__((noinline)) void dsv4_hc_pre_gated_block_f32( } } -static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, void * data) { +static void dsv4_hc_pre_thread_plain_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data; struct htp_ops_context * octx = dctx->octx; @@ -187,8 +187,8 @@ static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, voi const uint32_t row_stride = dctx->dma_row_stride; const uint32_t buffer_size = dctx->dma_buffer_size; - const uint32_t start = dctx->jobs_per_thread * ith; - const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs); + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); if (start >= end) { return; } @@ -198,7 +198,7 @@ static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, voi dctx->vtcm + 2*buffer_size*ith + buffer_size, }; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t prefetch_job = start; for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end); @@ -208,7 +208,7 @@ static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, voi prefetch_job += block.count; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; uint32_t job = start; uint32_t curr = 0; @@ -244,7 +244,7 @@ static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, voi (void) nth; } -static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, void * data) { +static void dsv4_hc_pre_thread_gated_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data; struct htp_ops_context * octx = dctx->octx; @@ -253,8 +253,8 @@ static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, voi const uint32_t row_stride = dctx->dma_row_stride; const uint32_t buffer_size = dctx->dma_buffer_size; - const uint32_t start = dctx->jobs_per_thread * ith; - const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs); + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); if (start >= end) { return; } @@ -264,7 +264,7 @@ static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, voi dctx->vtcm + 2*buffer_size*ith + buffer_size, }; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t prefetch_job = start; for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end); @@ -274,7 +274,7 @@ static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, voi prefetch_job += block.count; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; uint32_t job = start; uint32_t curr = 0; @@ -317,22 +317,22 @@ int op_dsv4_hc_pre(struct htp_ops_context * octx) { const struct htp_dsv4_hc_pre_kernel_params * kparams = (const struct htp_dsv4_hc_pre_kernel_params *) octx->kernel_params; - const uint32_t n_threads = kparams->n_threads; - if (!htp_ops_context_set_n_threads_div(octx, n_threads, kparams->div_n_threads)) { + const uint32_t n_hvx = kparams->n_hvx; + if (!htp_ops_context_set_n_hvx_div(octx, n_hvx, kparams->div_n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } uint32_t token_start = 0; uint32_t n_tokens = x->ne[2]; uint32_t total_jobs = kparams->total_jobs; - uint32_t jobs_per_thread = kparams->jobs_per_thread; + uint32_t jobs_per_hvx = kparams->jobs_per_hvx; if (octx->ctx->mdev.count > 1) { const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); token_start = range.start; n_tokens = range.count; total_jobs = n_tokens * kparams->n_chunks; - jobs_per_thread = fastdiv(total_jobs + n_threads - 1, &kparams->div_n_threads); + jobs_per_hvx = fastdiv(total_jobs + n_hvx - 1, &kparams->div_n_hvx); } if (n_tokens == 0) { @@ -348,7 +348,7 @@ int op_dsv4_hc_pre(struct htp_ops_context * octx) { .token_start = token_start, .n_chunks = kparams->n_chunks, .total_jobs = total_jobs, - .jobs_per_thread = jobs_per_thread, + .jobs_per_hvx = jobs_per_hvx, .scale = scale, .vtcm = (uint8_t *) octx->ctx->vtcm_base, .dma_block_chunks = kparams->dma_block_chunks, @@ -363,11 +363,11 @@ int op_dsv4_hc_pre(struct htp_ops_context * octx) { default: return HTP_STATUS_INVAL_PARAMS; } - FARF(HIGH, "dsv4-hc-pre: (%ux%ux%u) -> (%ux%u) : ktype %u tokens %u:%u nth %u\n", + FARF(HIGH, "dsv4-hc-pre: (%ux%ux%u) -> (%ux%u) : ktype %u tokens %u:%u nhvx %u\n", x->ne[0], x->ne[1], x->ne[2], dst->ne[0], dst->ne[1], kparams->kernel_type, - token_start, token_start + n_tokens, n_threads); + token_start, token_start + n_tokens, n_hvx); - work_queue_run(octx->ctx->work_queue, worker, &dctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker, &dctx, n_hvx); return HTP_STATUS_OK; } @@ -380,7 +380,7 @@ struct dsv4_hc_post_context { const struct htp_dsv4_hc_post_kernel_params * kparams; uint32_t token_start; uint32_t total_jobs; - uint32_t jobs_per_thread; + uint32_t jobs_per_hvx; uint8_t * vtcm; }; @@ -481,7 +481,7 @@ static inline void dsv4_hc_post_push_comb( 16, comb->nb[1], 16, 4); } -static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith, void * data) { +static void dsv4_hc_post_thread_identity_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data; struct htp_ops_context * octx = dctx->octx; @@ -491,8 +491,8 @@ static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith, const uint32_t block_chunks = dctx->kparams->dma_block_chunks; const uint32_t row_stride = dctx->kparams->dma_row_stride; const uint32_t buffer_size = dctx->kparams->dma_buffer_size; - const uint32_t start = dctx->jobs_per_thread * ith; - const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs); + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); if (start >= end) { return; } @@ -502,7 +502,7 @@ static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith, dctx->vtcm + 2*buffer_size*ith + buffer_size, }; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t prefetch_job = start; for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); @@ -512,7 +512,7 @@ static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith, prefetch_job += block.count; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; uint32_t job = start; uint32_t curr = 0; @@ -549,7 +549,7 @@ static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith, (void) nth; } -static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, void * data) { +static void dsv4_hc_post_thread_comb_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data; struct htp_ops_context * octx = dctx->octx; @@ -559,8 +559,8 @@ static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, voi const uint32_t block_chunks = dctx->kparams->dma_block_chunks; const uint32_t row_stride = dctx->kparams->dma_row_stride; const uint32_t buffer_size = dctx->kparams->dma_buffer_size; - const uint32_t start = dctx->jobs_per_thread * ith; - const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs); + const uint32_t start = dctx->jobs_per_hvx * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_hvx, dctx->total_jobs); if (start >= end) { return; } @@ -570,7 +570,7 @@ static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, voi dctx->vtcm + 2*buffer_size*ith + buffer_size, }; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint32_t prefetch_job = start; for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); @@ -580,7 +580,7 @@ static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, voi prefetch_job += block.count; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; uint32_t job = start; uint32_t curr = 0; @@ -627,22 +627,22 @@ int op_dsv4_hc_post(struct htp_ops_context * octx) { const struct htp_dsv4_hc_post_kernel_params * kparams = (const struct htp_dsv4_hc_post_kernel_params *) octx->kernel_params; - const uint32_t n_threads = kparams->n_threads; - if (!htp_ops_context_set_n_threads_div(octx, n_threads, kparams->div_n_threads)) { + const uint32_t n_hvx = kparams->n_hvx; + if (!htp_ops_context_set_n_hvx_div(octx, n_hvx, kparams->div_n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } uint32_t token_start = 0; uint32_t n_tokens = x->ne[1]; uint32_t total_jobs = kparams->total_jobs; - uint32_t jobs_per_thread = kparams->jobs_per_thread; + uint32_t jobs_per_hvx = kparams->jobs_per_hvx; if (octx->ctx->mdev.count > 1) { const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); token_start = range.start; n_tokens = range.count; total_jobs = n_tokens * kparams->n_chunks; - jobs_per_thread = fastdiv(total_jobs + n_threads - 1, &kparams->div_n_threads); + jobs_per_hvx = fastdiv(total_jobs + n_hvx - 1, &kparams->div_n_hvx); } if (n_tokens == 0) { @@ -654,7 +654,7 @@ int op_dsv4_hc_post(struct htp_ops_context * octx) { .kparams = kparams, .token_start = token_start, .total_jobs = total_jobs, - .jobs_per_thread = jobs_per_thread, + .jobs_per_hvx = jobs_per_hvx, .vtcm = (uint8_t *) octx->ctx->vtcm_base, }; @@ -665,11 +665,11 @@ int op_dsv4_hc_post(struct htp_ops_context * octx) { default: return HTP_STATUS_INVAL_PARAMS; } - FARF(HIGH, "dsv4-hc-post: (%ux%u) (%ux%ux%u) -> (%ux%ux%u) : ktype %u tokens %u:%u nth %u\n", + FARF(HIGH, "dsv4-hc-post: (%ux%u) (%ux%ux%u) -> (%ux%ux%u) : ktype %u tokens %u:%u nhvx %u\n", x->ne[0], x->ne[1], residual->ne[0], residual->ne[1], residual->ne[2], dst->ne[0], dst->ne[1], dst->ne[2], kparams->kernel_type, - token_start, token_start + n_tokens, n_threads); + token_start, token_start + n_tokens, n_hvx); - work_queue_run(octx->ctx->work_queue, worker, &dctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker, &dctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h index a0c01aa91..58a3d0d3c 100644 --- a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h +++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h @@ -18,10 +18,10 @@ enum htp_dsv4_hc_pre_kernel_type { struct htp_dsv4_hc_pre_kernel_params { uint32_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t n_chunks; uint32_t total_jobs; - uint32_t jobs_per_thread; + uint32_t jobs_per_hvx; uint32_t dma_block_chunks; uint32_t dma_row_stride; @@ -29,7 +29,7 @@ struct htp_dsv4_hc_pre_kernel_params { uint32_t vtcm_size; struct fastdiv_values div_n_chunks; - struct fastdiv_values div_n_threads; + struct fastdiv_values div_n_hvx; }; #if defined(__cplusplus) @@ -48,10 +48,10 @@ enum htp_dsv4_hc_post_kernel_type { struct htp_dsv4_hc_post_kernel_params { uint32_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t n_chunks; uint32_t total_jobs; - uint32_t jobs_per_thread; + uint32_t jobs_per_hvx; uint32_t has_comb; uint32_t dma_block_chunks; @@ -60,7 +60,7 @@ struct htp_dsv4_hc_post_kernel_params { uint32_t vtcm_size; struct fastdiv_values div_n_chunks; - struct fastdiv_values div_n_threads; + struct fastdiv_values div_n_hvx; }; #if defined(__cplusplus) diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c index 2cab31b5f..620263063 100644 --- a/ggml/src/ggml-hexagon/htp/fill-ops.c +++ b/ggml/src/ggml-hexagon/htp/fill-ops.c @@ -6,7 +6,7 @@ #include #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hvx-copy.h" #include "hvx-utils.h" @@ -36,27 +36,27 @@ struct htp_fill_context { struct htp_ops_context * octx; const struct htp_fill_kernel_params * kparams; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_rows; // ne1 * ne2 * ne3 uint32_t row_start; HVX_Vector splat_vec; uint32_t elem_size; }; -static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * data) { +static void fill_thread_contiguous(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_fill_context * fctx = (const struct htp_fill_context *) data; struct htp_ops_context * octx = fctx->octx; fill_preamble; // Parallelise over the flat row index spanning ne1*ne2*ne3 - const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_hvx, fctx->row_start + fctx->total_rows); if (ir0 >= ir1) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); const uint32_t elem_start = ir0 * ne0; @@ -68,21 +68,21 @@ static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * da FARF(HIGH, "fill-contig %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); } -static void fill_thread_strided(unsigned int nth, unsigned int ith, void * data) { +static void fill_thread_strided(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_fill_context * fctx = (const struct htp_fill_context *) data; struct htp_ops_context * octx = fctx->octx; const struct htp_fill_kernel_params * kparams = fctx->kparams; fill_preamble; // Parallelise over the flat row index spanning ne1*ne2*ne3 - const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_hvx * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_hvx, fctx->row_start + fctx->total_rows); if (ir0 >= ir1) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ++ir) { @@ -120,11 +120,11 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u ktype=%u\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, kparams->kernel_type); @@ -135,7 +135,7 @@ int op_fill(struct htp_ops_context * octx) { struct htp_fill_context fctx = { .octx = octx, .kparams = kparams, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_rows = nrows, .row_start = row_start, .elem_size = kparams->elem_size, @@ -156,7 +156,7 @@ int op_fill(struct htp_ops_context * octx) { ? fill_thread_contiguous : fill_thread_strided; - work_queue_run(octx->ctx->work_queue, worker, &fctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, worker, &fctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.h b/ggml/src/ggml-hexagon/htp/fill-ops.h index d62f99751..06fe08209 100644 --- a/ggml/src/ggml-hexagon/htp/fill-ops.h +++ b/ggml/src/ggml-hexagon/htp/fill-ops.h @@ -11,7 +11,7 @@ enum htp_fill_kernel_type { struct htp_fill_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_rows; uint32_t elem_size; uint32_t rows_per_chunk; diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index ee258172b..0b7fc2dae 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -14,7 +14,7 @@ #include "dma-queue.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hmx-queue.h" #include "hmx-utils.h" #include "hvx-utils.h" @@ -44,7 +44,7 @@ struct hvx_fa_ddr_spad { __fp16 slopes[512]; uint16_t active_blocks[FA_MAX_KV_BLOCKS]; - uint16_t thread_active_blocks[HTP_MAX_NTHREADS][FA_MAX_KV_BLOCKS] __attribute__((aligned(128))); + uint16_t thread_active_blocks[HTP_MAX_NHVX][FA_MAX_KV_BLOCKS] __attribute__((aligned(128))); }; struct hmx_fa_ddr_spad { @@ -95,7 +95,7 @@ struct htp_fa_context { size_t size_m_block; uint32_t qrows; - uint32_t qrows_per_thread; + uint32_t qrows_per_hvx; uint32_t qrow_start; bool is_q_fp32; @@ -111,7 +111,7 @@ struct htp_fa_context { float * spad_sinks; struct hvx_fa_ddr_spad * ddr; - dma_cache_dm * m_cache; // [n_threads] + dma_cache_dm * m_cache; // [n_hvx] const struct htp_tensor * k; const struct htp_tensor * v; @@ -122,8 +122,8 @@ struct hmx_fa_context { const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent const struct htp_tensor * k; const struct htp_tensor * v; - bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2 - uint32_t n_threads; + bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_hvx >= 2 + uint32_t n_hvx; // Op parameters __fp16 scale; @@ -171,7 +171,7 @@ struct hmx_fa_context { HVX_Vector * vtcm_l_vec; // Row sum [g_br] HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br] HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br] - HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64] + HVX_Vector * vtcm_row_bufs; // Per-HVX softmax row scratch [n_hvx][2][Bc/64] uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity) uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale) __fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block @@ -260,7 +260,7 @@ static uint32_t fa_scan_active_blocks( return n_active; } -static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) { +static void flash_attn_ext_f16_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_fa_context * factx = (struct htp_fa_context *) data; const struct htp_ops_context * octx = factx->octx; const struct htp_tensor * q = octx->src[0]; @@ -306,15 +306,15 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; // total rows in q - const uint32_t dr = factx->qrows_per_thread; + const uint32_t dr = factx->qrows_per_hvx; const uint32_t ir0 = factx->qrow_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows); if (ir0 >= ir1) return; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t DK = nek0; const uint32_t DV = nev0; @@ -774,7 +774,7 @@ typedef struct { size_t buf_idx; } fa_k_int_args_t; -static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) { +static void fa_k_interleave_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_k_int_args_t * args = (fa_k_int_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -787,7 +787,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); if (factx->k->type == HTP_TYPE_Q8_0) { for (uint32_t r = start; r < end; ++r) { @@ -801,18 +801,14 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) } static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start, size_t buf_idx) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { - n = factx->n_threads; + if (factx->n_hvx > 1 && kv_rows >= factx->n_hvx * 2) { + n = factx->n_hvx; } uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t, buf_idx }; - if (n > 1) { - work_queue_run(wp, fa_k_interleave_thread, &args, n); - } else { - fa_k_interleave_thread(1, 0, &args); - } + work_queue_run(wp, fa_k_interleave_thread, &args, n); } typedef struct { @@ -826,7 +822,7 @@ typedef struct { uint32_t rows_per_t; } fa_v_int_args_t; -static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) { +static void fa_v_interleave_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_v_int_args_t * args = (fa_v_int_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -841,7 +837,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) __fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst; - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); if (factx->v->type == HTP_TYPE_Q8_0) { for (uint32_t r = start; r < end; ++r) { @@ -861,18 +857,14 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx, void * v_tiles_dst, size_t n_col_tiles, uint32_t kv_start) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { - n = factx->n_threads; + if (factx->n_hvx > 1 && kv_rows >= factx->n_hvx * 2) { + n = factx->n_hvx; } uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t }; - if (n > 1) { - work_queue_run(wp, fa_v_interleave_thread, &args, n); - } else { - fa_v_interleave_thread(1, 0, &args); - } + work_queue_run(wp, fa_v_interleave_thread, &args, n); } typedef struct { @@ -888,7 +880,7 @@ typedef struct { atomic_uint barrier; } fa_q_load_args_t; -static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { +static void fa_q_load_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_q_load_args_t * args = (fa_q_load_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -902,7 +894,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { const size_t start = (size_t) i * rows_per_t; const size_t end = hex_smin(start + rows_per_t, factx->g_br); - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); // Parallel initialization of per-block state @@ -1010,10 +1002,10 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { - n = factx->n_threads; + if (factx->n_hvx > 1 && n_rows_g >= (size_t) (factx->n_hvx * 2)) { + n = factx->n_hvx; } size_t rows_per_t = hex_align_up(hmx_ceil_div(factx->g_br, n), 2); const uint32_t n_rows_q = hex_smin(factx->Br, factx->neq1 - q_start); @@ -1028,11 +1020,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, args.n_rows_q = n_rows_q; args.q_transposed = q->nb[1] < q->nb[2]; atomic_init(&args.barrier, n); - if (n > 1) { - work_queue_run(wp, fa_q_load_thread, &args, n); - } else { - fa_q_load_thread(1, 0, &args); - } + work_queue_run(wp, fa_q_load_thread, &args, n); } typedef struct { @@ -1046,7 +1034,7 @@ typedef struct { size_t rows_per_t; } fa_o_store_args_t; -static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { +static void fa_o_store_thread_f32(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_o_store_args_t * args = (fa_o_store_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -1062,7 +1050,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -1113,7 +1101,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); } -static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { +static void fa_o_store_thread_f16(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_o_store_args_t * args = (fa_o_store_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -1128,7 +1116,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -1190,19 +1178,15 @@ static void fa_phase_o_store(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; uint32_t n = 1; - if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { - n = factx->n_threads; + if (factx->n_hvx > 1 && n_rows_g >= (size_t) (factx->n_hvx * 2)) { + n = factx->n_hvx; } size_t rows_per_t = hmx_ceil_div(n_rows_g, n); fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t }; - worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; - if (n > 1) { - work_queue_run(wp, store_fn, &args, n); - } else { - store_fn(1, 0, &args); - } + work_queue_func_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; + work_queue_run(wp, store_fn, &args, n); } typedef struct { @@ -1229,7 +1213,7 @@ typedef struct { } fa_softmax_args_t; static inline void fa_softmax_impl( - unsigned int n, unsigned int i, void * data, + const htp_thread_t * th, unsigned int n, unsigned int i, void * data, const bool has_mask, const bool mask_broadcast, const bool is_g1, @@ -1259,7 +1243,7 @@ static inline void fa_softmax_impl( return; } - struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride @@ -1619,8 +1603,8 @@ static inline void fa_softmax_impl( htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); } -static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_nomask(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/false, /*mask_broadcast=*/false, /*is_g1=*/false, @@ -1628,8 +1612,8 @@ static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data /*has_softcap=*/false); } -static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_mask_broadcast_g1(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/true, /*mask_broadcast=*/true, /*is_g1=*/true, @@ -1637,8 +1621,8 @@ static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, /*has_softcap=*/false); } -static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, void * data) { - fa_softmax_impl(n, i, data, +static void fa_softmax_thread_mask_broadcast_gn(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(th, n, i, data, /*has_mask=*/true, /*mask_broadcast=*/true, /*is_g1=*/false, @@ -1646,7 +1630,7 @@ static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, /*has_softcap=*/false); } -static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { +static void fa_softmax_thread(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { fa_softmax_args_t * args = (fa_softmax_args_t *) data; struct hmx_fa_context * factx = args->factx; @@ -1656,7 +1640,7 @@ static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { const bool has_alibi = args->has_alibi; const bool has_softcap = factx->has_softcap; - fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); + fa_softmax_impl(th, n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); } static inline HVX_Vector hvx_fa_load_sinks(const float * sinks_data, size_t idx, size_t count) { @@ -1742,10 +1726,10 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, fa_softmax_args_t * sargs, size_t n_row_tiles, size_t n_row_tiles_g_br) { - work_queue_t wp = factx->octx->ctx->work_queue; + work_queue_t wp = factx->octx->ctx->hvx_queue; const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); - worker_callback_t softmax_fn = fa_softmax_thread; + work_queue_func_t softmax_fn = fa_softmax_thread; if (sargs->mask == NULL && !factx->has_softcap && !sargs->has_alibi) { softmax_fn = fa_softmax_thread_nomask; } else if (sargs->mask != NULL && factx->mask_broadcast && !factx->has_softcap && !sargs->has_alibi) { @@ -1756,13 +1740,12 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, } } - if (factx->n_threads > 1 && n_row_vec_cnt >= 2) { - uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt); + uint32_t n_use = 1; + if (factx->n_hvx > 1 && n_row_vec_cnt >= 2) { + n_use = (uint32_t) hex_smin((size_t) factx->n_hvx, n_row_vec_cnt); sargs->thread_div = init_fastdiv_values(n_use); - work_queue_run(wp, softmax_fn, sargs, n_use); - } else { - softmax_fn(1, 0, sargs); } + work_queue_run(wp, softmax_fn, sargs, n_use); } // ============================================================================ @@ -1780,7 +1763,7 @@ typedef struct { uint8_t * hmx_scales; } hmx_fa_qk_job_t; -static void hmx_fa_qk_dot_worker(void * data) { +static void hmx_fa_qk_dot_worker(const htp_thread_t * th, void * data) { hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_col_tiles = job->n_col_tiles; @@ -1822,7 +1805,7 @@ typedef struct { size_t DV; } hmx_fa_o_update_job_t; -static void hmx_fa_o_update_worker(void * data) { +static void hmx_fa_o_update_worker(const htp_thread_t * th, void * data) { hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_col_tiles = job->n_col_tiles; @@ -1867,7 +1850,7 @@ typedef struct { size_t DV; } hmx_fa_o_norm_job_t; -static void hmx_fa_o_norm_worker(void * data) { +static void hmx_fa_o_norm_worker(const htp_thread_t * th, void * data) { hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data; const size_t n_row_tiles = job->n_row_tiles; const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; @@ -1992,7 +1975,6 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor int hmx_flash_attn_ext(struct htp_ops_context * octx) { struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0]; - struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS]; const struct htp_tensor * q = octx->src[0]; const struct htp_tensor * k = octx->src[1]; const struct htp_tensor * v = octx->src[2]; @@ -2037,7 +2019,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { factx.sinks = octx->src[4]; // NULL if this op has no attention sinks factx.k = k; factx.v = v; - factx.n_threads = kparams->n_threads; + factx.n_hvx = kparams->n_hvx; factx.DK = DK; factx.DV = DV; factx.DK_pad = DK_pad; @@ -2077,7 +2059,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t Bc = factx.Bc; const uint32_t g_br = factx.g_br; const bool pipeline = factx.pipeline; - const uint32_t n_threads = factx.n_threads; + const uint32_t n_hvx = factx.n_hvx; const uint32_t G = factx.G; // Multi-device: prefer head-parallel partitioning (each core owns a disjoint head @@ -2147,7 +2129,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // Build the VTCM layout once (shared with the host estimator) and place every // scratch buffer at its computed offset. Padded head dims size the HMX tiles. struct hmx_fa_vtcm_layout L; - hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_threads, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads); + hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_hvx, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads); if (L.total_bytes > ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -2681,7 +2663,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -2776,16 +2758,16 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; factx.qrows = qrows; factx.qrow_start = qrow_start; - factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div); + factx.qrows_per_hvx = fastdiv(qrows + n_hvx - 1, &octx->n_hvx_div); const bool has_mask = (mask != NULL); const bool has_sinks = (octx->src[4] != NULL); struct hvx_fa_vtcm_layout L; - hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_threads); + hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_hvx); if (L.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -2822,7 +2804,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { dma_queue_pop(dma_q); } - work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads); + work_queue_run(octx->ctx->hvx_queue, flash_attn_ext_f16_thread, &factx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h index f2654f0a8..70cf3e41a 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -46,7 +46,7 @@ struct htp_fa_kernel_params { uint8_t kernel_type; // enum htp_fa_kernel_type uint8_t head_split; // 1 = partition by KV heads in multicore, 0 = token partition uint8_t flags; // HTP_FA_FLAG_* - uint8_t n_threads; // Number of threads to run + uint8_t n_hvx; // Number of threads to run // Common parameters uint16_t Br; @@ -155,7 +155,7 @@ struct hmx_fa_vtcm_layout { static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, size_t gqa_factor, size_t DK, size_t DV, - size_t Br, size_t Bc, size_t n_threads, + size_t Br, size_t Bc, size_t n_hvx, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS); @@ -207,7 +207,7 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_p_tiles[1], s_tile_size, pipeline); VTCM_LAYOUT_ALLOC(off_group_b, off_s_rowmax, col_vec_size); VTCM_LAYOUT_ALLOC(off_group_b, off_p_rowsum, col_vec_size); - VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_threads); + VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_hvx); const size_t group_b_size = off_group_b - off_group_b_c; @@ -251,9 +251,9 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, } // Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration. -static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { +static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_hvx, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { struct hmx_fa_vtcm_layout L; - hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline, is_q_fp32, has_sinks, n_heads); + hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_hvx, pipeline, is_q_fp32, has_sinks, n_heads); return L.total_bytes; } @@ -282,7 +282,7 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, - size_t n_threads) { + size_t n_hvx) { const size_t size_q_row_padded = hex_round_up(DK * (is_q_fp32 ? 4 : 2), 128); const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128); const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128); @@ -296,11 +296,11 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_threads); - VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_threads); - VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_threads); - VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_threads, has_mask); - VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_threads); + VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_hvx, has_mask); + VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_hvx); VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, size_sinks, has_sinks); L->size_q_block = size_q_block; @@ -312,9 +312,9 @@ static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, L->total_bytes = off; } -static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_threads) { +static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_hvx) { struct hvx_fa_vtcm_layout L; - hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_threads); + hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_hvx); return L.total_bytes; } @@ -329,7 +329,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, size_t qo_len, size_t kv_len, size_t vtcm_budget, - size_t n_threads, + size_t n_hvx, bool is_q_fp32, bool has_sinks, size_t n_heads, @@ -337,7 +337,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, const size_t T = HMX_FP16_TILE_N_ROWS; // 32 const size_t br_unit = hmx_ceil_div(T, gqa_factor); const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64 - const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_threads >= 2); + const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_hvx >= 2); // Br_max: largest Br aligned to br_unit that does not exceed qo_len. const size_t Br_max = qo_len >= br_unit ? hex_align_down(qo_len, br_unit) : br_unit; @@ -361,15 +361,15 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out, for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) { // Try all Bc candidates from Bc_limit down to bc_unit for (size_t Bc = Bc_limit; Bc >= bc_unit; Bc -= bc_unit) { - size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline, is_q_fp32, has_sinks, n_heads); + size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_hvx, can_pipeline, is_q_fp32, has_sinks, n_heads); if (vtcm_needed <= vtcm_budget) { // This Bc fits for this Br! const size_t q_blocks = (qo_len + Br - 1) / Br; const size_t kv_blocks = (kv_len + Bc - 1) / Bc; - const size_t actual_threads = (kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1; + const size_t actual_hvx = (kv_blocks >= 3 && n_hvx >= 2) ? n_hvx : 1; const size_t n_rows_g = Br * gqa_factor; const size_t n_row_vec_cnt = (n_rows_g + 63) / 64; - const size_t n_use = n_row_vec_cnt < actual_threads ? n_row_vec_cnt : actual_threads; + const size_t n_use = n_row_vec_cnt < actual_hvx ? n_row_vec_cnt : actual_hvx; const size_t vecs_per_t = n_use > 0 ? (n_row_vec_cnt + n_use - 1) / n_use : 1; const size_t c_iter_actual = c_iter_base + c_softmax * vecs_per_t; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c index a70f41185..0280f99a4 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c @@ -748,7 +748,7 @@ static inline void gdn_step_scalar_f32( } } -static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, void * data) { +static void gated_delta_net_f32_pp_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; struct htp_ops_context * octx = gctx->octx; const struct htp_gdn_kernel_params * kparams = gctx->kparams; @@ -777,10 +777,10 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo float * dst_base = (float *) (uintptr_t) dst->data; float * state_out_base = dst_cache ? (float *) (uintptr_t) dst_cache->data : (dst_base + S_v * H * n_tokens * n_seqs); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct htp_gdn_vtcm_layout * layout = &gctx->layout; float * s_work[2]; - s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_hvx * ith); s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); const struct fastdiv_values * fd_H = &kparams->div_H; @@ -817,7 +817,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; int curr_spad_idx = 0; for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { @@ -892,7 +892,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo dma_queue_flush(dma_q); } -static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, void * data) { +static void gated_delta_net_f32_tg_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; struct htp_ops_context * octx = gctx->octx; const struct htp_gdn_kernel_params * kparams = gctx->kparams; @@ -918,10 +918,10 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const float scale = kparams->scale; float * dst_base = (float *) (uintptr_t) dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct htp_gdn_vtcm_layout * layout = &gctx->layout; float * s_work[2]; - s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_hvx * ith); s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); const struct fastdiv_values * fd_H = &kparams->div_H; @@ -957,7 +957,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; int curr_spad_idx = 0; for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { @@ -1031,7 +1031,7 @@ struct htp_gdn_hmx_gemm_task { uint8_t * hmx_scales; }; -static void htp_gdn_hmx_gemm_worker(void * data) { +static void htp_gdn_hmx_gemm_worker(const htp_thread_t * th, void * data) { struct htp_gdn_hmx_gemm_task * task = (struct htp_gdn_hmx_gemm_task *) data; asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)task->hmx_scales)); @@ -1623,7 +1623,7 @@ static inline void gdn_init_head_ptrs( head->rows_inv = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_inv + h * row_vecs_sz); head->rows_a = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_a + h * row_vecs_sz); - head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_thread_scratch + h * (64 * 128)); + head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_hvx_scratch + h * (64 * 128)); head->vtcm_tmp = head->vtcm_m + 32; head->iv1 = base_iv1 + h; @@ -1654,10 +1654,10 @@ struct htp_gdn_batch_context { const struct htp_gdn_kernel_params * kparams; }; -static void gdn_hvx_init_state_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_init_state_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); struct htp_gdn_head_ptrs * head = &bctx->heads[i]; @@ -1690,10 +1690,10 @@ static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) { return hvx_vec_repl4(Q6_V_vror_VR(v, 124)); } -static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase1a_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); @@ -1763,10 +1763,10 @@ static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); } -static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase1b_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); @@ -1860,10 +1860,10 @@ static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); } -static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase2_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); @@ -1884,10 +1884,10 @@ static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); } -static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase3_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); @@ -1928,10 +1928,10 @@ static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); } -static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase4_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); @@ -1959,10 +1959,10 @@ static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); } -static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase5_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); @@ -1984,10 +1984,10 @@ static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); } -static void gdn_hvx_phase6_worker(unsigned int n, unsigned int i, void * data) { +static void gdn_hvx_phase6_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { (void) n; struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; - struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + struct htp_thread_trace * tr = th->trace; const uint16_t info = (uint16_t) bctx->c; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_STATE, info); @@ -2040,7 +2040,7 @@ static int gated_delta_net_f32_hmx_chunked( const uint32_t n_sv_tiles = S_v / 32; struct htp_gdn_hmx_vtcm_layout L; - htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_threads, kparams->pipeline != 0); + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_hvx, kparams->pipeline != 0); if (L.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; @@ -2062,7 +2062,7 @@ static int gated_delta_net_f32_hmx_chunked( hmx_queue_t hmx_q = octx->ctx->hmx_queue; dma_queue * dma_q = octx->ctx->dma[0]; - work_queue_t wp = octx->ctx->work_queue; + work_queue_t wp = octx->ctx->hvx_queue; struct htp_gdn_head_ptrs heads[8]; struct htp_gdn_hmx_gemm_task gemm_tasks[8][9]; @@ -2286,7 +2286,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) { } const struct htp_gdn_kernel_params * kparams = (const struct htp_gdn_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -2321,7 +2321,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) { return gated_delta_net_f32_hmx_chunked(octx, kparams, row_start, nrows); } - const uint32_t n_threads = (nrows < octx->n_threads) ? nrows : octx->n_threads; + const uint32_t n_hvx = (nrows < octx->n_hvx) ? nrows : octx->n_hvx; struct htp_gdn_context gctx; gctx.octx = octx; @@ -2329,7 +2329,7 @@ int op_gated_delta_net(struct htp_ops_context * octx) { gctx.row_start = row_start; gctx.nrows = nrows; - htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_threads); + htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_hvx); gctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, gctx.layout.off_state); if (gctx.layout.total_bytes > octx->ctx->vtcm_size) { @@ -2337,18 +2337,18 @@ int op_gated_delta_net(struct htp_ops_context * octx) { } FARF(HIGH, "gated-delta-net-f32: q(%ux%ux%ux%u) k(%ux%ux%ux%u) v(%ux%ux%ux%u) state(%ux%ux%ux%u) -> (%ux%ux%ux%u) : " - "vtcm-size %zu n_threads %u\n", + "vtcm-size %zu n_hvx %u\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->src[2]->ne[0], octx->src[2]->ne[1], octx->src[2]->ne[2], octx->src[2]->ne[3], octx->src[5]->ne[0], octx->src[5]->ne[1], octx->src[5]->ne[2], octx->src[5]->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - gctx.layout.total_bytes, n_threads); + gctx.layout.total_bytes, n_hvx); if (n_tokens == 1) { - work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, gated_delta_net_f32_tg_thread, &gctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, gated_delta_net_f32_pp_thread, &gctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h index 0d8818588..aff1eb645 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h @@ -29,7 +29,7 @@ struct htp_gdn_kernel_params { uint16_t n_chunks; uint16_t n_heads_batch; - uint32_t n_threads; + uint32_t n_hvx; uint32_t S_v; uint32_t H; uint32_t n_tokens; @@ -59,7 +59,7 @@ _Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_para struct htp_gdn_vtcm_layout { size_t state_aligned; - size_t bytes_per_thread; + size_t bytes_per_hvx; size_t off_state; size_t total_bytes; }; @@ -67,13 +67,13 @@ struct htp_gdn_vtcm_layout { static inline void htp_gdn_vtcm_layout_build( struct htp_gdn_vtcm_layout * L, uint32_t S_v, - uint32_t n_threads + uint32_t n_hvx ) { size_t state_bytes = (size_t) S_v * S_v * sizeof(float); L->state_aligned = hex_round_up(state_bytes, 128); - L->bytes_per_thread = 2 * L->state_aligned; + L->bytes_per_hvx = 2 * L->state_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_hvx * n_hvx); L->total_bytes = off; } @@ -128,7 +128,7 @@ struct htp_gdn_hmx_vtcm_layout { size_t off_rows_inv; size_t off_rows_a; - size_t off_thread_scratch; + size_t off_hvx_scratch; size_t off_scales_1; size_t state_f32_bytes; @@ -140,7 +140,7 @@ struct htp_gdn_hmx_vtcm_layout { size_t tile_64x64_bytes; uint32_t n_heads_batch; - uint32_t n_threads; + uint32_t n_hvx; bool pipeline; size_t total_bytes; }; @@ -150,16 +150,16 @@ static inline void htp_gdn_hmx_vtcm_layout_build( uint32_t S_v, uint32_t chunk_size, uint32_t n_heads_batch, - uint32_t n_threads, + uint32_t n_hvx, bool pipeline ) { memset(L, 0, sizeof(*L)); L->n_heads_batch = n_heads_batch; - L->n_threads = n_threads; + L->n_hvx = n_hvx; L->pipeline = pipeline; const size_t bh = (size_t) n_heads_batch; - const size_t nth = (size_t) (n_threads > 0 ? n_threads : 1); + const size_t nth = (size_t) (n_hvx > 0 ? n_hvx : 1); const size_t state_f32_sz = hex_round_up(S_v * S_v * sizeof(float), 2048); const size_t state_f16_sz = hex_round_up(S_v * S_v * sizeof(__fp16), 2048); @@ -248,9 +248,9 @@ static inline void htp_gdn_hmx_vtcm_layout_build( VTCM_LAYOUT_ALLOC(off, off_rows_inv, bh * row_vecs_sz); VTCM_LAYOUT_ALLOC(off, off_rows_a, bh * row_vecs_sz); - const size_t thread_scratch_sz = 64 * 128; + const size_t hvx_scratch_sz = 64 * 128; off = hex_align_up(off, HMX_FP16_TILE_SIZE); - VTCM_LAYOUT_ALLOC(off, off_thread_scratch, nth * thread_scratch_sz); + VTCM_LAYOUT_ALLOC(off, off_hvx_scratch, nth * hvx_scratch_sz); off = hex_align_up(off, HMX_FP16_TILE_SIZE); VTCM_LAYOUT_ALLOC(off, off_scales_1, HMX_FP16_TILE_SIZE); @@ -263,7 +263,7 @@ static inline bool htp_gdn_hmx_solve_layout( uint32_t chunk_size, uint32_t total_rows, size_t vtcm_budget, - uint32_t n_threads, + uint32_t n_hvx, bool pipeline, uint32_t * n_heads_batch_out ) { @@ -271,8 +271,8 @@ static inline bool htp_gdn_hmx_solve_layout( if (max_batch > total_rows) { max_batch = total_rows; } - if (max_batch > n_threads) { - max_batch = n_threads; + if (max_batch > n_hvx) { + max_batch = n_hvx; } static const uint32_t candidates[] = { 8, 6, 4, 2, 1 }; for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { @@ -281,7 +281,7 @@ static inline bool htp_gdn_hmx_solve_layout( continue; } struct htp_gdn_hmx_vtcm_layout L; - htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_threads, pipeline); + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_hvx, pipeline); if (L.total_bytes <= vtcm_budget) { *layout_out = L; *n_heads_batch_out = bh; @@ -289,7 +289,7 @@ static inline bool htp_gdn_hmx_solve_layout( } } if (pipeline) { - return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_threads, false, n_heads_batch_out); + return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_hvx, false, n_heads_batch_out); } return false; } diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.c index 1e88445f1..02804db27 100644 --- a/ggml/src/ggml-hexagon/htp/gather-rows.c +++ b/ggml/src/ggml-hexagon/htp/gather-rows.c @@ -3,7 +3,7 @@ #include #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hvx-utils.h" #include "dma-queue.h" #include "htp-ctx.h" @@ -23,7 +23,7 @@ struct gather_rows_task { dma_addr_t dst; uint32_t r_start; uint32_t rows; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; const struct htp_gather_rows_params * kparams; gather_rows_compute_fn_t compute; }; @@ -190,21 +190,21 @@ static void gather_rows_compute_f16_vgn( } } -static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void gather_rows_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct gather_rows_task * task = (const struct gather_rows_task *) data; const struct htp_gather_rows_params * kparams = task->kparams; const struct htp_tensor * a = task->a; const struct htp_tensor * b = task->b; - const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows); + const uint32_t r_offset = MIN(ith * task->rows_per_hvx, task->rows); const uint32_t r_beg = task->r_start + r_offset; - const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows); + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_hvx, task->rows); if (r_beg >= r_end) { return; } - struct htp_thread_trace * tr = &task->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; struct htp_gather_rows_vtcm_layout layout; htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 4); @@ -227,7 +227,7 @@ static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * da gather_rows_init_tables_f32(tab, inc, kparams, a, b); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); - dma_queue * q = task->octx->ctx->dma[ith]; + dma_queue * q = th->dma_q; const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); const uint32_t nr_max = kparams->nr_max; const uint32_t nr0 = MIN(nr_max, r_end - r_beg); @@ -271,21 +271,21 @@ static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * da dma_queue_flush(q); } -static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) { +static void gather_rows_thread_f16(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct gather_rows_task * task = (const struct gather_rows_task *) data; const struct htp_gather_rows_params * kparams = task->kparams; const struct htp_tensor * a = task->a; const struct htp_tensor * b = task->b; - const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows); + const uint32_t r_offset = MIN(ith * task->rows_per_hvx, task->rows); const uint32_t r_beg = task->r_start + r_offset; - const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows); + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_hvx, task->rows); if (r_beg >= r_end) { return; } - struct htp_thread_trace * tr = &task->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; struct htp_gather_rows_vtcm_layout layout; htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 2); @@ -308,7 +308,7 @@ static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * da gather_rows_init_tables_f16(tab, inc, kparams, a, b); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); - dma_queue * q = task->octx->ctx->dma[ith]; + dma_queue * q = th->dma_q; const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); const uint32_t nr_max = kparams->nr_max; @@ -357,13 +357,13 @@ int htp_gather_rows( const struct htp_tensor * a, const struct htp_tensor * b, const struct htp_gather_rows_params * kparams, - uint32_t n_threads) { + uint32_t n_hvx) { - if (!htp_ops_context_set_n_threads(octx, n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size); + assert(kparams->slice * octx->n_hvx <= octx->ctx->vtcm_size); const uint32_t total_rows = kparams->total_rows; uint32_t r_start = 0; @@ -389,7 +389,7 @@ int htp_gather_rows( task.r_start = r_start; task.rows = r_count; task.kparams = kparams; - task.rows_per_thread = kparams->rows_per_thread; + task.rows_per_hvx = kparams->rows_per_hvx; if (elem_size == 2) { task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn; @@ -398,6 +398,6 @@ int htp_gather_rows( } work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32; - work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, worker_func, &task, octx->n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.h index c08ef4d38..9ca215ac3 100644 --- a/ggml/src/ggml-hexagon/htp/gather-rows.h +++ b/ggml/src/ggml-hexagon/htp/gather-rows.h @@ -22,7 +22,7 @@ struct htp_gather_rows_params { uint32_t region_size; uint32_t slice; uint32_t total_rows; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint8_t b_dense; uint8_t pad[3]; }; @@ -35,7 +35,7 @@ struct htp_gather_rows_vtcm_layout { size_t off_out0; size_t off_region1; size_t off_out1; - size_t total_per_thread; + size_t total_per_hvx; }; static inline void htp_gather_rows_vtcm_layout_build( @@ -58,7 +58,7 @@ static inline void htp_gather_rows_vtcm_layout_build( VTCM_LAYOUT_ALLOC(off, off_out0, out_aligned); VTCM_LAYOUT_ALLOC(off, off_region1, region_aligned); VTCM_LAYOUT_ALLOC(off, off_out1, out_aligned); - L->total_per_thread = off; + L->total_per_hvx = off; } static inline bool htp_gather_rows_solve_layout( @@ -67,7 +67,7 @@ static inline bool htp_gather_rows_solve_layout( uint32_t nb, uint32_t b_nb1, bool b_dense, uint32_t elem_size, uint32_t total_rows, - uint32_t n_threads, + uint32_t n_hvx, uint32_t mdev_count, uint32_t vtcm_size) { @@ -86,13 +86,13 @@ static inline bool htp_gather_rows_solve_layout( const uint32_t rg = v_lanes / g; const uint32_t vg = ne / g; - const uint32_t slice = (vtcm_size / n_threads) & ~127u; + const uint32_t slice = (vtcm_size / n_hvx) & ~127u; const uint32_t total_chunks = total_rows / rg; const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count; const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count) ? (chunks_per_dev * rg + (total_rows % rg)) : total_rows; - const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg); + const uint32_t rows_per_hvx = hex_round_up((max_rows_per_dev + n_hvx - 1) / n_hvx, rg); const uint32_t fixed = 3 * vg * 128; const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size; @@ -101,7 +101,7 @@ static inline bool htp_gather_rows_solve_layout( } const uint32_t avail_per_buf = (slice - fixed) / 2; - uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread); + uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_hvx); if (elem_size == 2) { const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0); if (per_row_region > 0) { @@ -132,7 +132,7 @@ static inline bool htp_gather_rows_solve_layout( struct htp_gather_rows_vtcm_layout layout; htp_gather_rows_vtcm_layout_build(&layout, vg, region_size, nr_max, ne, elem_size); - if ((elem_size != 2 || region_size <= 32768) && layout.total_per_thread <= slice) { + if ((elem_size != 2 || region_size <= 32768) && layout.total_per_hvx <= slice) { break; } @@ -154,7 +154,7 @@ static inline bool htp_gather_rows_solve_layout( kparams->region_size = region_size; kparams->slice = slice; kparams->total_rows = total_rows; - kparams->rows_per_thread = rows_per_thread; + kparams->rows_per_hvx = rows_per_hvx; kparams->b_dense = b_dense ? 1 : 0; kparams->pad[0] = 0; kparams->pad[1] = 0; @@ -183,7 +183,7 @@ int htp_gather_rows( const struct htp_tensor * a, const struct htp_tensor * b, const struct htp_gather_rows_params * kparams, - uint32_t n_threads); + uint32_t n_hvx); #endif #endif // HTP_GATHER_ROWS_H diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index 39acdfda6..83e85f005 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -37,7 +37,7 @@ struct get_rows_context { get_rows_tiled_compute_fn_t compute_tiled; uint32_t task_start; uint32_t tasks; - uint32_t tasks_per_thread; + uint32_t tasks_per_hvx; uint32_t tile_size; uint32_t tile_stride; uint32_t src0_row_size; @@ -95,20 +95,20 @@ static __attribute__((noinline)) uint32_t get_rows_read_index_vtcm( return (uint32_t) Q6_R_vextract_VR(v, off); } -static void get_rows_thread_sametype(unsigned int nth, unsigned int ith, void * data) { +static void get_rows_thread_sametype(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct get_rows_context * grctx = (struct get_rows_context *) data; struct htp_ops_context * octx = grctx->octx; const struct htp_get_rows_kernel_params * kparams = grctx->kparams; get_rows_preamble; - const uint32_t dr = grctx->tasks_per_thread; + const uint32_t dr = grctx->tasks_per_hvx; const uint32_t ir0 = grctx->task_start + dr * ith; if (ir0 >= grctx->task_start + grctx->tasks) { return; } const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); const uint32_t row_size_bytes = grctx->src0_row_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t i = ir0; i < ir1; ++i) { const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); @@ -174,23 +174,23 @@ static inline struct get_rows_flat_task get_rows_flat_calc_task( return task; } -static void get_rows_thread_flat(unsigned int nth, unsigned int ith, void * data) { +static void get_rows_thread_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct get_rows_context * grctx = (struct get_rows_context *) data; struct htp_ops_context * octx = grctx->octx; get_rows_preamble; - const uint32_t dr = grctx->tasks_per_thread; + const uint32_t dr = grctx->tasks_per_hvx; const uint32_t ir0 = grctx->task_start + dr * ith; if (ir0 >= grctx->task_start + grctx->tasks) { return; } const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; - uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread; - uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread; + uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_hvx; + uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_hvx; const uint32_t src0_bytes = grctx->src0_row_size; const uint32_t dst_bytes = grctx->dst_row_size; get_rows_flat_compute_fn_t compute = grctx->compute_flat; @@ -276,12 +276,12 @@ static inline struct get_rows_tiled_task get_rows_tiled_calc_task( return task; } -static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) { +static void get_rows_thread_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct get_rows_context * grctx = (struct get_rows_context *) data; struct htp_ops_context * octx = grctx->octx; get_rows_preamble; - const uint32_t dr = grctx->tasks_per_thread; + const uint32_t dr = grctx->tasks_per_hvx; const uint32_t ir0 = grctx->task_start + dr * ith; if (ir0 >= grctx->task_start + grctx->tasks) { return; @@ -290,10 +290,10 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type); const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; - uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread; - uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_hvx; + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_hvx; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const uint32_t tile_size = grctx->tile_size; const uint32_t tile_stride = grctx->tile_stride; @@ -414,11 +414,11 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct get_rows_context grctx; grctx.octx = octx; @@ -428,14 +428,14 @@ int op_get_rows(struct htp_ops_context * octx) { grctx.compute_tiled = NULL; grctx.task_start = task_start; grctx.tasks = tasks; - grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); + grctx.tasks_per_hvx = fastdiv(tasks + n_hvx - 1, &octx->n_hvx_div); grctx.tile_size = 0; grctx.tile_stride = 0; grctx.src0_row_size = src0_row_size; grctx.dst_row_size = dst_row_size; grctx.idx_type = octx->src[1]->type; - htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_threads); + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_hvx); if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) { FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n", @@ -497,14 +497,14 @@ int op_get_rows(struct htp_ops_context * octx) { grctx.read_index = get_rows_read_index_vtcm; } - FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n", + FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-hvx %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - grctx.vtcm_layout.src0_bytes_per_thread * n_threads, - grctx.vtcm_layout.dst_bytes_per_thread * n_threads, - kparams->kernel_type, n_threads); + grctx.vtcm_layout.src0_bytes_per_hvx * n_hvx, + grctx.vtcm_layout.dst_bytes_per_hvx * n_hvx, + kparams->kernel_type, n_hvx); - work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, q_func, &grctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h index 4634be2e2..986966f11 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h @@ -16,7 +16,7 @@ enum htp_get_rows_kernel_type { struct htp_get_rows_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_tasks; uint32_t vtcm_size; @@ -34,8 +34,8 @@ struct htp_get_rows_vtcm_layout { size_t off_src0; size_t off_dst; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; @@ -48,7 +48,7 @@ static inline void htp_get_rows_vtcm_layout_build( uint32_t ne00, uint32_t total_tasks, int idx_type, - uint32_t n_threads) { + uint32_t n_hvx) { memset(L, 0, sizeof(*L)); @@ -71,14 +71,14 @@ static inline void htp_get_rows_vtcm_layout_build( L->src0_spad_half_size = (src0_row_size + 255) & ~255; L->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255; - L->src0_bytes_per_thread = 2 * L->src0_spad_half_size; - L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; + L->src0_bytes_per_hvx = 2 * L->src0_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; } size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src1, src1_alloc_bytes); - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.c index dc621bf46..223c1829f 100644 --- a/ggml/src/ggml-hexagon/htp/glu-ops.c +++ b/ggml/src/ggml-hexagon/htp/glu-ops.c @@ -96,7 +96,7 @@ struct htp_glu_context { uint32_t block; uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; int nc; @@ -104,9 +104,9 @@ struct htp_glu_context { uint8_t * vtcm_src1; uint8_t * vtcm_dst; - size_t vtcm_src0_size_per_thread; - size_t vtcm_src1_size_per_thread; - size_t vtcm_dst_size_per_thread; + size_t vtcm_src0_size_per_hvx; + size_t vtcm_src1_size_per_hvx; + size_t vtcm_dst_size_per_hvx; }; #define htp_glu_op_preamble \ @@ -722,11 +722,11 @@ static void sigglu_f16(const _Float16 * restrict src0, } #define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \ - static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \ + static void glu_##SUFFIX##_per_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ struct htp_glu_context * gctx = (struct htp_glu_context *) data; \ htp_glu_preamble; \ \ - struct htp_thread_trace * tr = gctx->octx->ctx ? &gctx->octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = th->trace; \ \ size_t src0_row_size = gctx->src0_row_size; \ size_t src1_row_size = gctx->src1_row_size; \ @@ -736,10 +736,10 @@ static void sigglu_f16(const _Float16 * restrict src0, size_t src1_row_stride = gctx->src1_row_stride; \ \ const uint32_t src0_nrows = gctx->src0_nrows; \ - const uint32_t src0_nrows_per_thread = gctx->src0_nrows_per_thread; \ + const uint32_t src0_nrows_per_hvx = gctx->src0_nrows_per_hvx; \ \ - const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, gctx->row_start + src0_nrows); \ + const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_hvx * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, gctx->row_start + src0_nrows); \ \ /* no work for this thread */ \ if (src0_start_row >= src0_end_row) { \ @@ -754,9 +754,9 @@ static void sigglu_f16(const _Float16 * restrict src0, const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \ const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \ \ - uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_thread); \ - uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_thread); \ - uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_thread); \ + uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_hvx); \ + uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_hvx); \ + uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_hvx); \ \ size_t src0_spad_half_size = gctx->src0_spad_half_size; \ size_t src1_spad_half_size = gctx->src1_spad_half_size; \ @@ -764,7 +764,7 @@ static void sigglu_f16(const _Float16 * restrict src0, \ const int BLOCK = gctx->block; \ \ - dma_queue * dma_q = gctx->octx->ctx->dma[ith]; \ + dma_queue * dma_q = th->dma_q; \ glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) gctx->compute; \ \ for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ @@ -871,7 +871,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -893,7 +893,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). @@ -904,28 +904,28 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; struct htp_glu_vtcm_layout L; - htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread); + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, kparams->vtcm_row_per_hvx); const size_t src0_row_size_aligned = L.row_size_aligned; const size_t src1_row_size_aligned = L.row_size_aligned; const size_t dst_row_size_aligned = L.row_size_aligned; // Make sure the reserved vtcm size is sufficient - if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { + if (kparams->vtcm_row_per_hvx == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, - (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_hvx); return HTP_STATUS_VTCM_TOO_SMALL; } if (src1) { FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, - L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_hvx * n_hvx, + L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } else { FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + L.src0_bytes_per_hvx * n_hvx, L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } // Prepare context @@ -934,7 +934,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { gctx.compute = (void *) compute_fn; gctx.op_str = op_type; - gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + gctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); gctx.src0_row_size = src0_row_size; gctx.src1_row_size = src1_row_size; @@ -952,13 +952,13 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread; - gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread; - gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; + gctx.vtcm_src0_size_per_hvx = (uint32_t) L.src0_bytes_per_hvx; + gctx.vtcm_src1_size_per_hvx = (uint32_t) L.src1_bytes_per_hvx; + gctx.vtcm_dst_size_per_hvx = (uint32_t) L.dst_bytes_per_hvx; - gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1; - gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1; - gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1; + gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_hvx >> 1; + gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_hvx >> 1; + gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_hvx >> 1; gctx.block = L.block; gctx.src0_nrows = nrows; @@ -987,7 +987,7 @@ static int execute_op_glu_f32(struct htp_ops_context * octx) { gctx.data_src1 = data_src1; gctx.data_dst = dst->data; - work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, (work_queue_func_t)glu_f32_per_hvx, &gctx, n_hvx); return HTP_STATUS_OK; } @@ -1037,7 +1037,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -1059,7 +1059,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). @@ -1070,28 +1070,28 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; struct htp_glu_vtcm_layout L; - htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread); + htp_glu_vtcm_layout_build(&L, dst_row_size, n_hvx, kparams->vtcm_row_per_hvx); const size_t src0_row_size_aligned = L.row_size_aligned; const size_t src1_row_size_aligned = L.row_size_aligned; const size_t dst_row_size_aligned = L.row_size_aligned; // Make sure the reserved vtcm size is sufficient - if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { + if (kparams->vtcm_row_per_hvx == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, - (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_hvx); return HTP_STATUS_VTCM_TOO_SMALL; } if (src1) { FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, - L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_hvx * n_hvx, + L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } else { FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + L.src0_bytes_per_hvx * n_hvx, L.src1_bytes_per_hvx * n_hvx, L.dst_bytes_per_hvx * n_hvx); } // Prepare context @@ -1100,7 +1100,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { gctx.compute = (void *) compute_fn; gctx.op_str = op_type; - gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + gctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); gctx.src0_row_size = src0_row_size; gctx.src1_row_size = src1_row_size; @@ -1118,13 +1118,13 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); - gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread; - gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread; - gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; + gctx.vtcm_src0_size_per_hvx = (uint32_t) L.src0_bytes_per_hvx; + gctx.vtcm_src1_size_per_hvx = (uint32_t) L.src1_bytes_per_hvx; + gctx.vtcm_dst_size_per_hvx = (uint32_t) L.dst_bytes_per_hvx; - gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1; - gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1; - gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1; + gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_hvx >> 1; + gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_hvx >> 1; + gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_hvx >> 1; gctx.block = L.block; gctx.src0_nrows = nrows; @@ -1153,7 +1153,7 @@ static int execute_op_glu_f16(struct htp_ops_context * octx) { gctx.data_src1 = data_src1; gctx.data_dst = dst->data; - work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &gctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, (work_queue_func_t)glu_f16_per_hvx, &gctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.h b/ggml/src/ggml-hexagon/htp/glu-ops.h index 41b3857b1..5e248f7d4 100644 --- a/ggml/src/ggml-hexagon/htp/glu-ops.h +++ b/ggml/src/ggml-hexagon/htp/glu-ops.h @@ -7,8 +7,8 @@ #include struct htp_glu_kernel_params { - uint32_t n_threads; - uint32_t vtcm_row_per_thread; + uint32_t n_hvx; + uint32_t vtcm_row_per_hvx; uint32_t vtcm_size; }; @@ -41,34 +41,34 @@ struct htp_glu_vtcm_layout { size_t off_dst; size_t row_size_aligned; - size_t src0_bytes_per_thread; - size_t src1_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; - uint32_t vtcm_row_per_thread; + uint32_t vtcm_row_per_hvx; uint32_t block; }; static inline void htp_glu_vtcm_layout_build( struct htp_glu_vtcm_layout * L, size_t dst_row_size, - uint32_t n_threads, - uint32_t vtcm_row_per_thread + uint32_t n_hvx, + uint32_t vtcm_row_per_hvx ) { const size_t row_size_aligned = hex_round_up(dst_row_size, 128); L->row_size_aligned = row_size_aligned; - L->vtcm_row_per_thread = vtcm_row_per_thread; - L->block = vtcm_row_per_thread >> 1; + L->vtcm_row_per_hvx = vtcm_row_per_hvx; + L->block = vtcm_row_per_hvx >> 1; - L->src0_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; - L->src1_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; - L->dst_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; + L->src0_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; + L->src1_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; + L->dst_bytes_per_hvx = row_size_aligned * vtcm_row_per_hvx; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.h deleted file mode 100644 index 5c7a1a488..000000000 --- a/ggml/src/ggml-hexagon/htp/hex-profile.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef HEX_PROFILE_H -#define HEX_PROFILE_H - -#include -#include -#include - -#include "hex-utils.h" -#include "htp-ops.h" - -#define HTP_TRACE_EVT_START 0 -#define HTP_TRACE_EVT_STOP 1 - -#ifndef HEX_NUM_PMU_COUNTERS -#define HEX_NUM_PMU_COUNTERS 8 -#endif - -static inline void hex_get_pmu(uint32_t counters[]) { -#if __HVX_ARCH__ >= 79 - asm volatile("%0 = upmucnt0" : "=r"(counters[0])); - asm volatile("%0 = upmucnt1" : "=r"(counters[1])); - asm volatile("%0 = upmucnt2" : "=r"(counters[2])); - asm volatile("%0 = upmucnt3" : "=r"(counters[3])); - asm volatile("%0 = upmucnt4" : "=r"(counters[4])); - asm volatile("%0 = upmucnt5" : "=r"(counters[5])); - asm volatile("%0 = upmucnt6" : "=r"(counters[6])); - asm volatile("%0 = upmucnt7" : "=r"(counters[7])); -#else - counters[0] = qurt_pmu_get(QURT_PMUCNT0); - counters[1] = qurt_pmu_get(QURT_PMUCNT1); - counters[2] = qurt_pmu_get(QURT_PMUCNT2); - counters[3] = qurt_pmu_get(QURT_PMUCNT3); - counters[4] = qurt_pmu_get(QURT_PMUCNT4); - counters[5] = qurt_pmu_get(QURT_PMUCNT5); - counters[6] = qurt_pmu_get(QURT_PMUCNT6); - counters[7] = qurt_pmu_get(QURT_PMUCNT7); -#endif -} - -struct htp_thread_trace { - uint32_t count; - uint32_t max_events; - struct htp_trace_desc * events; -}; - -static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { - if (!tr->max_events) { - return; - } - uint32_t i = tr->count++; - if (i < tr->max_events) { - tr->events[i].id = id; - tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); - tr->events[i].cycles = (uint32_t) hex_get_cycles(); - } -} - -static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { - htp_trace_event(tr, id, info, HTP_TRACE_EVT_START); -} - -static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { - htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP); -} - -#endif /* HEX_PROFILE_H */ diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index abb44da25..cb3093dec 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -25,6 +25,32 @@ static inline uint64_t hex_get_pktcnt() { return pktcnt; } +#ifndef HEX_NUM_PMU_COUNTERS +#define HEX_NUM_PMU_COUNTERS 8 +#endif + +static inline void hex_get_pmu(uint32_t counters[]) { +#if __HVX_ARCH__ >= 79 + asm volatile("%0 = upmucnt0" : "=r"(counters[0])); + asm volatile("%0 = upmucnt1" : "=r"(counters[1])); + asm volatile("%0 = upmucnt2" : "=r"(counters[2])); + asm volatile("%0 = upmucnt3" : "=r"(counters[3])); + asm volatile("%0 = upmucnt4" : "=r"(counters[4])); + asm volatile("%0 = upmucnt5" : "=r"(counters[5])); + asm volatile("%0 = upmucnt6" : "=r"(counters[6])); + asm volatile("%0 = upmucnt7" : "=r"(counters[7])); +#else + counters[0] = qurt_pmu_get(QURT_PMUCNT0); + counters[1] = qurt_pmu_get(QURT_PMUCNT1); + counters[2] = qurt_pmu_get(QURT_PMUCNT2); + counters[3] = qurt_pmu_get(QURT_PMUCNT3); + counters[4] = qurt_pmu_get(QURT_PMUCNT4); + counters[5] = qurt_pmu_get(QURT_PMUCNT5); + counters[6] = qurt_pmu_get(QURT_PMUCNT6); + counters[7] = qurt_pmu_get(QURT_PMUCNT7); +#endif +} + static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, uint32_t height) { const uint64_t control = Q6_P_combine_RR(stride, Q6_R_combine_RlRl(width, height)); Q6_l2fetch_AP((void *) p, control); @@ -37,11 +63,6 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) { hex_l2fetch(addr, width, width, height); } -#define HEX_L2_LINE_SIZE 128 -#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread) -#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) -#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) - static inline void hex_l2flush(void * addr, size_t size) { if (size == 0) return; const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1); diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 65f90c17a..590167492 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -18,8 +18,6 @@ typedef struct tiled_dequantize_state tiled_dequantize_state_t; typedef void (*tiled_dequant_task_fn_t)(const tiled_dequantize_state_t * state, uint32_t start_tile, uint32_t end_tile); struct tiled_dequantize_state { - struct htp_context * ctx; - struct htp_thread_trace * traces; __fp16 * dst; const uint8_t * src; tiled_dequant_task_fn_t task_fn; @@ -1235,7 +1233,6 @@ typedef struct { uint32_t dst_stride; // DDR row stride uint32_t src2_stride; // DDR row stride for residual uint32_t dst_cols; // Actual output columns - struct htp_thread_trace * traces; } output_transfer_task_state_t; // activations : fp32 -> fp16 diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.c b/ggml/src/ggml-hexagon/htp/hmx-queue.c index c369d3dd2..e40a28bed 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.c +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.c @@ -46,9 +46,9 @@ static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { case HMX_QUEUE_WAKEUP: hmx_lock(q); break; default: hmx_lock(q); - htp_trace_event_start(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); - d->func(d->data); - htp_trace_event_stop(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); + htp_trace_event_start(q->th.trace, HTP_TRACE_EVT_HMX_COMP, ir); + d->func(&q->th, d->data); + htp_trace_event_stop(q->th.trace, HTP_TRACE_EVT_HMX_COMP, ir); break; } @@ -63,7 +63,7 @@ static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { static void hmx_queue_thread(void * arg) { hmx_queue_t q = (hmx_queue_t) arg; - FARF(HIGH, "hmx-queue-thread: started"); + FARF(HIGH, "hmx-queue-thread (tid %u): started", q->th.tid); bool killed = false; @@ -90,7 +90,7 @@ static void hmx_queue_thread(void * arg) { hmx_queue_process(q, &killed); } - FARF(HIGH, "hmx-queue-thread: stopped"); + FARF(HIGH, "hmx-queue-thread (tid %u): stopped", q->th.tid); } size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) { @@ -105,7 +105,7 @@ size_t hmx_queue_alignof(void) { return HEX_L2_LINE_SIZE; } -hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) { +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, dma_queue_t * dma, struct htp_thread_trace * trace) { capacity = hex_ceil_pow2(capacity); size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); @@ -126,7 +126,7 @@ hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uin q->stack = block; memset(q->stack, 0, stack_size); - q->trace = trace; + htp_thread_init(&q->th, dma, trace); // Match caller thread priority (same pattern as worker-pool.c). int prio = qurt_thread_get_priority(qurt_thread_get_id()); diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.h b/ggml/src/ggml-hexagon/htp/hmx-queue.h index c2b1859a2..efefb1455 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.h +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.h @@ -11,7 +11,8 @@ #include #include "hex-utils.h" -#include "hex-profile.h" +#include "htp-trace.h" +#include "htp-thread.h" #ifdef __cplusplus extern "C" { @@ -23,7 +24,7 @@ extern "C" { #define HMX_QUEUE_POLL_COUNT 1 #endif -typedef void (*hmx_queue_func)(void *); +typedef void (*hmx_queue_func)(const htp_thread_t * th, void *); // Dummy funcs used as signals enum hmx_queue_signal { @@ -49,10 +50,10 @@ struct hmx_queue_s { atomic_uint seqn; // incremented for all pushes, used with futex qurt_thread_t thread; + htp_thread_t th; void * stack; uint32_t hap_rctx; bool hmx_locked; - struct htp_thread_trace * trace; bool external_mem; // memory owned externally }; @@ -60,7 +61,7 @@ typedef struct hmx_queue_s * hmx_queue_t; size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size); size_t hmx_queue_alignof(void); -hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace); +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, dma_queue_t * dma, struct htp_thread_trace * trace); void hmx_queue_free(hmx_queue_t q); static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) { diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index 5c35cd988..721fa2162 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -5,7 +5,7 @@ #include "hmx-queue.h" #include "htp-dcache.h" #include "htp-ops.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "work-queue.h" #include "hex-fastdiv.h" @@ -17,10 +17,6 @@ #include #include -#ifndef HTP_MAX_NTHREADS -#define HTP_MAX_NTHREADS 10 -#endif - #define HTP_MAX_MMAPS 64 // Memory mapping @@ -37,7 +33,7 @@ struct htp_spad { uint8_t * data; // pointer to an area in vtcm uint32_t stride; // stride used inside this spad uint32_t size; // total size - uint32_t size_per_thread; // size per thread + uint32_t size_per_hvx; // size per hvx }; struct htp_mdev_group { @@ -71,14 +67,14 @@ struct htp_ops_context { struct htp_spad dst_spad; uint32_t flags; - uint32_t n_threads; - struct fastdiv_values n_threads_div; + uint32_t n_hvx; + struct fastdiv_values n_hvx_div; int status; // Must stay last: not cleared by htp_ops_context_reset(), ops init the caches they use union { - dma_cache_dm dma_cache_dm[HTP_MAX_NTHREADS]; - dma_cache_fa dma_cache_fa[HTP_MAX_NTHREADS]; + dma_cache_dm dma_cache_dm[HTP_MAX_NHVX]; + dma_cache_fa dma_cache_fa[HTP_MAX_NHVX]; } __attribute__((aligned(128))); }; @@ -88,12 +84,13 @@ struct htp_context { struct htp_mmap mmap[HTP_MAX_MMAPS]; dma_queue_t dma[HTP_MAX_NTHREADS]; - struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; - work_queue_t work_queue; + struct htp_thread_trace trace[HTP_MAX_NTHREADS]; + work_queue_t hvx_queue; hmx_queue_t hmx_queue; uint32_t n_threads; - struct fastdiv_values n_threads_div; + uint32_t n_hvx; + struct fastdiv_values n_hvx_div; int thread_id; int thread_prio; @@ -126,37 +123,37 @@ struct htp_context { static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct htp_context * ctx) { memset(octx, 0, offsetof(struct htp_ops_context, dma_cache_dm)); - octx->ctx = ctx; - octx->n_threads = ctx->n_threads; - octx->n_threads_div = ctx->n_threads_div; + octx->ctx = ctx; + octx->n_hvx = ctx->n_hvx; + octx->n_hvx_div = ctx->n_hvx_div; } -static inline bool htp_ops_context_set_n_threads_div( +static inline bool htp_ops_context_set_n_hvx_div( struct htp_ops_context * octx, - uint32_t n_threads, - struct fastdiv_values n_threads_div) { - if (n_threads == 0 || n_threads > octx->ctx->n_threads) { + uint32_t n_hvx, + struct fastdiv_values n_hvx_div) { + if (n_hvx == 0 || n_hvx > octx->ctx->n_hvx) { return false; } - octx->n_threads = n_threads; - octx->n_threads_div = n_threads_div; + octx->n_hvx = n_hvx; + octx->n_hvx_div = n_hvx_div; return true; } -static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { - if (n_threads == 0 || n_threads > octx->ctx->n_threads) { +static inline bool htp_ops_context_set_n_hvx(struct htp_ops_context * octx, uint32_t n_hvx) { + if (n_hvx == 0 || n_hvx > octx->ctx->n_hvx) { return false; } - if (n_threads == octx->n_threads) { + if (n_hvx == octx->n_hvx) { return true; } - const struct fastdiv_values n_threads_div = n_threads == octx->ctx->n_threads - ? octx->ctx->n_threads_div - : init_fastdiv_values(n_threads); - return htp_ops_context_set_n_threads_div(octx, n_threads, n_threads_div); + const struct fastdiv_values n_hvx_div = n_hvx == octx->ctx->n_hvx + ? octx->ctx->n_hvx_div + : init_fastdiv_values(n_hvx); + return htp_ops_context_set_n_hvx_div(octx, n_hvx, n_hvx_div); } static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) { diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.c b/ggml/src/ggml-hexagon/htp/htp-dcache.c index 840b1edb7..52fc8e403 100644 --- a/ggml/src/ggml-hexagon/htp/htp-dcache.c +++ b/ggml/src/ggml-hexagon/htp/htp-dcache.c @@ -7,7 +7,7 @@ #include "hex-common.h" #include "hex-utils.h" #include "hex-fastdiv.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "work-queue.h" @@ -15,15 +15,14 @@ struct l2flush_range { uint32_t start; // line-aligned start address uint32_t end; // line-aligned end address uint32_t block_first; // global block index of this range's first block - uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial) + uint32_t n_blocks; // number of HTP_DCACHE_BLOCK_SIZE chunks (last may be partial) }; struct l2flush_multi_task { - struct htp_thread_trace * trace; struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES]; uint32_t n_ranges; uint32_t total_blocks; - uint32_t blocks_per_thread; + uint32_t blocks_per_hvx; }; static void flush_all_dcache(struct htp_context * ctx) { @@ -35,12 +34,12 @@ static void flush_all_dcache(struct htp_context * ctx) { memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); } -static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { +static void l2flush_multi_worker(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { struct l2flush_multi_task * task = (struct l2flush_multi_task *) data; (void) n; - const uint32_t gb_first = i * task->blocks_per_thread; - uint32_t gb_last = gb_first + task->blocks_per_thread; + const uint32_t gb_first = i * task->blocks_per_hvx; + uint32_t gb_last = gb_first + task->blocks_per_hvx; if (gb_last > task->total_blocks) { gb_last = task->total_blocks; } @@ -48,7 +47,7 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = &task->trace[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); for (uint32_t r = 0; r < task->n_ranges; r++) { @@ -62,8 +61,8 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { continue; } - const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE; - uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE; + const uint32_t s = rg->start + (lo - rb_first) * HTP_DCACHE_BLOCK_SIZE; + uint32_t e = rg->start + (hi - rb_first) * HTP_DCACHE_BLOCK_SIZE; if (e > rg->end) { e = rg->end; } @@ -163,7 +162,7 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te total_evict_size += r->end - r->start; } - if (total_evict_size > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_evict_size > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); for (uint32_t i = 0; i < n_pending; i++) { struct htp_dirty_range * r = &ctx->dirty_ranges[i]; @@ -174,9 +173,8 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te return; } - if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { + if (total_evict_size > HTP_DCACHE_FLUSH_WQ_THRESHOLD && ctx->n_hvx > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { struct l2flush_multi_task task; - task.trace = ctx->trace; task.n_ranges = n_evict; uint32_t block_acc = 0; @@ -188,14 +186,14 @@ void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_te rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + rg->n_blocks = (rg->end - rg->start + HTP_DCACHE_BLOCK_SIZE - 1) / HTP_DCACHE_BLOCK_SIZE; block_acc += rg->n_blocks; } - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + task.total_blocks = block_acc; + task.blocks_per_hvx = fastdiv(block_acc + ctx->n_hvx - 1, &ctx->n_hvx_div); - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + work_queue_run(ctx->hvx_queue, l2flush_multi_worker, &task, ctx->n_hvx); } else { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); @@ -270,9 +268,8 @@ static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_te } static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) { - if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + if (total_dirty >= HTP_DCACHE_FLUSH_WQ_THRESHOLD && ctx->n_hvx > 1) { struct l2flush_multi_task task; - task.trace = ctx->trace; task.n_ranges = n_ranges; uint32_t block_acc = 0; @@ -282,14 +279,14 @@ static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_ rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + rg->n_blocks = (rg->end - rg->start + HTP_DCACHE_BLOCK_SIZE - 1) / HTP_DCACHE_BLOCK_SIZE; block_acc += rg->n_blocks; } - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + task.total_blocks = block_acc; + task.blocks_per_hvx = fastdiv(block_acc + ctx->n_hvx - 1, &ctx->n_hvx_div); - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + work_queue_run(ctx->hvx_queue, l2flush_multi_worker, &task, ctx->n_hvx); } else { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); @@ -319,7 +316,7 @@ void htp_dcache_flush_dirty_ranges(struct htp_context * ctx) { return; } - if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_dirty > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); return; } @@ -348,7 +345,7 @@ void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_t return; } - if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + if (total_dirty > HTP_DCACHE_FLUSH_ALL_THRESHOLD) { flush_all_dcache(ctx); return; } diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.h b/ggml/src/ggml-hexagon/htp/htp-dcache.h index 1996ab258..c1c37573c 100644 --- a/ggml/src/ggml-hexagon/htp/htp-dcache.h +++ b/ggml/src/ggml-hexagon/htp/htp-dcache.h @@ -2,9 +2,13 @@ #define HTP_DCACHE_H #include +#include "hex-common.h" #include "htp-ops.h" -#define HTP_MAX_DIRTY_RANGES 64 +#define HTP_MAX_DIRTY_RANGES 64 +#define HTP_DCACHE_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread) +#define HTP_DCACHE_FLUSH_WQ_THRESHOLD (4 * 1024) +#define HTP_DCACHE_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) struct htp_dirty_range { uint32_t start; diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index a3ff51ebe..7c4b97307 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -127,14 +127,17 @@ enum htp_op_code { HTP_OP_INVALID }; -#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS -#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS -#define HTP_OP_MAX_OUTPUTS 4 -#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS +#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS +#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS +#define HTP_OP_MAX_OUTPUTS 4 +#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS #define HTP_OP_MAX_KERN_PARAMS 32 -#define HTP_OP_MAX_BUFS 64 -#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16) +#define HTP_OP_MAX_BUFS 64 +#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16) + +#define HTP_OP_MAX_NTHREADS 10 +#define HTP_OP_MAX_NHVX 8 #define HTP_FENCE_TIMEOUT (1000000000ULL) #define HTP_FENCE_SLOT_SIZE 128 @@ -191,8 +194,20 @@ struct htp_op_desc { uint16_t pad[2]; // padding to align to 64 bits }; -#ifndef HTP_MAX_NTHREADS +#ifdef __HVX_ARCH__ +#if __HVX_ARCH__ <= 73 +#define HTP_MAX_NTHREADS 6 +#define HTP_MAX_NHVX 4 +#elif __HVX_ARCH__ <= 75 +#define HTP_MAX_NTHREADS 8 +#define HTP_MAX_NHVX 6 +#else #define HTP_MAX_NTHREADS 10 +#define HTP_MAX_NHVX 8 +#endif + +_Static_assert(HTP_MAX_NTHREADS <= HTP_OP_MAX_NTHREADS, "HTP_MAX_NTHREADS exceeds HTP_OP_MAX_NTHREADS"); +_Static_assert(HTP_MAX_NHVX <= HTP_OP_MAX_NHVX, "HTP_MAX_NHVX exceeds HTP_OP_MAX_NHVX"); #endif #define HTP_TRACE_MAX_EVENTS 256 @@ -285,7 +300,7 @@ struct htp_opbatch_rsp { uint32_t n_tensors; // Number of tensors uint32_t n_ops; // Number of op profile descriptors uint32_t usecs; // Number of usec - uint32_t n_traces[HTP_MAX_NTHREADS + 1]; + uint32_t n_traces[HTP_OP_MAX_NTHREADS]; // struct htp_prof_desc profs[]; -- dspqueue buf 0 }; diff --git a/ggml/src/ggml-hexagon/htp/htp-thread.h b/ggml/src/ggml-hexagon/htp/htp-thread.h new file mode 100644 index 000000000..e4ca874de --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-thread.h @@ -0,0 +1,33 @@ +#ifndef HTP_THREAD_H +#define HTP_THREAD_H + +#include +#include + +#include "dma-queue.h" +#include "htp-trace.h" + +typedef struct { + unsigned int tid; + dma_queue * dma_q; + struct htp_thread_trace * trace; +} htp_thread_t; + +extern atomic_uint htp_thread_count; + +static inline void htp_thread_reset(void) { + atomic_store_explicit(&htp_thread_count, 0, memory_order_relaxed); +} + +static inline unsigned int htp_thread_alloc_id(void) { + return atomic_fetch_add_explicit(&htp_thread_count, 1, memory_order_relaxed); +} + +static inline void htp_thread_init(htp_thread_t * th, dma_queue_t * dma, struct htp_thread_trace * trace) { + unsigned int tid = htp_thread_alloc_id(); + th->tid = tid; + th->dma_q = dma ? dma[tid] : NULL; + th->trace = trace ? &trace[tid] : NULL; +} + +#endif /* HTP_THREAD_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-trace.h b/ggml/src/ggml-hexagon/htp/htp-trace.h new file mode 100644 index 000000000..66bd9110d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-trace.h @@ -0,0 +1,38 @@ +#ifndef HTP_TRACE_H +#define HTP_TRACE_H + +#include + +#include "hex-utils.h" +#include "htp-ops.h" + +#define HTP_TRACE_EVT_START 0 +#define HTP_TRACE_EVT_STOP 1 + +struct htp_thread_trace { + uint32_t count; + uint32_t max_events; + struct htp_trace_desc * events; +}; + +static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { + if (!tr->max_events) { + return; + } + uint32_t i = tr->count++; + if (i < tr->max_events) { + tr->events[i].id = id; + tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); + tr->events[i].cycles = (uint32_t) hex_get_cycles(); + } +} + +static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { + htp_trace_event(tr, id, info, HTP_TRACE_EVT_START); +} + +static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { + htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP); +} + +#endif /* HTP_TRACE_H */ diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c index 9e662694b..602ee1598 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.c +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c @@ -15,7 +15,7 @@ #include "htp-ops.h" #include "hvx-utils.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-vtcm.h" #include "htp-tensor.h" #include "im2col-ops.h" @@ -25,16 +25,16 @@ struct htp_im2col_context { uint32_t pe_row_base; // first output row index assigned to this dev (DMA path) uint32_t pe_nrows; // number of output rows assigned to this dev (DMA path) - uint32_t pe_rows_per_thread; // output rows per worker - uint32_t pe_blocks_per_thread; // output-col blocks per worker (blocked DMA path) + uint32_t pe_rows_per_hvx; // output rows per worker + uint32_t pe_blocks_per_hvx; // output-col blocks per worker (blocked DMA path) uint32_t pe_src_row_bytes; // one output row's source, rounded 256 uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 // Patch-embed DMA path VTCM ping-pong. uint8_t * pe_vtcm_src; // base of the 2x src buffers region uint8_t * pe_vtcm_dst; // base of the 2x dst buffers region - uint32_t pe_src_size_per_thread; // 2 * pe_src_row_bytes - uint32_t pe_dst_size_per_thread; // 2 * pe_dst_row_bytes + uint32_t pe_src_size_per_hvx; // 2 * pe_src_row_bytes + uint32_t pe_dst_size_per_hvx; // 2 * pe_dst_row_bytes uint32_t pe_owb; // output-col block size uint32_t pe_wb; // staged source window width @@ -50,10 +50,10 @@ struct htp_im2col_context { uint32_t patch_stride; // Per-thread initial coordinates (removes division from thread prologue) - uint32_t thread_in[HTP_MAX_NTHREADS]; - uint32_t thread_iod[HTP_MAX_NTHREADS]; - uint32_t thread_ioh[HTP_MAX_NTHREADS]; - uint32_t thread_c0[HTP_MAX_NTHREADS]; + uint32_t thread_in[HTP_MAX_NHVX]; + uint32_t thread_iod[HTP_MAX_NHVX]; + uint32_t thread_ioh[HTP_MAX_NHVX]; + uint32_t thread_c0[HTP_MAX_NHVX]; void * compute; uint32_t dst_elem; @@ -321,11 +321,11 @@ static void FNAME( IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f16, uint16_t, hvx_copy_f16_f32_uu) IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f32, float, hvx_copy_f32_uu) -static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void im2col_blocked_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; struct htp_ops_context * octx = ictx->octx; - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * restrict tr = th->trace; const uint32_t OW = ictx->OW; const uint32_t KH = ictx->KH; const uint32_t owb = ictx->pe_owb; @@ -334,16 +334,16 @@ static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * const uint32_t dst_elem = ictx->dst_elem; im2col_blocked_compute_fn_t compute = (im2col_blocked_compute_fn_t) ictx->compute; const dma_addr_t dst_data = octx->dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; - uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes); // work is split over output-col blocks, so a single output row (1D) still uses every thread const uint32_t blk_end_max = ictx->pe_nrows * nbpr; - const uint32_t blk_start = ictx->pe_blocks_per_thread * ith; - const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_thread, blk_end_max); + const uint32_t blk_start = ictx->pe_blocks_per_hvx * ith; + const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_hvx, blk_end_max); if (blk_start >= blk_end) { return; } @@ -510,24 +510,24 @@ static void im2col_patchembed_compute_f32( } } -static void im2col_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void im2col_patchembed_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; struct htp_ops_context * octx = ictx->octx; - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * restrict tr = th->trace; const uint32_t OW = ictx->OW; const uint32_t patch_stride = ictx->patch_stride; const uint32_t dst_elem = ictx->dst_elem; im2col_patchembed_compute_fn_t compute = (im2col_patchembed_compute_fn_t) ictx->compute; const dma_addr_t dst_data = octx->dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; - uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; - const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; - const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); if (row_start >= row_end) { return; } @@ -718,11 +718,11 @@ static void FNAME(const struct htp_im2col_context * ictx, IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u) IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u) -static void im2col_3d_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void im2col_3d_blocked_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; struct htp_ops_context * octx = ictx->octx; - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * restrict tr = th->trace; const struct htp_tensor * restrict dst = octx->dst; const uint32_t OW = ictx->OW; const uint32_t KD = ictx->KD, KH = ictx->KH; @@ -732,15 +732,15 @@ static void im2col_3d_blocked_dma_thread(unsigned int nth, unsigned int ith, voi const uint32_t nbpr = ictx->pe_nbpr; const uint32_t dst_elem = ictx->dst_elem; const dma_addr_t dst_data = dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; - uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + dma_queue * dma_q = th->dma_q; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes); const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; - const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; - const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); if (row_start >= row_end) { return; } @@ -860,28 +860,28 @@ static void im2col_3d_patchembed_compute_f32( } } -static void im2col_3d_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) { +static void im2col_3d_patchembed_dma_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; struct htp_ops_context * octx = ictx->octx; - struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * restrict tr = th->trace; const struct htp_tensor * restrict dst = octx->dst; const uint32_t KD = ictx->KD; const uint32_t OW = ictx->OW; const uint32_t patch_stride = ictx->patch_stride; const uint32_t dst_elem = ictx->dst_elem; const dma_addr_t dst_data = dst->data; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; - const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; - const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_hvx, row_end_max); if (row_start >= row_end) { return; } const uint32_t nrows_local = row_end - row_start; - uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; - uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_hvx; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_hvx; float * srcb2[2] = { (float *) src_base, (float *) (src_base + ictx->pe_src_row_bytes) }; void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; struct im2col_cursor cur = { @@ -948,29 +948,29 @@ int op_im2col(struct htp_ops_context * octx) { // the blocked path splits work over output-col blocks, so a single output row (1D) can use every thread const bool blocked = kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA; const uint32_t nwork = blocked ? nrows * kparams->nbpr : nrows; - if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nwork))) { + if (!htp_ops_context_set_n_hvx(octx, MIN((uint32_t) kparams->n_hvx, nwork))) { return HTP_STATUS_INVAL_PARAMS; } struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads, + htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_hvx, htp_im2col_scratch_bytes(kparams->kernel_type, kparams->KH, kparams->patch_stride)); struct htp_im2col_context ictx = { 0 }; uint8_t * const base = octx->ctx->vtcm_base; - const uint32_t pth = octx->n_threads; + const uint32_t pth = octx->n_hvx; ictx.octx = octx; ictx.pe_row_base = row_base; ictx.pe_nrows = nrows; - ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div); - ictx.pe_blocks_per_thread = fastdiv(nwork + pth - 1, &octx->n_threads_div); + ictx.pe_rows_per_hvx = fastdiv(nrows + pth - 1, &octx->n_hvx_div); + ictx.pe_blocks_per_hvx = fastdiv(nwork + pth - 1, &octx->n_hvx_div); ictx.pe_src_row_bytes = kparams->src_row_bytes; ictx.pe_dst_row_bytes = kparams->dst_row_bytes; ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src); ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst); - ictx.pe_src_size_per_thread = layout.src_bytes_per_thread; - ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread; + ictx.pe_src_size_per_hvx = layout.src_bytes_per_hvx; + ictx.pe_dst_size_per_hvx = layout.dst_bytes_per_hvx; ictx.pe_owb = kparams->owb; ictx.pe_wb = kparams->wb; ictx.pe_nbpr = kparams->nbpr; @@ -990,10 +990,10 @@ int op_im2col(struct htp_ops_context * octx) { // a thread starts at a row, or at an output-col block (possibly mid-row) on the blocked path for (uint32_t ith = 0; ith < pth; ith++) { - uint32_t r = row_base + ictx.pe_rows_per_thread * ith; + uint32_t r = row_base + ictx.pe_rows_per_hvx * ith; uint32_t c0 = 0; if (blocked) { - const uint32_t blk = ictx.pe_blocks_per_thread * ith; + const uint32_t blk = ictx.pe_blocks_per_hvx * ith; const uint32_t q = fastdiv(blk, &kparams->div_nbpr); r = row_base + q; c0 = (blk - q * kparams->nbpr) * kparams->owb; @@ -1009,7 +1009,7 @@ int op_im2col(struct htp_ops_context * octx) { if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) { ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_patchembed_compute_f16 : (void *) im2col_patchembed_compute_f32; - work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth); + work_queue_run(octx->ctx->hvx_queue, im2col_patchembed_dma_thread, &ictx, pth); return HTP_STATUS_OK; } else if (blocked) { if (kparams->KH == 1) { @@ -1019,7 +1019,7 @@ int op_im2col(struct htp_ops_context * octx) { ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16 : (void *) im2col_blocked_compute_f32; } - work_queue_run(octx->ctx->work_queue, im2col_blocked_dma_thread, &ictx, pth); + work_queue_run(octx->ctx->hvx_queue, im2col_blocked_dma_thread, &ictx, pth); return HTP_STATUS_OK; } @@ -1049,27 +1049,27 @@ int op_im2col_3d(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN((uint32_t) kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } struct htp_im2col_vtcm_layout layout; - htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads, 256); + htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_hvx, 256); struct htp_im2col_context ictx = { 0 }; uint8_t * const base = octx->ctx->vtcm_base; - const uint32_t pth = octx->n_threads; + const uint32_t pth = octx->n_hvx; ictx.octx = octx; ictx.pe_row_base = row_base; ictx.pe_nrows = nrows; - ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div); + ictx.pe_rows_per_hvx = fastdiv(nrows + pth - 1, &octx->n_hvx_div); ictx.pe_src_row_bytes = kparams->src_row_bytes; ictx.pe_dst_row_bytes = kparams->dst_row_bytes; ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src); ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst); - ictx.pe_src_size_per_thread = layout.src_bytes_per_thread; - ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread; + ictx.pe_src_size_per_hvx = layout.src_bytes_per_hvx; + ictx.pe_dst_size_per_hvx = layout.dst_bytes_per_hvx; ictx.pe_owb = kparams->owb; ictx.pe_wb = kparams->wb; ictx.pe_nbpr = kparams->nbpr; @@ -1087,7 +1087,7 @@ int op_im2col_3d(struct htp_ops_context * octx) { ictx.patch_stride = kparams->patch_stride; for (uint32_t ith = 0; ith < pth; ith++) { - const uint32_t rem = row_base + ictx.pe_rows_per_thread * ith; + const uint32_t rem = row_base + ictx.pe_rows_per_hvx * ith; const uint32_t q_oh = fastdiv(rem, &kparams->div_OH); ictx.thread_ioh[ith] = rem - q_oh * kparams->OH; const uint32_t in = fastdiv(q_oh, &kparams->div_OD); @@ -1100,12 +1100,12 @@ int op_im2col_3d(struct htp_ops_context * octx) { if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) { ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_patchembed_compute_f16 : (void *) im2col_3d_patchembed_compute_f32; - work_queue_run(octx->ctx->work_queue, im2col_3d_patchembed_dma_thread, &ictx, pth); + work_queue_run(octx->ctx->hvx_queue, im2col_3d_patchembed_dma_thread, &ictx, pth); return HTP_STATUS_OK; } else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) { ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_blocked_compute_f16 : (void *) im2col_3d_blocked_compute_f32; - work_queue_run(octx->ctx->work_queue, im2col_3d_blocked_dma_thread, &ictx, pth); + work_queue_run(octx->ctx->hvx_queue, im2col_3d_blocked_dma_thread, &ictx, pth); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.h b/ggml/src/ggml-hexagon/htp/im2col-ops.h index b34e6a09d..23497d01d 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.h +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.h @@ -16,28 +16,28 @@ struct htp_im2col_vtcm_layout { size_t total_bytes; size_t off_src; size_t off_dst; - uint32_t src_bytes_per_thread; - uint32_t dst_bytes_per_thread; + uint32_t src_bytes_per_hvx; + uint32_t dst_bytes_per_hvx; }; -// scratch_bytes: per-thread VTCM after the two src buffers (at least 256) +// scratch_bytes: per-hvx VTCM after the two src buffers (at least 256) static inline void htp_im2col_vtcm_layout_build( struct htp_im2col_vtcm_layout * L, uint32_t src_row_bytes, uint32_t dst_row_bytes, - uint32_t n_threads, + uint32_t n_hvx, uint32_t scratch_bytes) { - L->src_bytes_per_thread = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256); - L->dst_bytes_per_thread = 2 * dst_row_bytes; + L->src_bytes_per_hvx = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256); + L->dst_bytes_per_hvx = 2 * dst_row_bytes; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } -// per-thread scratch: the KH == 1 blocked path keeps its offset table and one gathered row there, +// per-hvx scratch: the KH == 1 blocked path keeps its offset table and one gathered row there, // both padded to whole vectors static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t KH, uint32_t patch_stride) { if (kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA && KH == 1) { @@ -48,7 +48,7 @@ static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t K struct htp_im2col_kernel_params { uint8_t kernel_type; - uint8_t n_threads; + uint8_t n_hvx; uint32_t vtcm_size; uint32_t src_row_bytes; diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c index c8a42eb4e..5d60c506e 100644 --- a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c +++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c @@ -6,7 +6,7 @@ #include #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "hvx-utils.h" #include "hvx-copy.h" #include "htp-vtcm.h" @@ -30,7 +30,7 @@ struct htp_lid_context { const struct htp_lid_kernel_params * kp; uint32_t n_blocks; - uint32_t blocks_per_thread; + uint32_t blocks_per_hvx; // current stream and token tile uint32_t is; @@ -72,7 +72,7 @@ static inline void lid_prep_push_step(dma_queue * dma_q, const struct htp_lid_co // Splat every Q element (f16) and every head weight (f32) into a full vector. // Work is split over (token, head) pairs. -static void lid_prep_thread(unsigned int nth, unsigned int ith, void * data) { +static void lid_prep_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_lid_context * lctx = (struct htp_lid_context *) data; const struct htp_ops_context * octx = lctx->octx; const struct htp_tensor * q = octx->src[0]; @@ -83,9 +83,9 @@ static void lid_prep_thread(unsigned int nth, unsigned int ith, void * data) { const uint32_t D = q->ne[0]; const uint32_t n_head = q->ne[1]; - // nth is min(n_threads, n_items): the same chunk as a split over n_threads + // nth is min(n_hvx, n_items): the same chunk as a split over n_hvx const uint32_t n_items = lctx->nt * n_head; - const uint32_t chunk = fastdiv(n_items + octx->n_threads - 1, &octx->n_threads_div); + const uint32_t chunk = fastdiv(n_items + octx->n_hvx - 1, &octx->n_hvx_div); const uint32_t i0 = ith * chunk; const uint32_t i1 = MIN(i0 + chunk, n_items); @@ -93,8 +93,8 @@ static void lid_prep_thread(unsigned int nth, unsigned int ith, void * data) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; const uint32_t w_row_size = hex_round_up(n_head * sizeof(float), 128); const size_t qs_size = n_head * L->size_qs_row; @@ -506,7 +506,7 @@ static inline void lid_compute_row2(float * restrict o0, } // Each thread owns a range of kv blocks and streams K / mask in, output out (HTP_LID_N_BUF buffers) -static void lid_compute_thread(unsigned int nth, unsigned int ith, void * data) { +static void lid_compute_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_lid_context * lctx = (struct htp_lid_context *) data; const struct htp_ops_context * octx = lctx->octx; const struct htp_tensor * q = octx->src[0]; @@ -514,8 +514,8 @@ static void lid_compute_thread(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * m = octx->src[3]; const struct htp_tensor * dst = octx->dst; - const uint32_t b0 = lctx->blocks_per_thread * ith; - const uint32_t b1 = MIN(b0 + lctx->blocks_per_thread, lctx->n_blocks); + const uint32_t b0 = lctx->blocks_per_hvx * ith; + const uint32_t b1 = MIN(b0 + lctx->blocks_per_hvx, lctx->n_blocks); if (b0 >= b1) { return; @@ -556,8 +556,8 @@ static void lid_compute_thread(unsigned int nth, unsigned int ith, void * data) HVX_Vector * sc = kt_t1 + HTP_LID_KV_BLOCK; const HVX_Vector g_offs = hvx_vec_gather_offsets_h(L->size_k_raw_row); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; for (uint32_t b = b0, i = 0; b < b1 && i < HTP_LID_N_BUF; ++b, ++i) { const uint32_t kv0 = b * HTP_LID_KV_BLOCK; @@ -796,7 +796,7 @@ static inline void lid_hmx_dot(const __fp16 * a, const __fp16 * b, uint32_t n) { } // Per segment: channel tile outer, row tile inner, one bias load per channel tile. S holds [n_rt][n_sg] tiles. -static void lid_hmx_worker(void * data) { +static void lid_hmx_worker(const htp_thread_t * th, void * data) { const lid_hmx_job_t * job = (const lid_hmx_job_t *) data; const size_t dot_stride = (size_t) job->n_dot * HMX_FP16_TILE_N_ELMS; @@ -837,7 +837,7 @@ static void lid_hmx_worker(void * data) { // Thread slices: K rows by row pairs of the padded block, mask / output rows by token. // Block 0 is prepped together with Q: its K rows go to the threads without Q work. -// K rows are split only in stages on all n_threads threads. +// K rows are split only in stages on all n_hvx threads. static inline void lid_hmx_rows(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g, uint32_t n_rows, uint32_t * r0, uint32_t * r1) { uint32_t per; @@ -849,22 +849,22 @@ static inline void lid_hmx_rows(const struct htp_lid_hmx_context * hctx, uint32_ ith -= hctx->n_qthr; per = (n_rows + nth - hctx->n_qthr - 1) / (nth - hctx->n_qthr); } else { - per = fastdiv(n_rows + hctx->octx->n_threads - 1, &hctx->octx->n_threads_div); + per = fastdiv(n_rows + hctx->octx->n_hvx - 1, &hctx->octx->n_hvx_div); } per = hex_round_up(per, 2); *r0 = MIN(ith * per, n_rows); *r1 = MIN(*r0 + per, n_rows); } -// Split over n_threads; a stage on one thread has one token, which this split also gives to thread 0 +// Split over n_hvx; a stage on one thread has one token, which this split also gives to thread 0 static inline void lid_hmx_toks(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nt, uint32_t * t0, uint32_t * t1) { - const uint32_t per = fastdiv(nt + hctx->octx->n_threads - 1, &hctx->octx->n_threads_div); + const uint32_t per = fastdiv(nt + hctx->octx->n_hvx - 1, &hctx->octx->n_hvx_div); *t0 = MIN(ith * per, nt); *t1 = MIN((ith + 1) * per, nt); } // Push (or only count, push == false) the K row and mask DMAs of block g for this thread, one per segment -static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g, bool push) { +static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g, bool push) { const struct htp_ops_context * octx = hctx->octx; const struct htp_tensor * k = octx->src[1]; const struct htp_tensor * m = octx->src[3]; @@ -878,7 +878,7 @@ static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, uint32_t ith lid_hmx_rows(hctx, ith, nth, g, n_rows, &r0, &r1); lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t k_row_size = hctx->kp->k_row_size; uint32_t n = 0; @@ -907,8 +907,8 @@ static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, uint32_t ith return n; } -static inline uint32_t lid_hmx_push_in(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { - return lid_hmx_in(hctx, ith, nth, g, true); +static inline uint32_t lid_hmx_push_in(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g) { + return lid_hmx_in(hctx, th, ith, nth, g, true); } // Row pair (a, b) quants of two blocks (64 values each, bytes 0..63) to the two tile vectors, scaled by s0, s1 @@ -1066,7 +1066,7 @@ static __attribute__((noinline)) void lid_hmx_prep_k(const struct htp_lid_hmx_co } // O column of each token (64 kv per gather) + mask -> f32 rows, then one DMA out per segment -static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { +static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth, uint32_t g) { const struct htp_ops_context * octx = hctx->octx; const struct htp_tensor * dst = octx->dst; const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; @@ -1081,7 +1081,7 @@ static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_c uint32_t n_rows; const uint32_t n_seg = lid_hmx_segs(hctx, g, seg, &n_rows); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) g); const uint32_t n_u = hctx->n_u_run; @@ -1138,7 +1138,7 @@ static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_c } const struct lid_hmx_set S = hctx->sets[seg[i].set]; - dma_queue_push(octx->ctx->dma[ith], + dma_queue_push(th->dma_q, dma_make_data(dst->data + S.is * dst->nb[3] + (S.t0 + ta) * dst->nb[1] + seg[i].kv0 * sizeof(float), o_buf + ta * L->size_o_row + seg[i].x0 * sizeof(float)), dst->nb[1], L->size_o_row, seg[i].nv * sizeof(float), tb - ta); @@ -1159,15 +1159,15 @@ static inline uint32_t lid_hmx_n_out(const struct htp_lid_hmx_context * hctx, ui return lid_hmx_segs(hctx, g, seg, &n_rows); } -static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth); +static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth); // Pipeline stage g (HMX runs block g meanwhile): prep block g + 1, extract block g - 1 -static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth) { +static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth) { const int32_t g = hctx->g; const int32_t n = (int32_t) hctx->n_blocks; - struct htp_thread_trace * tr = &hctx->octx->ctx->trace[ith]; - dma_queue * dma_q = hctx->octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; // FIFO order per thread: OUT(g - 3), IN(g + 1), OUT(g - 2). When this stage preps next run Q, also pop // OUT(g - 2) so the queue is empty for the Q DMAs; else leave it in flight for one more stage. @@ -1178,7 +1178,7 @@ static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith, n_pop += lid_hmx_n_out(hctx, ith, nth, g - 3); } if (g >= 0 && g + 1 < n) { - n_pop += lid_hmx_in(hctx, ith, nth, g + 1, false); + n_pop += lid_hmx_in(hctx, th, ith, nth, g + 1, false); } if (lag == 2 && g - 2 >= 0) { n_pop += lid_hmx_n_out(hctx, ith, nth, g - 2); @@ -1189,11 +1189,11 @@ static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith, if (hctx->nx_on && g >= 0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g); - lid_hmx_q_chunk(hctx, ith, nth); + lid_hmx_q_chunk(hctx, th, ith, nth); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g); } if (g + 2 < n) { - lid_hmx_push_in(hctx, ith, nth, g + 2); + lid_hmx_push_in(hctx, th, ith, nth, g + 2); } if (g + 1 < n) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1)); @@ -1201,7 +1201,7 @@ static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith, htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1)); } if (g - 1 >= 0) { - lid_hmx_extract(hctx, ith, nth, g - 1); + lid_hmx_extract(hctx, th, ith, nth, g - 1); } if (g == n) { uint32_t n_tail = lid_hmx_n_out(hctx, ith, nth, n - 1); @@ -1214,8 +1214,8 @@ static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith, } } -static void lid_hmx_stage_thread(unsigned int nth, unsigned int ith, void * data) { - lid_hmx_stage((const struct htp_lid_hmx_context *) data, ith, nth); +static void lid_hmx_stage_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { + lid_hmx_stage((const struct htp_lid_hmx_context *) data, th, ith, nth); } // Push the Q rows of channel tile c of set S, one DMA per token. Returns the number of DMA descriptors. @@ -1361,19 +1361,19 @@ static inline uint32_t lid_hmx_q_chunk_push(const struct htp_lid_hmx_context * h } // Stage g share of the next run's Q prep (the DMA queue of the thread is empty here) -static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth) { +static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, const htp_thread_t * th, uint32_t ith, uint32_t nth) { const struct htp_ops_context * octx = hctx->octx; const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; const uint32_t n_tot = hctx->nx_n_run * L->n_ch; const uint32_t a = fastdiv(hctx->g * n_tot, &hctx->nx_stg_div); const uint32_t b = fastdiv((hctx->g + 1) * n_tot, &hctx->nx_stg_div); - const uint32_t per = fastdiv(b - a + octx->n_threads - 1, &octx->n_threads_div); + const uint32_t per = fastdiv(b - a + octx->n_hvx - 1, &octx->n_hvx_div); const uint32_t gc0 = MIN(a + ith * per, b); const uint32_t gc1 = MIN(gc0 + per, b); const uint32_t qb = hctx->qbuf ^ 1; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; uint8_t * qs[2]; qs[0] = hctx->qs + ith * L->size_qs; qs[1] = qs[0] + 32 * L->size_qs_row; @@ -1422,7 +1422,7 @@ static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_c // Q rows to weight tiles, per channel scales |w| and G tiles for all sets of the run, then stage -1 (prep block 0). // Q work goes to the first threads (thread 0 starts first), the K rows of block 0 to the others. -static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data) { +static void lid_hmx_qprep_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_lid_hmx_context * hctx = (struct htp_lid_hmx_context *) data; const struct htp_ops_context * octx = hctx->octx; const struct htp_tensor * q = octx->src[0]; @@ -1431,11 +1431,11 @@ static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data const uint32_t nh = q->ne[1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; const uint32_t n_tot = hctx->n_run * L->n_ch; - const uint32_t per = fastdiv(n_tot + octx->n_threads - 1, &octx->n_threads_div); + const uint32_t per = fastdiv(n_tot + octx->n_hvx - 1, &octx->n_hvx_div); const uint32_t gc0 = MIN(ith * per, n_tot); const uint32_t gc1 = MIN(gc0 + per, n_tot); @@ -1470,7 +1470,7 @@ static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data n_dma += lid_hmx_push_q(hctx, dma_q, qs, &S, first - set * L->n_ch); } - uint32_t n_in = lid_hmx_push_in(hctx, ith, nth, 0); + uint32_t n_in = lid_hmx_push_in(hctx, th, ith, nth, 0); for (uint32_t i = 0; i < n_dma; ++i) { dma_queue_pop(dma_q); @@ -1509,18 +1509,18 @@ static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data dma_queue_pop(dma_q); } - lid_hmx_stage(hctx, ith, nth); + lid_hmx_stage(hctx, th, ith, nth); } // Run start when its Q was prepped in the previous run: block 0 K and mask, then stage -1 -static void lid_hmx_first_thread(unsigned int nth, unsigned int ith, void * data) { +static void lid_hmx_first_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_lid_hmx_context * hctx = (const struct htp_lid_hmx_context *) data; - const uint32_t n_in = lid_hmx_push_in(hctx, ith, nth, 0); + const uint32_t n_in = lid_hmx_push_in(hctx, th, ith, nth, 0); for (uint32_t i = 0; i < n_in; ++i) { - dma_queue_pop(hctx->octx->ctx->dma[ith]); + dma_queue_pop(th->dma_q); } - lid_hmx_stage(hctx, ith, nth); + lid_hmx_stage(hctx, th, ith, nth); } static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kernel_params * kparams) { @@ -1529,7 +1529,7 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne const uint32_t D = q->ne[0]; const uint32_t n_head = q->ne[1]; const uint32_t n_stream = q->ne[3]; - const uint32_t n_threads = kparams->n_threads; + const uint32_t n_hvx = kparams->n_hvx; const uint32_t n_qsets = kparams->n_qsets; struct htp_lid_hmx_context hctx; @@ -1541,7 +1541,7 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne hctx.n_tile = kparams->n_tokens_tile; hctx.n_tt = kparams->n_tt; hctx.n_qsets = n_qsets; - htp_lid_hmx_vtcm_layout_build(&hctx.L, D, n_head, hctx.n_tile, hctx.Bc, n_qsets * kparams->n_qbuf, n_threads, + htp_lid_hmx_vtcm_layout_build(&hctx.L, D, n_head, hctx.n_tile, hctx.Bc, n_qsets * kparams->n_qbuf, n_hvx, kparams->k_row_size); uint8_t * const base = octx->ctx->vtcm_base; @@ -1588,7 +1588,7 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne const int32_t n = (int32_t) hctx.n_blocks; const uint32_t n_tot = hctx.n_run * L->n_ch; - const uint32_t per = fastdiv(n_tot + n_threads - 1, &octx->n_threads_div); + const uint32_t per = fastdiv(n_tot + n_hvx - 1, &octx->n_hvx_div); hctx.n_qthr = (n_tot + per - 1) / per; hctx.g = -1; @@ -1597,9 +1597,9 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne if (q_ready) { hctx.n_qthr = 0; hctx.nx_on = false; - work_queue_run(octx->ctx->work_queue, lid_hmx_first_thread, &hctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, lid_hmx_first_thread, &hctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, lid_hmx_qprep_thread, &hctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, lid_hmx_qprep_thread, &hctx, n_hvx); } // with two Q buffers the stages of this run prep the Q of the next one @@ -1644,7 +1644,7 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne // stage g has work only if there is a next block, a previous one or next run Q work hctx.g = g; if (g + 1 < n || g >= 1 || hctx.nx_on) { - work_queue_run(octx->ctx->work_queue, lid_hmx_stage_thread, &hctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, lid_hmx_stage_thread, &hctx, n_hvx); } hmx_queue_pop(hmx_q); @@ -1652,8 +1652,8 @@ static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kerne // last stage only extracts; with one token only thread 0 has work (same token split for nth 1) hctx.g = n; - const uint32_t n_last = hctx.nt_run == 1 && !hctx.nx_on ? 1 : n_threads; - work_queue_run(octx->ctx->work_queue, lid_hmx_stage_thread, &hctx, n_last); + const uint32_t n_last = hctx.nt_run == 1 && !hctx.nx_on ? 1 : n_hvx; + work_queue_run(octx->ctx->hvx_queue, lid_hmx_stage_thread, &hctx, n_last); } return HTP_STATUS_OK; @@ -1674,7 +1674,7 @@ int op_lightning_indexer(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -1686,9 +1686,9 @@ int op_lightning_indexer(struct htp_ops_context * octx) { lctx.octx = octx; lctx.kp = kparams; lctx.n_blocks = kparams->n_kv_blocks; - lctx.blocks_per_thread = kparams->blocks_per_thread; + lctx.blocks_per_hvx = kparams->blocks_per_hvx; - htp_lid_vtcm_layout_build(&lctx.L, q->ne[0], q->ne[1], kparams->n_tokens_tile, kparams->n_threads, kparams->k_row_size); + htp_lid_vtcm_layout_build(&lctx.L, q->ne[0], q->ne[1], kparams->n_tokens_tile, kparams->n_hvx, kparams->k_row_size); uint8_t * const base = octx->ctx->vtcm_base; @@ -1703,7 +1703,7 @@ int op_lightning_indexer(struct htp_ops_context * octx) { const uint32_t n_tokens = q->ne[2]; const uint32_t n_stream = q->ne[3]; const uint32_t n_tile = kparams->n_tokens_tile; - const uint32_t n_threads = kparams->n_threads; + const uint32_t n_hvx = kparams->n_hvx; for (uint32_t is = 0; is < n_stream; ++is) { lctx.is = is; @@ -1713,8 +1713,8 @@ int op_lightning_indexer(struct htp_ops_context * octx) { lctx.t0 = t0; lctx.nt = MIN(n_tile, n_tokens - t0); - work_queue_run(octx->ctx->work_queue, lid_prep_thread, &lctx, MIN(n_threads, lctx.nt * q->ne[1])); - work_queue_run(octx->ctx->work_queue, lid_compute_thread, &lctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, lid_prep_thread, &lctx, MIN(n_hvx, lctx.nt * q->ne[1])); + work_queue_run(octx->ctx->hvx_queue, lid_compute_thread, &lctx, n_hvx); } } diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h index 6571db32b..14b3503f6 100644 --- a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h +++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h @@ -34,10 +34,10 @@ enum htp_lid_kernel_type { struct htp_lid_kernel_params { uint8_t kernel_type; // enum htp_lid_kernel_type - uint32_t n_threads; + uint32_t n_hvx; uint32_t n_tokens_tile; // tokens per Q tile uint32_t n_kv_blocks; // HVX: per stream, HMX: per run of n_qsets sets - uint32_t blocks_per_thread; + uint32_t blocks_per_hvx; uint32_t vtcm_size; uint32_t k_row_size; // K row bytes uint32_t kv_block; // HMX: kv rows per block (multiple of 64) @@ -61,7 +61,7 @@ _Static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_para // VTCM layout shared by host (fit check) and device (buffer offsets). // Shared regions: Q splats (one f16 vector per q element) and head weight splats for the current token tile. -// Per-thread regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch. +// Per-HVX regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch. struct htp_lid_vtcm_layout { size_t off_q; size_t off_w; @@ -87,7 +87,7 @@ struct htp_lid_vtcm_layout { size_t total_bytes; }; -static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) { +static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_hvx, size_t k_row_size) { L->size_q_row = hex_round_up(D * sizeof(uint16_t), 128); L->size_qs_row = hex_round_up(D * sizeof(float), 128); L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128); @@ -104,18 +104,18 @@ static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, siz size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_q, n_tok * n_head * D * 128); VTCM_LAYOUT_ALLOC(off, off_w, n_tok * n_head * 128); - VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs); - VTCM_LAYOUT_ALLOC(off, off_k, n_threads * HTP_LID_N_BUF * L->size_k_buf); - VTCM_LAYOUT_ALLOC(off, off_kt, n_threads * L->size_kt); - VTCM_LAYOUT_ALLOC(off, off_m, n_threads * HTP_LID_N_BUF * L->size_m_buf); - VTCM_LAYOUT_ALLOC(off, off_o, n_threads * HTP_LID_N_BUF * L->size_o_buf); + VTCM_LAYOUT_ALLOC(off, off_qs, n_hvx * L->size_qs); + VTCM_LAYOUT_ALLOC(off, off_k, n_hvx * HTP_LID_N_BUF * L->size_k_buf); + VTCM_LAYOUT_ALLOC(off, off_kt, n_hvx * L->size_kt); + VTCM_LAYOUT_ALLOC(off, off_m, n_hvx * HTP_LID_N_BUF * L->size_m_buf); + VTCM_LAYOUT_ALLOC(off, off_o, n_hvx * HTP_LID_N_BUF * L->size_o_buf); L->total_bytes = off; } -static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) { +static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_hvx, size_t k_row_size) { struct htp_lid_vtcm_layout L; - htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_threads, k_row_size); + htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_hvx, k_row_size); return L.total_bytes; } @@ -156,7 +156,7 @@ struct htp_lid_hmx_vtcm_layout { }; static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout * L, - size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) { + size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_hvx, size_t k_row_size) { L->n_ch = (uint32_t) ((n_tok * n_head + 31) / 32); L->n_u = (uint32_t) ((n_tok + 31) / 32); L->n_dot = (uint32_t) (D / 32); @@ -186,7 +186,7 @@ static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout VTCM_LAYOUT_ALLOC(off, off_sc, n_qsets * L->n_ch * 256); VTCM_LAYOUT_ALLOC(off, off_sc_id, 256); VTCM_LAYOUT_ALLOC(off, off_w, n_qsets * L->size_w); - VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs); + VTCM_LAYOUT_ALLOC(off, off_qs, n_hvx * L->size_qs); for (int i = 0; i < 2; i++) { VTCM_LAYOUT_ALLOC(off, off_k_raw[i], Bc * L->size_k_raw_row); } @@ -200,9 +200,9 @@ static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout L->total_bytes = off; } -static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) { +static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_hvx, size_t k_row_size) { struct htp_lid_hmx_vtcm_layout L; - htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_threads, k_row_size); + htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_hvx, k_row_size); return L.total_bytes; } diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 1424f1213..ad457d1a6 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -20,6 +20,7 @@ #include #include +#include "htp-thread.h" #include "hex-utils.h" #include "dma-queue.h" #include "hmx-queue.h" @@ -32,7 +33,7 @@ #include "htp-ops.h" #include "htp_iface.h" #include "work-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "allreduce-ops.h" #include "dsv4-hc-ops.h" #include "htp-fence.h" @@ -43,8 +44,7 @@ #define WORK_QUEUE_STACK_SIZE 16384 #define MAIN_THREAD_STACK_SIZE 32768 -_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS, - "work-queue thread cap must be >= HTP_MAX_NTHREADS"); +atomic_uint htp_thread_count = 0; struct htp_handle { struct htp_context * ctx; @@ -491,17 +491,23 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp qurt_sysenv_max_hthreads_t hw_threads; qurt_sysenv_get_max_hw_threads(&hw_threads); uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + uint32_t hw_nth = hw_threads.max_hthreads; + if (hw_nth > HTP_MAX_NTHREADS) { + hw_nth = HTP_MAX_NTHREADS; + } if (n_hvx == 0) { n_hvx = hw_nhvx; } - if (n_hvx > hw_threads.max_hthreads) { - n_hvx = hw_threads.max_hthreads; + if (n_hvx > HTP_MAX_NHVX) { + n_hvx = HTP_MAX_NHVX; } - if (n_hvx > HTP_MAX_NTHREADS) { - n_hvx = HTP_MAX_NTHREADS; + if (n_hvx > hw_nth - 1) { + n_hvx = hw_nth - 1; } + uint32_t n_threads = hw_nth; + // layout segments of our contiguous block // 1. htp_context : sits at the base (block is 4K-aligned via memalign below) @@ -520,13 +526,13 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp size_t wq_align = work_queue_alignof(); size_t offset_wq = hex_align_up(offset, wq_align); size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); - offset = offset_wq + size_wq; + offset = offset_wq + size_wq; // 4. dma_queue size_t dma_align = dma_queue_alignof(); size_t offset_dma = hex_align_up(offset, dma_align); size_t size_dma = 0; - for (uint32_t i = 0; i < n_hvx; i++) { + for (uint32_t i = 0; i < n_threads; i++) { size_dma = hex_align_up(size_dma, dma_queue_alignof()); size_dma += dma_queue_sizeof(256); } @@ -657,16 +663,9 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp } #endif - ctx->hmx_enabled = n_hmx; - ctx->hmx_queue = NULL; - if (n_hmx) { - void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); - ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]); - } - FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); - - ctx->n_threads = n_hvx; - ctx->n_threads_div = init_fastdiv_values(ctx->n_threads); + ctx->n_threads = n_threads; + ctx->n_hvx = n_hvx; + ctx->n_hvx_div = init_fastdiv_values(ctx->n_hvx); // Initialize DMA queues uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma); @@ -681,8 +680,18 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp ctx->ddr_spad_size = HTP_DDR_SPAD_SIZE; ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size); + htp_thread_reset(); + void * wq_ptr = (void *) ((uintptr_t) block + offset_wq); - ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + ctx->hvx_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE, ctx->dma, ctx->trace); + + ctx->hmx_enabled = n_hmx; + ctx->hmx_queue = NULL; + if (n_hmx) { + void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); + ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, ctx->dma, ctx->trace); + } + FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); ctx->main_stack = NULL; ctx->main_thread = 0; @@ -708,7 +717,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp } FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n", - sess_id, hw_nhvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio); + sess_id, ctx->n_hvx, ctx->vtcm_size, ctx->vtcm_rctx, ctx->n_threads, ctx->thread_id, ctx->thread_prio); return AEE_SUCCESS; } @@ -733,7 +742,7 @@ AEEResult htp_iface_stop(remote_handle64 handle) { return err; } - work_queue_free(ctx->work_queue); + work_queue_free(ctx->hvx_queue); for (int i = 0; i < ctx->n_threads; i++) { dma_queue_free(ctx->dma[i]); @@ -769,16 +778,20 @@ AEEResult htp_iface_hwinfo(remote_handle64 handle, uint32_t * n_threads, uint32_ qurt_sysenv_get_max_hw_threads(&hw_threads); uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; - uint32_t n_hvx_val = hw_nhvx; - if (n_hvx_val > hw_threads.max_hthreads) { - n_hvx_val = hw_threads.max_hthreads; - } - if (n_hvx_val > HTP_MAX_NTHREADS) { - n_hvx_val = HTP_MAX_NTHREADS; + uint32_t n_threads_val = hw_threads.max_hthreads; + if (n_threads_val > HTP_MAX_NTHREADS) { + n_threads_val = HTP_MAX_NTHREADS; } - // for now we force n_threads == n_hvx - *n_threads = n_hvx_val; + uint32_t n_hvx_val = hw_nhvx; + if (n_hvx_val > HTP_MAX_NHVX) { + n_hvx_val = HTP_MAX_NHVX; + } + if (n_hvx_val > n_threads_val - 1) { + n_hvx_val = n_threads_val - 1; + } + + *n_threads = n_threads_val; *n_hvx = n_hvx_val; *n_hmx = 1; @@ -1227,10 +1240,10 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) { memcpy(octx->op_params, op->params, sizeof(octx->op_params)); memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params)); - octx->flags = op->flags; - octx->op = op->opcode; - octx->n_threads = octx->ctx->n_threads; - octx->n_threads_div = octx->ctx->n_threads_div; + octx->flags = op->flags; + octx->op = op->opcode; + octx->n_hvx = octx->ctx->n_hvx; + octx->n_hvx_div = octx->ctx->n_hvx_div; FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags); @@ -1292,7 +1305,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops; - const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc); + const uint32_t tr_size = ctx->n_threads * req->n_traces * sizeof(struct htp_trace_desc); if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) { FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size); @@ -1315,7 +1328,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r memset(ctx->trace, 0, sizeof(ctx->trace)); if (ctx->profiler == HTP_PROF_TRACE) { struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size); - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + for (uint32_t t = 0; t < ctx->n_threads; t++) { ctx->trace[t].events = &trace_events[t * req->n_traces]; ctx->trace[t].max_events = req->n_traces; } @@ -1339,7 +1352,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r mdev_group_init(ctx, req); - work_queue_wakeup(ctx->work_queue); + work_queue_wakeup(ctx->hvx_queue); if (ctx->hmx_queue) { hmx_queue_wakeup(ctx->hmx_queue); } @@ -1372,7 +1385,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r hmx_queue_suspend(ctx->hmx_queue); hmx_queue_flush(ctx->hmx_queue); } - work_queue_suspend(ctx->work_queue); + work_queue_suspend(ctx->hvx_queue); // Flush remaining dirty tensors at the end of the batch htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); @@ -1395,7 +1408,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r rsp.cycles_stop = batch_prof.cycles_stop; if (ctx->profiler == HTP_PROF_TRACE) { - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + for (uint32_t t = 0; t < ctx->n_threads; t++) { rsp.n_traces[t] = ctx->trace[t].count; } } diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index b4b943058..fb15310cf 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -100,7 +100,7 @@ struct htp_mm_context { const float * restrict sz0, const float * restrict sz1); // Precomputed values - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t src0_row_start; uint32_t src0_row_end; uint32_t src0_row_size_padded; @@ -115,12 +115,12 @@ struct htp_mm_context { struct fastdiv_values mm_div_ne11; // Precomputed block-parallel quantization values - uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS]; - uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_ib_first[HTP_MAX_NHVX]; + uint32_t quant_ib_last[HTP_MAX_NHVX]; + uint32_t quant_r[HTP_MAX_NHVX]; + uint32_t quant_c[HTP_MAX_NHVX]; uint32_t n_quant_tasks; - uint32_t n_quant_rows_per_thread; + uint32_t n_quant_rows_per_hvx; // Fields for scattered mapping & HMX support in MUL_MAT_ID const uint32_t * matrix_row_counts; @@ -140,21 +140,21 @@ struct htp_mm_context { uint32_t vtcm_act_raw_stride; // Cached thread offsets/sizes - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_act_size_per_thread; - uint32_t vtcm_dst_size_per_thread; + uint32_t vtcm_src0_size_per_hvx; + uint32_t vtcm_act_size_per_hvx; + uint32_t vtcm_dst_size_per_hvx; }; static int htp_mm_init_context( struct htp_ops_context * octx, const struct htp_mm_kernel_params * kparams ) { - if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, (uint32_t) kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } if (kparams->n_hmx) { - if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) { + if (kparams->n_act_hvx <= 0 || kparams->n_act_hvx > (int32_t) octx->n_hvx) { return HTP_STATUS_INVAL_PARAMS; } } @@ -236,8 +236,8 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { #define htp_matmul_preamble \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ - uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; \ + dma_queue * dma_q = th->dma_q; \ + uint32_t src0_nrows_per_hvx = mmctx->src0_nrows_per_hvx; \ htp_matmul_tensors_preamble; @@ -248,17 +248,17 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { #include "hvx-mm-kernels-float.h" #include "hmx-mm-kernels-tiled.h" -static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -269,8 +269,8 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { const size_t act_stride = mmctx->vtcm_act_stride; const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0; - uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; const dma_addr_t src0_row = src0->data; @@ -347,15 +347,15 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -365,8 +365,8 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { const size_t act_row_size = nb11; const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * act_data = mmctx->vtcm_act; float * tmp = (float *) vtcm_dst_ptr; @@ -441,7 +441,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_nx_2d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = data; struct htp_ops_context * octx = mmctx->octx; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -452,10 +452,10 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) const uint32_t act_nrows = act->ne[1] * act->ne[2] * act->ne[3]; const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); @@ -463,7 +463,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict src_w = octx->src[widx]; const struct htp_tensor * restrict dst = octx->dsts[widx]; if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); @@ -487,11 +487,11 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) } const uint32_t nrows = src0_end_row - src0_start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); - const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row); + const uint32_t start_row = src0_start_row + src0_nrows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + src0_nrows_per_hvx, src0_end_row); if (start_row >= end_row) continue; uint32_t ct_start = start_row / 32; @@ -612,25 +612,25 @@ static void hvx_mm_transfer_act_dma( } #define QUANTIZE_IMPL(name, log_name, kernel_fn, dst_row_size_expr) \ -static void name(unsigned int nth, unsigned int ith, void * data) { \ +static void name(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { \ (void) nth; \ struct htp_mm_context * mmctx = data; \ struct htp_ops_context * octx = mmctx->octx; \ const struct htp_tensor * src = mmctx->act; \ const uint32_t ne0 = src->ne[0]; \ const uint32_t nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; \ - const uint32_t nrows_per_thread = mmctx->n_quant_rows_per_thread; \ + const uint32_t nrows_per_hvx = mmctx->n_quant_rows_per_hvx; \ \ - const uint32_t ir_first = nrows_per_thread * ith; \ + const uint32_t ir_first = nrows_per_hvx * ith; \ if (ir_first >= nrows) { \ return; \ } \ \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + struct htp_thread_trace * tr = th->trace; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ \ uint8_t * restrict dst = mmctx->vtcm_act; \ - const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); \ + const uint32_t ir_last = MIN(ir_first + nrows_per_hvx, nrows); \ const size_t raw_row_size = mmctx->vtcm_act_raw_stride; \ const size_t dst_row_size = (dst_row_size_expr); \ \ @@ -650,14 +650,14 @@ QUANTIZE_IMPL(quantize_f32_f32, "quantize-f32-f32", quanti QUANTIZE_IMPL(quantize_f32_f16, "quantize-f32-f16", quantize_f32_f16_kernel, mmctx->vtcm_act_stride) QUANTIZE_IMPL(quantize_f16_f16, "quantize-f16-f16", quantize_f16_f16_kernel, mmctx->vtcm_act_stride) -static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_0_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -678,14 +678,14 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_1_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -706,14 +706,14 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_q8_1_s16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -734,14 +734,14 @@ static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f32_f16_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f32_f16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -762,14 +762,14 @@ static void quantize_f32_f16_tiled_block(unsigned int nth, unsigned int ith, voi htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); } -static void quantize_f16_f16_tiled_block(unsigned int nth, unsigned int ith, void * data) { +static void quantize_f16_f16_tiled_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; struct htp_mm_context * mmctx = data; if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { return; } struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = mmctx->act; @@ -791,7 +791,7 @@ static void quantize_f16_f16_tiled_block(unsigned int nth, unsigned int ith, voi } // q8_1 for weight types with offsets (q8_1_s16 for Q2_K), f16_tiled for F16, otherwise q8_0 -static inline worker_callback_t htp_mm_act_quant_row_func(int weight_type, int act_type) { +static inline work_queue_func_t htp_mm_act_quant_row_func(int weight_type, int act_type) { if (weight_type == HTP_TYPE_F16) { return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled : quantize_f32_f16_tiled; } @@ -801,7 +801,7 @@ static inline worker_callback_t htp_mm_act_quant_row_func(int weight_type, int a return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } -static inline worker_callback_t htp_mm_act_quant_block_func(int weight_type, int act_type) { +static inline work_queue_func_t htp_mm_act_quant_block_func(int weight_type, int act_type) { if (weight_type == HTP_TYPE_F16) { return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled_block : quantize_f32_f16_tiled_block; } @@ -811,17 +811,17 @@ static inline worker_callback_t htp_mm_act_quant_block_func(int weight_type, int return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; } -static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_4d_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -830,7 +830,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { const size_t dst_row_size = nb1; const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); @@ -935,7 +935,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_2d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -947,11 +947,11 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; @@ -961,8 +961,8 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; const dma_addr_t src0_row = src0->data; @@ -1038,15 +1038,15 @@ static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_2d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; @@ -1056,8 +1056,8 @@ static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) { const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; - uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_hvx * ith; + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * act_data = mmctx->vtcm_act; float * tmp = (float *) vtcm_dst_ptr; @@ -1143,7 +1143,7 @@ static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_4d_flat(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1155,18 +1155,18 @@ static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) { const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; const size_t src0_stride = mmctx->vtcm_src0_stride; const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; @@ -1292,22 +1292,22 @@ static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) { #define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * mmctx->mapping_stride + (i1)] -static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_id_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_tensor * restrict ids = octx->src[2]; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert const uint32_t act_nrows = ne11; - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); if (src0_start_row >= src0_end_row) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1325,7 +1325,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { const size_t act_stride = mmctx->vtcm_act_stride; // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { @@ -1380,21 +1380,21 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_id_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; const struct htp_tensor * restrict ids = octx->src[2]; const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert - const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, mmctx->src0_row_end); if (src0_start_row >= src0_end_row) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1409,7 +1409,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { const uint32_t n_ids = ne02; // num experts // Per-thread VTCMs for all tensors - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert @@ -1457,7 +1457,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { } } -static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mv_id_nx_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = (struct htp_mm_context *) data; struct htp_ops_context * octx = mmctx->octx; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1467,7 +1467,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); @@ -1475,7 +1475,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const uint32_t n_aids = ids->ne[0]; const uint32_t n_ids = src0->ne[2]; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { @@ -1487,7 +1487,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t ne01 = src_w->ne[1]; uint32_t start_row = 0; @@ -1500,11 +1500,11 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) } const uint32_t nrows = end_row - start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); - const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); + const uint32_t src0_start_row = start_row + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, end_row); if (src0_start_row >= src0_end_row) continue; const dma_addr_t src0_row = src_w->data + eid * src_w->nb[2]; @@ -1546,7 +1546,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) } } -static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) { +static void hvx_mm_id_nx_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = (struct htp_mm_context *) data; struct htp_ops_context * octx = mmctx->octx; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; @@ -1556,7 +1556,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); @@ -1568,7 +1568,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const size_t act_stride = mmctx->vtcm_act_stride; - uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_hvx * ith; uint8_t * restrict act_data = mmctx->vtcm_act; for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { @@ -1579,7 +1579,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t ne01 = src_w->ne[1]; uint32_t start_row = 0; @@ -1592,11 +1592,11 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) } const uint32_t nrows = end_row - start_row; - uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); - src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); + uint32_t src0_nrows_per_hvx = fastdiv(nrows + nth - 1, &octx->n_hvx_div); + src0_nrows_per_hvx = hex_round_up(src0_nrows_per_hvx, 32); - const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); + const uint32_t src0_start_row = start_row + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, end_row); if (src0_start_row >= src0_end_row) continue; const dma_addr_t src0_row = src_w->data + cur_a * src_w->nb[2]; @@ -1751,12 +1751,12 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const bool is_tiled = htp_tensor_is_tiled(src0); - // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + // Compute src0_nrows_per_hvx + mmctx->src0_nrows_per_hvx = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); if (is_tiled) { - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + mmctx->src0_nrows_per_hvx = hex_round_up(mmctx->src0_nrows_per_hvx, 32); } else { - mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even + mmctx->src0_nrows_per_hvx += (mmctx->src0_nrows_per_hvx & 1); // round up to even } const size_t src0_row_size = nb01; @@ -1765,8 +1765,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - worker_callback_t quant_task_func; - worker_callback_t matmul_job_func; + work_queue_func_t quant_task_func; + work_queue_func_t matmul_job_func; uint32_t n_quant_tasks = 1; const bool is_batched = (ne12 > 1 || ne13 > 1 || ne02 > 1 || ne03 > 1); if (is_batched) { @@ -1821,8 +1821,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const uint32_t nb = (ne10 + qk - 1) / qk; const uint32_t total_nb = act_nrows * nb; - if (act_nrows < octx->n_threads && !is_batched) { - n_quant_tasks = MIN(total_nb, octx->n_threads); + if (act_nrows < octx->n_hvx && !is_batched) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; @@ -1833,7 +1833,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } act_row_size = kparams->act_row_size; @@ -1846,7 +1846,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const uint32_t m_layout_rows = m_chunk; struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_hvx, dst_row_size, src0_row_size, act_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false); if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || @@ -1854,13 +1854,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { kparams->kernel_type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { - mmctx->vtcm_act_size_per_thread = L.act_bytes; + mmctx->vtcm_act_size_per_hvx = L.act_bytes; } else { - mmctx->vtcm_act_size_per_thread = fastdiv(L.act_bytes, &octx->n_threads_div); + mmctx->vtcm_act_size_per_hvx = fastdiv(L.act_bytes, &octx->n_hvx_div); } - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); const size_t vtcm_size = L.total_bytes; @@ -1910,8 +1910,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const uint32_t total_nb = cur_m_rows * nb; uint32_t quant_tasks; work_queue_func_t q_func; - if (cur_m_rows < octx->n_threads && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) { - quant_tasks = MIN(total_nb, octx->n_threads); + if (cur_m_rows < octx->n_hvx && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) { + quant_tasks = MIN(total_nb, octx->n_hvx); q_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / quant_tasks; @@ -1922,17 +1922,17 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - quant_tasks = MIN(cur_m_rows, octx->n_threads); + quant_tasks = MIN(cur_m_rows, octx->n_hvx); q_func = quant_task_func; - mmctx->n_quant_rows_per_thread = (cur_m_rows + quant_tasks - 1) / quant_tasks; + mmctx->n_quant_rows_per_hvx = (cur_m_rows + quant_tasks - 1) / quant_tasks; } mmctx->n_quant_tasks = quant_tasks; - work_queue_run(octx->ctx->work_queue, q_func, mmctx, quant_tasks); + work_queue_run(octx->ctx->hvx_queue, q_func, mmctx, quant_tasks); } else { hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, m_start, cur_m_rows); } - work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, matmul_job_func, mmctx, octx->n_hvx); } } else { mmctx->cur_m_start = 0; @@ -1940,22 +1940,22 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { if (need_quant) { hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); } else { hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, 0, act_nrows); } - work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, matmul_job_func, mmctx, octx->n_hvx); } return HTP_STATUS_OK; } -static void dequantize_tiled_worker_loop(unsigned int n, unsigned int i, void * data) { +static void dequantize_tiled_worker_loop(const htp_thread_t * th, unsigned int n, unsigned int i, void * data) { tiled_dequantize_state_t * state = (tiled_dequantize_state_t *) data; - struct htp_thread_trace * tr = &state->traces[i]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); for (unsigned int task_id = i; task_id < (unsigned int) state->n_tasks; task_id += n) { int start = task_id * state->n_tiles_per_task; @@ -1965,10 +1965,10 @@ static void dequantize_tiled_worker_loop(unsigned int n, unsigned int i, void * htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); } -static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { output_transfer_task_state_t *st = (output_transfer_task_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int start_chunk_idx = i * st->n_chunks_per_task; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx); @@ -1986,8 +1986,6 @@ static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void } typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; __fp16 * dst; dma_addr_t act_dma_addr; float * vtcm_f32_act; @@ -1997,15 +1995,13 @@ typedef struct { uint32_t k_block; uint32_t k_stride; uint32_t k_valid; - size_t vtcm_f32_act_bytes_per_thread; + size_t vtcm_f32_act_bytes_per_hvx; uint32_t dma_step_rows; uint32_t dma_step_rows_shift; uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1 } activation_transfer_task_state_t; typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; __fp16 * dst; dma_addr_t act_dma_addr; float * vtcm_f32_act; @@ -2014,7 +2010,7 @@ typedef struct { uint32_t k_stride; uint32_t k_valid; uint32_t n_col_chunks; - struct fastdiv_values n_threads_div; + struct fastdiv_values n_hvx_div; size_t vtcm_f32_act_bytes; uint32_t dma_step_rows; uint32_t dma_step_rows_shift; @@ -2088,16 +2084,16 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( } } -static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_col_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; uint32_t n_blocks = st->k_block / 32; - uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); - uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t b_first = fastdiv(n_blocks * i, &st->n_hvx_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_hvx_div); uint32_t c_first = b_first * 32; - uint32_t c_last = b_last * 32; - uint32_t c_len = c_last - c_first; + uint32_t c_last = b_last * 32; + uint32_t c_len = c_last - c_first; if (c_len == 0) { return; @@ -2110,11 +2106,11 @@ static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsign __fp16 *dst = st->dst; - size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128); + size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_hvx_div), 128); float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes); transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( - st->ctx->dma[i], dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, + th->dma_q, dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift ); } @@ -2202,10 +2198,10 @@ static void transfer_activation_chunk_to_fp16_dma_pipelined( } } -static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { int chunk_idx = task_id * st->n_chunks_per_task; @@ -2214,15 +2210,14 @@ static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, __fp16 *dst = st->dst + chunk_idx * st->k_block; const dma_addr_t act_dma_addr = st->act_dma_addr + (size_t) chunk_idx * st->k_stride * st->act_elem_size; - uint8_t *thread_act = (uint8_t *) st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_thread; + uint8_t *thread_act = (uint8_t *) st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_hvx; transfer_activation_chunk_to_fp16_dma_pipelined( - st->ctx->dma[i], dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_act, st->act_elem_size, tr, st->dma_step_rows, st->dma_step_rows_shift + th->dma_q, dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_act, st->act_elem_size, tr, st->dma_step_rows, st->dma_step_rows_shift ); } } typedef struct { - struct htp_thread_trace * traces; const struct mmid_row_mapping * matrix_rows; __fp16 * dst; const float * src; @@ -2242,7 +2237,6 @@ typedef struct { } activation_transfer_gathered_task_state_t; typedef struct { - struct htp_thread_trace * traces; const struct mmid_row_mapping * matrix_rows; const __fp16 * vtcm_src; float * dst; @@ -2258,9 +2252,9 @@ typedef struct { uint32_t cne1; } output_transfer_scattered_task_state_t; -static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_gathered_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2276,9 +2270,9 @@ static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigne } } -static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_activation_chunk_gathered_worker_flat_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2294,9 +2288,9 @@ static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, un } } -static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_scattered_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { output_transfer_scattered_task_state_t *st = data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; int vtcm_start_row = chunk_idx * chunk_size; @@ -2341,7 +2335,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( int n_cols, int k_block, size_t row_stride, int weight_type, int n_k_tiles, struct fastdiv_values n_k_tiles_div, - tiled_dequant_task_fn_t dequant_worker_fn, int n_threads) { + tiled_dequant_task_fn_t dequant_worker_fn, int n_hvx) { if (!dequant_worker_fn) { return; @@ -2353,7 +2347,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( const uint32_t n_weight_tiles = k_block / htp_tensor_get_tile_k(weight_type); const size_t n_tot_tiles = n_col_tiles * n_weight_tiles; - size_t n_tiles_per_task = (n_threads == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_threads); + size_t n_tiles_per_task = (n_hvx == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_hvx); tiled_dequantize_state_t state; state.n_tasks = (n_tot_tiles + n_tiles_per_task - 1) / n_tiles_per_task; @@ -2369,23 +2363,15 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( state.n_k_tiles = n_k_tiles; state.n_weight_tiles = n_weight_tiles; state.n_k_tiles_div = n_k_tiles_div; - state.traces = ctx->trace; - state.ctx = ctx; state.tile_size = htp_tensor_get_tile_size(weight_type); state.aligned_tile_size = htp_tensor_get_tile_padded_size(weight_type); - if (state.n_tasks == 1 || n_threads == 1) { - dequantize_tiled_worker_loop(1, 0, &state); - } else { - int n_tasks = hex_smin((int) state.n_tasks, n_threads); - worker_pool_run_func(ctx->worker_pool, dequantize_tiled_worker_loop, &state, n_tasks); - } + int n_tasks = hex_smin((int) state.n_tasks, n_hvx); + work_queue_run(ctx->hvx_queue, dequantize_tiled_worker_loop, &state, n_tasks); } typedef struct { - struct htp_context * ctx; - struct htp_thread_trace * traces; float * dst; const __fp16 * vtcm_src; const float * src2; @@ -2394,19 +2380,19 @@ typedef struct { uint32_t dst_stride; uint32_t src2_stride; uint32_t dst_cols; - struct fastdiv_values n_threads_div; + struct fastdiv_values n_hvx_div; } output_transfer_col_chunk_state_t; -static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { +static void transfer_output_chunk_col_chunk_worker_fn(const htp_thread_t * th, unsigned int n, unsigned int i, void *data) { (void) n; output_transfer_col_chunk_state_t *st = (output_transfer_col_chunk_state_t *) data; - struct htp_thread_trace * tr = &st->traces[i]; + struct htp_thread_trace * tr = th->trace; // Round up: the last block is partial when N is not 32-aligned. Its pad // columns are dropped by the dst_cols clamp inside the store. uint32_t n_blocks = hmx_ceil_div(st->n_cols, 32); - uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); - uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t b_first = fastdiv(n_blocks * i, &st->n_hvx_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_hvx_div); uint32_t c_first = b_first * 32; uint32_t c_last = b_last * 32; uint32_t c_len = c_last - c_first; @@ -2431,12 +2417,12 @@ static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned i } static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, const float *src2, const __fp16 *vtcm_src, - int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_threads) { + int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_hvx) { if (n_rows <= 0) return; uint32_t n_blocks = hmx_ceil_div((uint32_t) n_cols, 32); - if (n_threads > 1 && n_blocks >= (uint32_t)n_threads) { - struct fastdiv_values n_threads_div = (n_threads == (int)ctx->n_threads) ? ctx->n_threads_div : init_fastdiv_values(n_threads); + if (n_hvx > 1 && n_blocks >= (uint32_t)n_hvx) { + struct fastdiv_values n_hvx_div = (n_hvx == (int)ctx->n_hvx) ? ctx->n_hvx_div : init_fastdiv_values(n_hvx); output_transfer_col_chunk_state_t col_state; col_state.dst = dst; col_state.src2 = src2; @@ -2446,22 +2432,20 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, col_state.dst_stride = (uint32_t)dst_stride; col_state.src2_stride = src2_stride; col_state.dst_cols = (uint32_t)dst_cols; - col_state.n_threads_div = n_threads_div; - col_state.traces = ctx->trace; - col_state.ctx = ctx; + col_state.n_hvx_div = n_hvx_div; - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_threads); + work_queue_run(ctx->hvx_queue, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_hvx); return; } size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_threads); + size_t n_chunks_per_task = (n_hvx == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_hvx); n_chunks_per_task = hex_align_up(n_chunks_per_task, 2); - int actual_threads = hmx_ceil_div(n_rows, n_chunks_per_task); + int actual_hvx = hmx_ceil_div(n_rows, n_chunks_per_task); output_transfer_task_state_t state; - state.n_tasks = actual_threads; + state.n_tasks = actual_hvx; state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; @@ -2471,13 +2455,8 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, state.dst_stride = dst_stride; state.src2_stride = src2_stride; state.dst_cols = dst_cols; - state.traces = ctx->trace; - if (actual_threads <= 1) { - transfer_output_chunk_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, transfer_output_chunk_worker_fn, &state, actual_hvx); } struct activation_transfer_params { @@ -2487,8 +2466,8 @@ struct activation_transfer_params { int n_rows; int k_block; int k_stride; - int n_threads; - const struct fastdiv_values * act_threads_div; + int n_hvx; + const struct fastdiv_values * act_hvx_div; const struct fastdiv_values * k_div; int k_valid; float * vtcm_f32_act; @@ -2503,8 +2482,8 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ int n_rows = params->n_rows; int k_block = params->k_block; int k_stride = params->k_stride; - int n_threads = params->n_threads; - const struct fastdiv_values * act_threads_div = params->act_threads_div; + int n_hvx = params->n_hvx; + const struct fastdiv_values * act_hvx_div = params->act_hvx_div; const struct fastdiv_values * k_div = params->k_div; int k_valid = params->k_valid; float * vtcm_f32_act = params->vtcm_f32_act; @@ -2517,12 +2496,12 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ } const size_t n_tasks = (n_rows + 31) >> 5; - if (act_elem_size == sizeof(float) && n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) { + if (act_elem_size == sizeof(float) && n_hvx > 1 && k_block > 32 && n_tasks < (size_t) n_hvx) { // Calculate step rows parameters for column-chunked dma pipelining uint32_t dma_step_rows = 2; uint32_t dma_step_rows_shift = 1; if (vtcm_f32_act_bytes > 0) { - size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_hvx_div), 128); size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float); size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); if (dma_step_rows_max >= 4) { @@ -2538,26 +2517,24 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ col_state.k_block = k_block; col_state.k_stride = k_stride; col_state.k_valid = k_valid; - col_state.n_col_chunks = n_threads; - col_state.n_threads_div = *act_threads_div; + col_state.n_col_chunks = n_hvx; + col_state.n_hvx_div = *act_hvx_div; col_state.vtcm_f32_act = vtcm_f32_act; col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes; - col_state.traces = ctx->trace; - col_state.ctx = ctx; col_state.dma_step_rows = dma_step_rows; col_state.dma_step_rows_shift = dma_step_rows_shift; - worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads); + work_queue_run(ctx->hvx_queue, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_hvx); return; } assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0); size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address + size_t n_chunks_per_task = (n_hvx == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address activation_transfer_task_state_t state; - state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); + state.n_tasks = (n_hvx == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; @@ -2565,17 +2542,15 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ state.k_block = k_block; state.k_stride = k_stride; state.k_valid = k_valid; - state.traces = ctx->trace; - state.ctx = ctx; state.vtcm_f32_act = vtcm_f32_act; - state.act_elem_size = act_elem_size; + state.act_elem_size = act_elem_size; - state.vtcm_f32_act_bytes_per_thread = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + state.vtcm_f32_act_bytes_per_hvx = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_hvx_div), 128); uint32_t dma_step_rows = 2; uint32_t dma_step_rows_shift = 1; - if (state.vtcm_f32_act_bytes_per_thread > 0) { - size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float); + if (state.vtcm_f32_act_bytes_per_hvx > 0) { + size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_hvx / sizeof(float); size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); if (dma_step_rows_max >= 4) { dma_step_rows = 4; @@ -2585,12 +2560,8 @@ static void transfer_activation_chunk_threaded(const struct activation_transfer_ state.dma_step_rows = dma_step_rows; state.dma_step_rows_shift = dma_step_rows_shift; - int active_threads = hex_smin(n_threads, (int)state.n_tasks); - if (state.n_tasks == 1 || n_threads == 1) { - transfer_activation_chunk_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_worker_fn, &state, active_threads); - } + int active_hvx = hex_smin(n_hvx, (int)state.n_tasks); + work_queue_run(ctx->hvx_queue, transfer_activation_chunk_worker_fn, &state, active_hvx); } // --- Async HMX matmul job (for pipeline overlap) --- @@ -2604,7 +2575,7 @@ typedef struct { uint32_t n_dot_tiles; } hmx_matmul_job_t; -static void hmx_matmul_worker_fn(void * data) { +static void hmx_matmul_worker_fn(const htp_thread_t * th, void * data) { hmx_matmul_job_t * job = (hmx_matmul_job_t *) data; FARF(HIGH, "hmx-mm-job: n_row_tiles %u n_col_tiles %u n_dot_tiles %u", job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); @@ -2646,9 +2617,9 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, int m_chunk, int n_chunk, int pipeline, - int n_threads, - int act_threads, - const struct fastdiv_values * act_threads_div, + int n_hvx, + int act_hvx, + const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div, int tile_size, int aligned_tile_size, @@ -2686,7 +2657,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, size_t vtcm_used = vtcm_size; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, src2_bytes); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_hvx, aligned_tile_size, src2_bytes); vtcm_used = L.total_bytes; if (vtcm_used > vtcm_budget) { @@ -2751,8 +2722,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, @@ -2789,7 +2760,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 4], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); // 3. push A_{i+2} (if i+2 < n_chunk_cnt) if (i + 2 < n_chunk_cnt) { @@ -2814,7 +2785,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_prev) : NULL; int chunk_dst_cols = dst_cols - (int)nc_prev; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } @@ -2827,7 +2798,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_last) : NULL; int chunk_dst_cols = dst_cols - (int)nc_last; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } else { @@ -2846,8 +2817,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, @@ -2876,7 +2847,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); // Start weight DMA for the next chunk early const size_t nc_next = nc + n_chunk_n_cols; @@ -2896,7 +2867,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc) : NULL; int chunk_dst_cols = dst_cols - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_hvx); } } } @@ -2947,16 +2918,16 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const int m_chunk_n_rows = kparams->m_chunk; const int n_chunk_n_cols = kparams->n_chunk; const int pipeline = kparams->pipeline; - const int n_threads = octx->n_threads; - const int act_threads = kparams->n_act_threads; - const struct fastdiv_values * act_threads_div = &kparams->div_n_act_threads; + const int n_hvx = octx->n_hvx; + const int act_hvx = kparams->n_act_hvx; + const struct fastdiv_values * act_hvx_div = &kparams->div_n_act_hvx; const struct fastdiv_values * k_div = &kparams->div_ne00_padded; const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride; const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, 0); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_hvx, aligned_tile_size, 0); if (L.total_bytes > vtcm_budget) { FARF(ERROR, "hmx-mm-nx-2d: VTCM overflow: used %zu budget %zu, m %d k %d mc %d nc %d", @@ -3034,8 +3005,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, @@ -3085,7 +3056,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 4], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); if (i + 2 < n_chunk_cnt) { const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2; @@ -3106,7 +3077,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k float *output_chunk = dst_ptr + (mr * dst_stride + nc_prev); int chunk_dst_cols = dst_cols - (int)nc_prev; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_hvx); } } } @@ -3117,7 +3088,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k float *output_chunk = dst_ptr + (mr * dst_stride + nc_last); int chunk_dst_cols = dst_cols - (int)nc_last; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_hvx); } } } @@ -3136,8 +3107,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k .n_rows = (int) n_rows, .k_block = k, .k_stride = act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = k_valid, .vtcm_f32_act = vtcm_f32_act, @@ -3180,7 +3151,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx); const size_t nc_next = nc + n_chunk_n_cols; if (nc_next < n) { @@ -3196,7 +3167,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k float *output_chunk = dst_ptr + (mr * dst_stride + nc); int chunk_dst_cols = dst_cols - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_hvx); } } } @@ -3228,8 +3199,8 @@ static inline float *hmx_mm_dst_batch_ptr(const hmx_mm_batched_params_t *params, static int hmx_mm_batched_simple(struct htp_context *ctx, const hmx_mm_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size, - const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) { + int m_chunk, int n_chunk, int pipeline, int n_hvx, int act_hvx, int vtcm_size, + const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div) { int ret = 0; for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { @@ -3243,16 +3214,16 @@ static int hmx_mm_batched_simple(struct htp_context *ctx, params->m, params->k, params->n, params->act_stride, params->act_elem_size, params->weight_stride, params->weight_type, params->k, params->dst_stride, params->bias_stride, params->dst_cols, - m_chunk, n_chunk, pipeline, n_threads, act_threads, - act_threads_div, k_div, params->tile_size, params->aligned_tile_size, vtcm_size); + m_chunk, n_chunk, pipeline, n_hvx, act_hvx, + act_hvx_div, k_div, params->tile_size, params->aligned_tile_size, vtcm_size); } } return ret; } static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, - const struct fastdiv_values * act_threads_div, + int m_chunk, int n_chunk, int pipeline, int n_hvx, int act_hvx, + const struct fastdiv_values * act_hvx_div, const struct fastdiv_values * k_div, int vtcm_size) { const size_t row_stride = htp_tensor_get_row_size(params->weight_type, params->k); @@ -3272,7 +3243,7 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t // Grouped path is only valid for flat F16 if group_size > 1 and it fits within VTCM budget. bool run_grouped = (params->weight_type == HTP_TYPE_F16 && params->tile_size == 0 && group_size > 1 && (size_t)vtcm_size <= vtcm_budget); if (!run_grouped) { - return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); + return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_hvx, act_hvx, vtcm_size, act_hvx_div, k_div); } struct htp_thread_trace * tr = &ctx->trace[0]; @@ -3282,12 +3253,12 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t size_t n_chunk_n_cols = n_chunk; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_threads, 0, params->bias_bytes); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_hvx, 0, params->bias_bytes); if (L.total_bytes > vtcm_budget) { FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); + return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_hvx, act_hvx, vtcm_size, act_hvx_div, k_div); } uint8_t * const base = (uint8_t *) ctx->vtcm_base; @@ -3345,8 +3316,8 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t .n_rows = (int) n_rows, .k_block = params->k, .k_stride = params->act_stride, - .n_threads = act_threads, - .act_threads_div = act_threads_div, + .n_hvx = act_hvx, + .act_hvx_div = act_hvx_div, .k_div = k_div, .k_valid = params->k, .vtcm_f32_act = vtcm_f32_act, @@ -3405,7 +3376,7 @@ static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t int chunk_dst_cols = params->dst_cols - (int)nc; if (chunk_dst_cols > 0) { transfer_output_chunk_threaded(ctx, output, bias_chunk, vtcm_output, (int) n_rows, (int) n_cols, - params->dst_stride, params->bias_stride, chunk_dst_cols, n_threads); + params->dst_stride, params->bias_stride, chunk_dst_cols, n_hvx); } } } @@ -3431,20 +3402,20 @@ static void transfer_activation_chunk_gathered_threaded( size_t nb11, size_t nb12, int cne1, - int n_threads, + int n_hvx, int k_valid) { if (n_rows <= 0) return; - int chunks_per_thread = hmx_ceil_div(n_rows, n_threads); - chunks_per_thread = hex_align_up(chunks_per_thread, 2); + int chunks_per_hvx = hmx_ceil_div(n_rows, n_hvx); + chunks_per_hvx = hex_align_up(chunks_per_hvx, 2); - int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread); + int actual_hvx = hmx_ceil_div(n_rows, chunks_per_hvx); activation_transfer_gathered_task_state_t state = { .dst = dst, .src = src, - .n_tasks = actual_threads, + .n_tasks = actual_hvx, .n_tot_chunks = n_rows, - .n_chunks_per_task = chunks_per_thread, + .n_chunks_per_task = chunks_per_hvx, .k_block = k_block, .matrix_rows = matrix_rows, .cur_a = cur_a, @@ -3456,17 +3427,12 @@ static void transfer_activation_chunk_gathered_threaded( .start_row = start_row, .cne1 = cne1, .k_valid = k_valid, - .traces = ctx->trace, }; - worker_callback_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn : + work_queue_func_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn : transfer_activation_chunk_gathered_worker_fn; - if (actual_threads <= 1) { - worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, worker_fn, &state, actual_hvx); } static void transfer_output_chunk_scattered_threaded( @@ -3482,19 +3448,19 @@ static void transfer_output_chunk_scattered_threaded( size_t dst_nb1, size_t dst_nb2, int cne1, - int n_threads) { + int n_hvx) { if (n_rows <= 0) return; - int chunks_per_thread = hmx_ceil_div(n_rows, n_threads); - chunks_per_thread = hex_align_up(chunks_per_thread, 2); + int chunks_per_hvx = hmx_ceil_div(n_rows, n_hvx); + chunks_per_hvx = hex_align_up(chunks_per_hvx, 2); - int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread); + int actual_hvx = hmx_ceil_div(n_rows, chunks_per_hvx); output_transfer_scattered_task_state_t state = { .vtcm_src = vtcm_src, .dst = dst, - .n_tasks = actual_threads, + .n_tasks = actual_hvx, .n_tot_chunks = n_rows, - .n_chunks_per_task = chunks_per_thread, + .n_chunks_per_task = chunks_per_hvx, .n_cols = n_cols, .matrix_rows = matrix_rows, .cur_a = cur_a, @@ -3503,14 +3469,9 @@ static void transfer_output_chunk_scattered_threaded( .dst_nb2 = dst_nb2, .start_row = start_row, .cne1 = cne1, - .traces = ctx->trace, }; - if (actual_threads <= 1) { - transfer_output_chunk_scattered_worker_fn(1, 0, &state); - } else { - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_scattered_worker_fn, &state, actual_threads); - } + work_queue_run(ctx->hvx_queue, transfer_output_chunk_scattered_worker_fn, &state, actual_hvx); } static int hmx_mm_id_2d_f32(struct htp_context *ctx, @@ -3532,7 +3493,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, int mapping_stride, int m_start, int m_end, - int n_threads) { + int n_hvx) { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); @@ -3614,7 +3575,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, transfer_activation_chunk_gathered_threaded( ctx, vtcm_f16_act, activation, (int) mr, (int) n_rows, k, - matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_threads, k_valid); + matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_hvx, k_valid); // A0: Pre-fetch the first weight chunk (nc = 0) if (n > 0) { @@ -3636,7 +3597,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, dequantize_tiled_weight_chunk_to_fp16_tiles( ctx, vtcm_weight_bufs[i % 2], curr_raw, n_cols, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_hvx ); // Start weight DMA for the next chunk early @@ -3656,7 +3617,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, // D: Output Store transfer_output_chunk_scattered_threaded( ctx, dst + nc, vtcm_output, (int) mr, (int) n_rows, (int) n_cols, - matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_threads); + matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_hvx); } } @@ -3710,7 +3671,7 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k const dma_addr_t act_addr = act->data + (size_t) m_start * act_stride * act_elem_size; int ret = -1; - const int n_threads = kparams->n_threads; + const int n_hvx = kparams->n_hvx; if (kparams->kernel_type == HTP_MM_KERNEL_HMX_BATCHED) { hmx_mm_batched_params_t batch_params = { .dst = dst_ptr, @@ -3750,9 +3711,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k }; ret = hmx_mm_batched(octx->ctx, &batch_params, kparams->m_chunk, kparams->n_chunk, - kparams->pipeline, n_threads, - kparams->n_act_threads, - &kparams->div_n_act_threads, + kparams->pipeline, n_hvx, + kparams->n_act_hvx, + &kparams->div_n_act_hvx, &kparams->div_ne00_padded, kparams->vtcm_size); } else { @@ -3761,9 +3722,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k act_addr, src0->data, m_rows, k, n, act_stride, act_elem_size, (int) src0->nb[1], (int) src0->type, (int) act->ne[0], dst_stride, src2_stride, (int)dst->ne[0], - kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, - kparams->n_act_threads, - &kparams->div_n_act_threads, + kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_hvx, + kparams->n_act_hvx, + &kparams->div_n_act_hvx, &kparams->div_ne00_padded, kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size ); @@ -3849,7 +3810,7 @@ static int hmx_mm_op_matmul_id( (int) src0->nb[1], (int) src0->type, kparams->tile_size, kparams->aligned_tile_size, matrix_rows, cur_a, mmctx->mapping_stride, - m_start, m_end, (int) octx->n_threads); + m_start, m_end, (int) octx->n_hvx); if (ret != 0) { FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret); return HTP_STATUS_NO_SUPPORT; @@ -3881,8 +3842,8 @@ static int hvx_mm_matmul_id( work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; @@ -3893,13 +3854,13 @@ static int hvx_mm_matmul_id( mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, ne10); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_hvx, 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false); const size_t vtcm_size = L.total_bytes; @@ -3929,9 +3890,9 @@ static int hvx_mm_matmul_id( mmctx->vtcm_act_stride = act_row_size; mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10); - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_act_size_per_thread = L.act_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); mmctx->cur_m_start = 0; mmctx->cur_m_rows = act_nrows; @@ -3940,11 +3901,11 @@ static int hvx_mm_matmul_id( hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, hvx_mmid_task_func, mmctx, octx->n_hvx); return HTP_STATUS_OK; } @@ -4013,7 +3974,7 @@ static int hmx_mm_op_matmul_id_nx( (int) src_w->nb[1], (int) src_w->type, kparams->tile_size, kparams->aligned_tile_size, matrix_rows, cur_a, mmctx->mapping_stride, - m_start, m_end, (int) octx->n_threads); + m_start, m_end, (int) octx->n_hvx); if (ret != 0) { FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret); return HTP_STATUS_NO_SUPPORT; @@ -4048,8 +4009,8 @@ static int hvx_mm_matmul_id_nx( work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; @@ -4060,13 +4021,13 @@ static int hvx_mm_matmul_id_nx( mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_hvx, 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false); const size_t vtcm_size = L.total_bytes; @@ -4087,27 +4048,27 @@ static int hvx_mm_matmul_id_nx( mmctx->vtcm_act_stride = act_row_size; mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]); - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_act_size_per_thread = L.act_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); mmctx->cur_m_start = 0; mmctx->cur_m_rows = act_nrows; - FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, act %d:%d:%d nrows %u, vtcm %zu/%zu, threads %d\n", + FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, act %d:%d:%d nrows %u, vtcm %zu/%zu, nhvx %d\n", src0->ne[0], src0->ne[1], src0->ne[2], mmctx->type, src0->ne[1], act->ne[0], act->ne[1], act->ne[2], act_nrows, - L.total_bytes, octx->ctx->vtcm_size, octx->n_threads); + L.total_bytes, octx->ctx->vtcm_size, octx->n_hvx); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, hvx_mmid_task_func, mmctx, octx->n_hvx); return HTP_STATUS_OK; } @@ -4386,8 +4347,8 @@ int op_matmul_id(struct htp_ops_context * octx) { mmctx->src0_row_start = src0_row_start; mmctx->src0_row_end = src0_row_end; - mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + mmctx->src0_nrows_per_hvx = fastdiv(nrows + octx->n_hvx - 1, &octx->n_hvx_div); + mmctx->src0_nrows_per_hvx = hex_round_up(mmctx->src0_nrows_per_hvx, 32); if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) == 0) { s = hvx_mm_matmul_id(octx, mmctx, act_nrows > 1 ? hvx_mm_id_tiled : hvx_mv_id_tiled); @@ -4530,10 +4491,10 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) { const uint32_t nb = (act->ne[0] + qk - 1) / qk; const uint32_t total_nb = act_nrows * nb; - worker_callback_t quant_task_func; + work_queue_func_t quant_task_func; uint32_t n_quant_tasks = 1; - if (act_nrows < octx->n_threads) { - n_quant_tasks = MIN(total_nb, octx->n_threads); + if (act_nrows < octx->n_hvx) { + n_quant_tasks = MIN(total_nb, octx->n_hvx); quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { uint32_t ib_first = (total_nb * ith) / n_quant_tasks; @@ -4544,14 +4505,14 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) { mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_tasks = MIN(act_nrows, octx->n_threads); + n_quant_tasks = MIN(act_nrows, octx->n_hvx); quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } const size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]); struct htp_mm_hvx_vtcm_layout L; - htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads, + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_hvx, 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, false, true); const size_t vtcm_size = L.total_bytes; @@ -4572,19 +4533,19 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) { mmctx->vtcm_act_stride = act_row_size; mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]); - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); - mmctx->vtcm_act_size_per_thread = L.act_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); + mmctx->vtcm_src0_size_per_hvx = fastdiv(L.src0_bytes, &octx->n_hvx_div); + mmctx->vtcm_act_size_per_hvx = L.act_bytes; + mmctx->vtcm_dst_size_per_hvx = fastdiv(L.dst_bytes, &octx->n_hvx_div); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_rows_per_hvx = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->n_quant_tasks = n_quant_tasks; - work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + work_queue_run(octx->ctx->hvx_queue, quant_task_func, mmctx, n_quant_tasks); - work_queue_run(octx->ctx->work_queue, hvx_mm_nx_2d_tiled, mmctx, octx->n_threads); + work_queue_run(octx->ctx->hvx_queue, hvx_mm_nx_2d_tiled, mmctx, octx->n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 4971ed958..0c16fe7b8 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -64,8 +64,8 @@ struct htp_mm_kernel_params { uint8_t unused; // reserved uint8_t n_hmx; // 1 = use HMX, 0 = use HVX - uint8_t n_threads; // Number of threads to spawn - uint8_t n_act_threads; // Number of threads for activation preparation + uint8_t n_hvx; // Number of threads to spawn + uint8_t n_act_hvx; // Number of threads for activation preparation uint8_t n_prefetch; // Prefetch lookahead buffers/rows in VTCM uint8_t n_weights; // Number of weights for fused NX @@ -83,7 +83,7 @@ struct htp_mm_kernel_params { struct fastdiv_values div_r2; struct fastdiv_values div_r3; struct fastdiv_values div_ne12; - struct fastdiv_values div_n_act_threads; + struct fastdiv_values div_n_act_hvx; struct fastdiv_values div_ne00_padded; }; @@ -310,7 +310,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( size_t nc, uint32_t group_size, bool pipeline, - uint32_t act_threads, + uint32_t act_hvx, uint32_t aligned_tile_size, size_t bias_size ) { @@ -323,7 +323,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t activation_area_size = hex_align_up(group_size * act_head_stride * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); const size_t output_area_size = hex_align_up(group_size * mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); const size_t scratch_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + const size_t min_f32_size = hex_align_up(act_hvx * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); // Group A: Permanent activation tiles and scales size_t off_group_a = 0; @@ -343,7 +343,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t group_b_size = off_group_b - off_group_a; // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) - const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t max_f32_size = act_hvx * 64 * k * sizeof(float); const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); size_t off_group_c = off_group_a; VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); @@ -367,7 +367,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t vec_dot_size = k * sizeof(uint16_t); const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype); - const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + const size_t min_f32_size = hex_align_up(act_hvx * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); const size_t weight_area_size = is_quant ? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) : hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE); @@ -395,7 +395,7 @@ static inline void htp_mm_hmx_vtcm_layout_build( const size_t group_b_size = off_group_b - off_group_a; // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) - const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t max_f32_size = act_hvx * 64 * k * sizeof(float); const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); size_t off_group_c = off_group_a; VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); @@ -423,7 +423,7 @@ static inline void htp_mm_hvx_vtcm_layout_build( int wtype, uint32_t ne10, // k uint32_t act_nrows, // m_total - uint32_t n_threads, + uint32_t n_hvx, size_t dst_row_size, size_t src0_row_size, size_t act_row_size, @@ -443,13 +443,13 @@ static inline void htp_mm_hvx_vtcm_layout_build( uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); + size_t weight_sz_per_hvx = hex_round_up(n_prefetch * tile_row_size, 128); size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128); size_t raw_row_size = htp_mm_act_raw_row_size(ne10); - src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer + src0_sz = weight_sz_per_hvx * n_hvx; // shared single-weight prefetch buffer act_sz = q_act_sz; // quantized activation buffer bias_sz = 0; dst_sz = 0; @@ -459,13 +459,13 @@ static inline void htp_mm_hvx_vtcm_layout_build( const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); const uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * tile_row_size, 256); + size_t src0_sz_per_hvx = htp_mm_round_up(n_prefetch * tile_row_size, 256); act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256); size_t raw_row_size = htp_mm_act_raw_row_size(ne10); - src0_sz = src0_sz_per_thread * n_threads; + src0_sz = src0_sz_per_hvx * n_hvx; dst_sz = 0; bias_sz = 0; act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); @@ -477,8 +477,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( case HTP_MM_KERNEL_HVX_F16_F16_VTCM: { size_t f16_act_row_size = htp_mm_round_up(ne10 * 2, 128); act_sz = htp_mm_round_up(f16_act_row_size * act_nrows, 256); - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; - dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_hvx; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_hvx : 0; act_raw_sz = hex_round_up(hex_round_up(ne10 * sizeof(float), 128) * act_nrows, 128); break; } @@ -486,8 +486,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: { size_t f32_act_row_size = htp_mm_round_up(ne10 * 4, 128); act_sz = htp_mm_round_up(f32_act_row_size * act_nrows, 256); - src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; - dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_hvx; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_hvx : 0; act_raw_sz = 0; break; } @@ -499,11 +499,11 @@ static inline void htp_mm_hvx_vtcm_layout_build( uint32_t tile_row_size = n_weight_tiles * aligned_tile_size; size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - src0_sz = tiled_vtcm_size * n_threads; + src0_sz = tiled_vtcm_size * n_hvx; act_sz = htp_mm_round_up(q_act_row_size * act_nrows, 256); - size_t dst_slice_per_thread = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0; - dst_sz = dst_slice_per_thread * n_threads; + size_t dst_slice_per_hvx = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_hvx - 1) / n_hvx, 128) : 0; + dst_sz = dst_slice_per_hvx * n_hvx; size_t raw_row_size = htp_mm_act_raw_row_size(ne10); act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); break; @@ -542,7 +542,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params( int wtype, uint32_t ne10, uint32_t act_nrows, - uint32_t n_threads, + uint32_t n_hvx, size_t dst_row_size, size_t src0_row_size, size_t act_row_size, @@ -554,7 +554,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params( ) { struct htp_mm_hvx_vtcm_layout L; htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, act_nrows, n_threads, + &L, kernel_type, wtype, ne10, act_nrows, n_hvx, dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); @@ -599,14 +599,14 @@ static inline bool htp_mm_hvx_solve_vtcm_params( } htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, m_chunk, n_threads, + &L, kernel_type, wtype, ne10, m_chunk, n_hvx, dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); while (m_chunk > 2 && L.total_bytes > vtcm_budget) { m_chunk -= 2; htp_mm_hvx_vtcm_layout_build( - &L, kernel_type, wtype, ne10, m_chunk, n_threads, + &L, kernel_type, wtype, ne10, m_chunk, n_hvx, dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false ); } @@ -621,18 +621,18 @@ static inline bool htp_mm_hvx_solve_vtcm_params( } static inline size_t htp_mm_hmx_get_2d_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size, size_t bias_size + int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_hvx, uint32_t aligned_tile_size, size_t bias_size ) { struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_threads, aligned_tile_size, bias_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_hvx, aligned_tile_size, bias_size); return L.total_bytes; } static inline size_t htp_mm_hmx_get_batched_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_threads, size_t bias_size) { + int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_hvx, size_t bias_size) { (void)pipeline; struct htp_mm_hmx_vtcm_layout L; - htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_threads, 0, bias_size); + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_hvx, 0, bias_size); return L.total_bytes; } @@ -642,23 +642,23 @@ static inline bool htp_mm_hmx_solve_batched_params( uint32_t ne01_padded, uint32_t ne11, uint32_t group_size, - int n_threads, + int n_hvx, bool pipeline, size_t bias_size, size_t vtcm_budget, size_t * m_chunk_out, size_t * n_chunk_out, - int * act_threads_out, + int * act_hvx_out, size_t * vtcm_size_out ) { size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; + int best_act_hvx = 0; size_t best_m_chunk = 0; size_t best_n_chunk = 0; size_t best_vtcm_size = 0; - int act_threads = n_threads; - while (act_threads >= 1) { + int act_hvx = n_hvx; + while (act_hvx >= 1) { size_t group_overhead = htp_mm_hmx_get_batched_overhead() + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0); size_t group_size_per_n, group_size_per_m, group_size_per_mn; htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); @@ -670,30 +670,30 @@ static inline bool htp_mm_hmx_solve_batched_params( if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_threads, bias_size); + size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_hvx, bias_size); if (exact_size <= vtcm_budget) { size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_hvx > best_act_hvx)) { best_mblocks = mblocks; - best_act_threads = act_threads; + best_act_hvx = act_hvx; best_m_chunk = m_chunk_candidate; best_n_chunk = n_chunk_candidate; best_vtcm_size = exact_size; } } } - if (act_threads == 1) { - act_threads = 0; + if (act_hvx == 1) { + act_hvx = 0; } else { - act_threads /= 2; + act_hvx /= 2; } } - if (best_act_threads > 0) { + if (best_act_hvx > 0) { *m_chunk_out = best_m_chunk; *n_chunk_out = best_n_chunk; *vtcm_size_out = best_vtcm_size; - *act_threads_out = best_act_threads; + *act_hvx_out = best_act_hvx; return true; } return false; @@ -706,7 +706,7 @@ static inline bool htp_mm_hmx_solve_2d_params( uint32_t ne01_padded, uint32_t ne11_padded, uint32_t m_for_cost, - int n_threads, + int n_hvx, bool pipeline, bool is_matmul_id, uint32_t aligned_tile_size, @@ -714,19 +714,19 @@ static inline bool htp_mm_hmx_solve_2d_params( size_t vtcm_budget, size_t * m_chunk_out, size_t * n_chunk_out, - int * act_threads_out, + int * act_hvx_out, size_t * vtcm_size_out ) { size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; + int best_act_hvx = 0; size_t best_m_chunk = 0; size_t best_n_chunk = 0; size_t best_vtcm_size = 0; const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded; - int act_threads = n_threads; - while (act_threads >= 1) { + int act_hvx = n_hvx; + while (act_hvx >= 1) { size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id) + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0); size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); @@ -738,30 +738,30 @@ static inline bool htp_mm_hmx_solve_2d_params( if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size, bias_size); + size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_hvx, aligned_tile_size, bias_size); if (exact_size <= vtcm_budget) { size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_hvx > best_act_hvx)) { best_mblocks = mblocks; - best_act_threads = act_threads; + best_act_hvx = act_hvx; best_m_chunk = m_chunk_candidate; best_n_chunk = n_chunk_candidate; best_vtcm_size = exact_size; } } } - if (act_threads == 1) { - act_threads = 0; + if (act_hvx == 1) { + act_hvx = 0; } else { - act_threads /= 2; + act_hvx /= 2; } } - if (best_act_threads > 0) { + if (best_act_hvx > 0) { *m_chunk_out = best_m_chunk; *n_chunk_out = best_n_chunk; *vtcm_size_out = best_vtcm_size; - *act_threads_out = best_act_threads; + *act_hvx_out = best_act_hvx; return true; } return false; diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c index cda6df318..b77e95a90 100644 --- a/ggml/src/ggml-hexagon/htp/pad-ops.c +++ b/ggml/src/ggml-hexagon/htp/pad-ops.c @@ -13,7 +13,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -100,7 +100,7 @@ struct htp_pad_context { int32_t lp2, rp2; int32_t lp3, rp3; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_dst_rows; uint32_t row_start; @@ -136,8 +136,8 @@ struct htp_pad_context { \ const size_t type_size = pctx->type_size; \ \ - const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \ - const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows); + const uint32_t row_start = pctx->row_start + pctx->nrows_per_hvx * ith; \ + const uint32_t row_end = MIN(row_start + pctx->nrows_per_hvx, pctx->row_start + pctx->total_dst_rows); #define htp_pad_dma_preamble \ @@ -146,21 +146,21 @@ struct htp_pad_context { const size_t dst_row_size = pctx->dst_row_size; \ const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ \ - uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \ - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \ + uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; \ + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; \ \ - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; // --------------------------------------------------------------------------- // HVX vectorized PAD kernel // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { @@ -213,7 +213,7 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da // HVX + DMA PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; @@ -252,7 +252,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void // Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops, // push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; @@ -318,12 +318,12 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void // HVX circular PAD kernel // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_circular(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { @@ -387,7 +387,7 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, // HVX + DMA circular PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- -static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) { +static void pad_job_per_hvx_hvx_circular_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; @@ -417,7 +417,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i // Main loop: pop completed DMAs, assemble circular row in VTCM with // aligned HVX ops, push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; @@ -521,11 +521,11 @@ int op_pad(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { return HTP_STATUS_NO_SUPPORT; @@ -533,12 +533,12 @@ int op_pad(struct htp_ops_context * octx) { struct htp_pad_vtcm_layout layout = { 0 }; if (kparams->use_dma) { - htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads); + htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_hvx); - octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; - octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; - octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; - octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.size_per_hvx = layout.src0_bytes_per_hvx; + octx->dst_spad.size_per_hvx = layout.dst_bytes_per_hvx; + octx->src0_spad.size = n_hvx * layout.src0_bytes_per_hvx; + octx->dst_spad.size = n_hvx * layout.dst_bytes_per_hvx; octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); octx->src0_spad.src = NULL; @@ -552,7 +552,7 @@ int op_pad(struct htp_ops_context * octx) { .lp1 = lp1, .rp1 = rp1, .lp2 = lp2, .rp2 = rp2, .lp3 = lp3, .rp3 = rp3, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_dst_rows = nrows, .row_start = row_start, .type_size = type_size, @@ -571,16 +571,16 @@ int op_pad(struct htp_ops_context * octx) { switch (kparams->kernel_type) { case HTP_PAD_KERNEL_HVX_CIRCULAR_DMA: - work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_circular_dma, &pctx, n_hvx); break; case HTP_PAD_KERNEL_HVX_CIRCULAR: - work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_circular, &pctx, n_hvx); break; case HTP_PAD_KERNEL_HVX_DMA: - work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx_dma, &pctx, n_hvx); break; default: - work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, pad_job_per_hvx_hvx, &pctx, n_hvx); break; } diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.h b/ggml/src/ggml-hexagon/htp/pad-ops.h index 5b8557538..15357e5f6 100644 --- a/ggml/src/ggml-hexagon/htp/pad-ops.h +++ b/ggml/src/ggml-hexagon/htp/pad-ops.h @@ -19,31 +19,31 @@ struct htp_pad_vtcm_layout { size_t off_dst; size_t src_row_size_aligned; size_t dst_row_size_aligned; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; }; static inline void htp_pad_vtcm_layout_build( struct htp_pad_vtcm_layout * L, size_t src_row_size, size_t dst_row_size, - uint32_t n_threads + uint32_t n_hvx ) { L->src_row_size_aligned = hex_round_up(src_row_size, 128); L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); - L->src0_bytes_per_thread = 2 * L->src_row_size_aligned; - L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned; + L->src0_bytes_per_hvx = 2 * L->src_row_size_aligned; + L->dst_bytes_per_hvx = 2 * L->dst_row_size_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } struct htp_pad_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_rows; uint32_t use_dma; uint32_t elem_size; diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.c b/ggml/src/ggml-hexagon/htp/pool-ops.c index b41c7bed2..12cd8ea90 100644 --- a/ggml/src/ggml-hexagon/htp/pool-ops.c +++ b/ggml/src/ggml-hexagon/htp/pool-ops.c @@ -5,7 +5,7 @@ #include "hex-common.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -249,29 +249,29 @@ struct pool_2d_context { const struct htp_pool_2d_kernel_params * kparams; struct htp_pool_vtcm_layout layout; pool_plane_fn_t pool_plane; - uint32_t n_threads; + uint32_t n_hvx; uint32_t plane_start; uint32_t plane_count; - uint32_t planes_per_thread; + uint32_t planes_per_hvx; }; -static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) { +static void pool_2d_thread(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct pool_2d_context * ctx = (struct pool_2d_context *) data; const struct htp_pool_2d_kernel_params * p = ctx->kparams; const struct htp_pool_vtcm_layout * L = &ctx->layout; const struct htp_tensor * src0 = ctx->octx->src[0]; const struct htp_tensor * dst = ctx->octx->dst; pool_plane_fn_t pool_plane = ctx->pool_plane; - const uint32_t planes_per_thread = ctx->planes_per_thread; - const uint32_t first = ctx->plane_start + ith * planes_per_thread; - const uint32_t last = MIN(first + planes_per_thread, ctx->plane_start + ctx->plane_count); + const uint32_t planes_per_hvx = ctx->planes_per_hvx; + const uint32_t first = ctx->plane_start + ith * planes_per_hvx; + const uint32_t last = MIN(first + planes_per_hvx, ctx->plane_start + ctx->plane_count); if (first >= last) { return; } - struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith]; - dma_queue * dma_queue = ctx->octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_queue = th->dma_q; const uint32_t src_plane_bytes = L->src_plane_bytes; const uint32_t dst_plane_bytes = L->dst_plane_bytes; @@ -279,8 +279,8 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) { const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base; - uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread; - uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread; + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) }; @@ -355,12 +355,12 @@ int op_pool_2d(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = MIN(p->n_threads, plane_count); - if (!htp_ops_context_set_n_threads(octx, n_threads)) { + const uint32_t n_hvx = MIN(p->n_hvx, plane_count); + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t planes_per_thread = fastdiv(plane_count + n_threads - 1, &octx->n_threads_div); + const uint32_t planes_per_hvx = fastdiv(plane_count + n_hvx - 1, &octx->n_hvx_div); pool_plane_fn_t pool_plane; switch (p->kernel_type) { @@ -375,13 +375,13 @@ int op_pool_2d(struct htp_ops_context * octx) { .octx = octx, .kparams = p, .pool_plane = pool_plane, - .n_threads = n_threads, + .n_hvx = n_hvx, .plane_start = plane_start, .plane_count = plane_count, - .planes_per_thread = planes_per_thread, + .planes_per_hvx = planes_per_hvx, }; - htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_threads); - work_queue_run(octx->ctx->work_queue, pool_2d_thread, &ctx, n_threads); + htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_hvx); + work_queue_run(octx->ctx->hvx_queue, pool_2d_thread, &ctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.h b/ggml/src/ggml-hexagon/htp/pool-ops.h index 178e5d00f..e113bb9dd 100644 --- a/ggml/src/ggml-hexagon/htp/pool-ops.h +++ b/ggml/src/ggml-hexagon/htp/pool-ops.h @@ -29,7 +29,7 @@ struct htp_pool_2d_kernel_params { uint32_t stride_y; int32_t pad_x; int32_t pad_y; - uint32_t n_threads; + uint32_t n_hvx; uint32_t planes; uint32_t planes_per_chunk; uint32_t pool_op; @@ -51,8 +51,8 @@ struct htp_pool_vtcm_layout { size_t off_dst; uint32_t src_plane_bytes; uint32_t dst_plane_bytes; - size_t src_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src_spad_half_size; size_t dst_spad_half_size; }; @@ -63,19 +63,19 @@ static inline void htp_pool_vtcm_layout_build( uint32_t src_y, uint32_t dst_x, uint32_t dst_y, - uint32_t n_threads + uint32_t n_hvx ) { // Full-plane double buffering (256 bytes guard space for vector loads) L->src_plane_bytes = src_x * src_y * sizeof(float); L->dst_plane_bytes = dst_x * dst_y * sizeof(float); L->src_spad_half_size = hex_round_up(L->src_plane_bytes + 256, 128); L->dst_spad_half_size = hex_round_up(L->dst_plane_bytes, 128); - L->src_bytes_per_thread = 2 * L->src_spad_half_size; - L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } @@ -85,10 +85,10 @@ static inline bool htp_pool_solve_layout( uint32_t src_y, uint32_t dst_x, uint32_t dst_y, - uint32_t n_threads, + uint32_t n_hvx, size_t vtcm_budget ) { - htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_threads); + htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_hvx); return layout->total_bytes <= vtcm_budget; } diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c index 827ef24c5..a9e4d5c86 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.c +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c @@ -13,7 +13,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" #include "repeat-ops.h" @@ -22,14 +22,14 @@ struct htp_repeat_context { struct htp_ops_context * octx; const struct htp_repeat_kernel_params * kparams; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t total_dst_rows; // ne1 * ne2 * ne3 uint32_t row_start; size_t type_size; }; -static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * data) { +static void repeat_job_per_hvx(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_repeat_context * rctx = (const struct htp_repeat_context *) data; struct htp_ops_context * octx = rctx->octx; const struct htp_repeat_kernel_params * kparams = rctx->kparams; @@ -60,13 +60,13 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat const size_t row_bytes = ne00 * rctx->type_size; - const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_hvx, rctx->row_start + rctx->total_dst_rows); if (row_start >= row_end) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { @@ -119,7 +119,7 @@ int op_repeat(struct htp_ops_context * octx) { const struct htp_repeat_kernel_params * kparams = (const struct htp_repeat_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -141,12 +141,12 @@ int op_repeat(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_repeat_context rctx = { .octx = octx, .kparams = kparams, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .total_dst_rows = nrows, .row_start = row_start, .type_size = type_size, @@ -157,7 +157,7 @@ int op_repeat(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], kparams->nr0, kparams->nr1, kparams->nr2, kparams->nr3); - work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, repeat_job_per_hvx, &rctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.h b/ggml/src/ggml-hexagon/htp/repeat-ops.h index 47759e037..1561994d4 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.h +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.h @@ -5,7 +5,7 @@ #include struct htp_repeat_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t nr0; uint32_t nr1; uint32_t nr2; diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.c b/ggml/src/ggml-hexagon/htp/roll-ops.c index 77ded0fe9..aa7a0ad4b 100644 --- a/ggml/src/ggml-hexagon/htp/roll-ops.c +++ b/ggml/src/ggml-hexagon/htp/roll-ops.c @@ -14,7 +14,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" #include "roll-ops.h" @@ -25,7 +25,7 @@ struct htp_roll_context { uint32_t row_start; uint32_t nrows; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; }; static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) { @@ -199,19 +199,19 @@ static int roll_dma_f32_strided(struct htp_ops_context * octx) { return HTP_STATUS_OK; } -static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void roll_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_roll_context * rctx = (struct htp_roll_context *) data; struct htp_ops_context * octx = rctx->octx; htp_roll_preamble; - const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->nrows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_hvx * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_hvx, rctx->row_start + rctx->nrows); if (row_start >= row_end) { return; } - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t row = row_start; row < row_end; row++) { @@ -275,20 +275,20 @@ int execute_op_roll_f32(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, nrows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_roll_context rctx = { .octx = octx, .kparams = kparams, .row_start = row_start, .nrows = nrows, - .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), }; - work_queue_run(octx->ctx->work_queue, roll_thread_f32, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, roll_thread_f32, &rctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.h b/ggml/src/ggml-hexagon/htp/roll-ops.h index 2646aaa4c..f98fb8ea9 100644 --- a/ggml/src/ggml-hexagon/htp/roll-ops.h +++ b/ggml/src/ggml-hexagon/htp/roll-ops.h @@ -12,7 +12,7 @@ enum htp_roll_kernel_type { struct htp_roll_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_rows; uint32_t i0_src0; uint32_t n0; diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 8c55ac3af..1c73de10b 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -66,12 +66,12 @@ struct htp_rope_context { float theta_powers[32]; float corr_dims[2]; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; struct htp_ops_context * octx; uint8_t * vtcm_base; - size_t spad_per_thread; + size_t spad_per_hvx; size_t theta_cache_offset; size_t src0_row_size; @@ -532,7 +532,7 @@ static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t } } -static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { +static void rope_job_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_rope_context * rctx = (struct htp_rope_context *) data; struct htp_ops_context * octx = rctx->octx; @@ -544,10 +544,10 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_rope_preamble; const uint32_t src0_nrows = rctx->nrows; - const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = rctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows); + const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, rctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { @@ -560,12 +560,12 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); // VTCM setup - uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread); + uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_hvx); float * theta_cache = (float *) (src0_spad_base); src0_spad_base = src0_spad_base + rctx->theta_cache_offset; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const int32_t * pos = (const int32_t *) (uintptr_t) src1->data; const float * freq_factors = rctx->freq_factors; @@ -701,7 +701,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } assert(octx->ctx->vtcm_size >= kparams->vtcm_size); @@ -729,7 +729,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; const uint32_t ne0 = dst->ne[0]; const size_t src0_row_size = src0->ne[0] * sizeof(float); @@ -738,14 +738,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const size_t dst_row_stride = dst->nb[1]; struct htp_rope_vtcm_layout layout; - htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_threads, src2 ? src2->ne[0] : 0); + htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_hvx, src2 ? src2->ne[0] : 0); struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); rctx.octx = octx; rctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_spad); - rctx.spad_per_thread = layout.bytes_per_thread; + rctx.spad_per_hvx = layout.bytes_per_hvx; rctx.theta_cache_offset = layout.theta_cache_size_aligned; if (src2) { @@ -790,14 +790,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rctx.src0_nrows = nrows; rctx.nrows = nrows; rctx.row_start = row_start; - rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + rctx.src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); rctx.div_ne2_ne1 = kparams->div_ne2_ne1; rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, rope_job_f32, &rctx, n_hvx); return err; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h index 0c28ef826..a21b85c06 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.h +++ b/ggml/src/ggml-hexagon/htp/rope-ops.h @@ -10,7 +10,7 @@ #define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) struct htp_rope_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t vtcm_size; struct fastdiv_values div_ne2_ne1; @@ -27,7 +27,7 @@ struct htp_rope_vtcm_layout { size_t total_bytes; size_t off_spad; size_t off_freq_factors; - size_t bytes_per_thread; + size_t bytes_per_hvx; size_t theta_cache_size_aligned; size_t src0_row_size_aligned; size_t freq_factors_size_aligned; @@ -36,7 +36,7 @@ struct htp_rope_vtcm_layout { static inline void htp_rope_vtcm_layout_build( struct htp_rope_vtcm_layout * L, uint32_t ne00, - uint32_t n_threads, + uint32_t n_hvx, uint32_t n_freq_factors ) { const size_t src0_row_size = ne00 * sizeof(float); @@ -47,10 +47,10 @@ static inline void htp_rope_vtcm_layout_build( L->src0_row_size_aligned = src0_row_size_aligned; L->theta_cache_size_aligned = theta_cache_size_aligned; L->freq_factors_size_aligned = freq_factors_size_aligned; - L->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; + L->bytes_per_hvx = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_hvx * n_hvx); VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_freq_factors, freq_factors_size_aligned, n_freq_factors > 0); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c index 191cf0c6a..add91408f 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c @@ -61,32 +61,33 @@ struct set_rows_context { uint8_t * vtcm_base; uint32_t task_start; uint32_t tasks; - uint32_t tasks_per_thread; + uint32_t tasks_per_hvx; }; #define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \ -static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ +static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(const htp_thread_t * th, unsigned int nth, \ + unsigned int ith, void *data) { \ struct set_rows_context * srctx = (struct set_rows_context *)data; \ struct htp_ops_context * octx = srctx->octx; \ const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ set_rows_preamble; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = srctx->tasks_per_thread; \ + struct htp_thread_trace * tr = th->trace; \ + const uint32_t dr = srctx->tasks_per_hvx; \ const uint32_t ir0 = srctx->task_start + dr * ith; \ if (ir0 >= srctx->task_start + srctx->tasks) { \ return; \ } \ const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \ - dma_queue * dma_q = octx->ctx->dma[ith]; \ + dma_queue * dma_q = th->dma_q; \ const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_src0) + \ - ith * vtcm_layout->src0_bytes_per_thread; \ + ith * vtcm_layout->src0_bytes_per_hvx; \ uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_dst) + \ - ith * vtcm_layout->dst_bytes_per_thread; \ + ith * vtcm_layout->dst_bytes_per_hvx; \ const uint32_t src0_row_size = ne00 * sizeof(float); \ const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \ - const uint32_t nrows_per_thread = ir1 - ir0; \ - const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \ + const uint32_t nrows_per_hvx = ir1 - ir0; \ + const uint32_t total_steps = ne03 * ne02 * nrows_per_hvx; \ uint32_t pi_step = 0; \ uint32_t pi02 = 0; \ uint32_t pi03 = 0; \ @@ -102,7 +103,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig src0_data), \ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ pi_step++; \ - if (pi_step == nrows_per_thread) { \ + if (pi_step == nrows_per_hvx) { \ pi_step = 0; \ pi02++; \ if (pi02 == ne02) { \ @@ -147,7 +148,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig dma_make_data(src_spad, psrc0_data), \ vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ pi_step++; \ - if (pi_step == nrows_per_thread) { \ + if (pi_step == nrows_per_hvx) { \ pi_step = 0; \ pi02++; \ if (pi02 == ne02) { \ @@ -157,7 +158,7 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig } \ } \ ci_step++; \ - if (ci_step == nrows_per_thread) { \ + if (ci_step == nrows_per_hvx) { \ ci_step = 0; \ ci02++; \ ci11_base++; \ @@ -211,11 +212,11 @@ int op_set_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, (uint32_t) kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // l2fetch the src1 (indices) tensor in the main thread hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]); @@ -225,9 +226,9 @@ int op_set_rows(struct htp_ops_context * octx) { srctx.kparams = kparams; srctx.task_start = task_start; srctx.tasks = tasks; - srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); + srctx.tasks_per_hvx = fastdiv(tasks + n_hvx - 1, &octx->n_hvx_div); - htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads); + htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_hvx); srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; work_queue_func_t q_func = NULL; @@ -240,15 +241,15 @@ int op_set_rows(struct htp_ops_context * octx) { default: return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n", + FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-hvx %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - srctx.vtcm_layout.src0_bytes_per_thread * n_threads, - srctx.vtcm_layout.dst_bytes_per_thread * n_threads, - n_threads); + srctx.vtcm_layout.src0_bytes_per_hvx * n_hvx, + srctx.vtcm_layout.dst_bytes_per_hvx * n_hvx, + n_hvx); - work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, q_func, &srctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.h index b9cb6cc90..85086b08e 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h @@ -6,7 +6,7 @@ #include "htp-vtcm.h" struct htp_set_rows_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_tasks; uint32_t vtcm_size; }; @@ -16,8 +16,8 @@ struct htp_set_rows_vtcm_layout { size_t off_src0; size_t off_dst; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; @@ -27,7 +27,7 @@ static inline void htp_set_rows_vtcm_layout_build( struct htp_set_rows_vtcm_layout * L, int dst_type, uint32_t ne00, - uint32_t n_threads) { + uint32_t n_hvx) { size_t src0_row_size = ne00 * sizeof(float); size_t dst_row_size = htp_tensor_get_row_size(dst_type, ne00); @@ -38,12 +38,12 @@ static inline void htp_set_rows_vtcm_layout_build( L->src0_spad_half_size = src0_row_size_aligned; L->dst_spad_half_size = dst_row_size_aligned; - L->src0_bytes_per_thread = src0_row_size_aligned * 2; - L->dst_bytes_per_thread = dst_row_size_aligned * 2; + L->src0_bytes_per_hvx = src0_row_size_aligned * 2; + L->dst_bytes_per_hvx = dst_row_size_aligned * 2; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c index d65c3fae3..22aebcd5e 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.c +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c @@ -13,7 +13,7 @@ #include "hvx-utils.h" #include "hex-fastdiv.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -38,9 +38,9 @@ struct htp_softmax_context { uint8_t * vtcm_src1; uint8_t * vtcm_dst; - uint32_t vtcm_src0_size_per_thread; - uint32_t vtcm_src1_size_per_thread; - uint32_t vtcm_dst_size_per_thread; + uint32_t vtcm_src0_size_per_hvx; + uint32_t vtcm_src1_size_per_hvx; + uint32_t vtcm_dst_size_per_hvx; uint32_t src0_spad_half_size; uint32_t src1_spad_half_size; @@ -65,7 +65,7 @@ struct htp_softmax_context { struct fastdiv_values div_ne12; struct fastdiv_values div_ne13; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; uint32_t nrows; @@ -287,19 +287,19 @@ static const softmax_compute_fn_t softmax_kernels[HTP_SOFTMAX_KERNEL_COUNT] = { [HTP_SOFTMAX_KERNEL_MASK_F16] = compute_fast_softmax_f32_mask_f16, }; -static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) { +static void softmax_thread_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; struct htp_ops_context * octx = smctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t src0_nrows = smctx->nrows; - const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = smctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, smctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -311,13 +311,13 @@ static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) const size_t src0_row_size = src0->ne[0] * sizeof(float); const size_t dst_row_size = src0->ne[0] * sizeof(float); - uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half = smctx->src0_spad_half_size; const size_t dst_vtcm_half = smctx->dst_spad_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { dma_addr_t cur_dst = data_dst + r * dst_row_size; @@ -357,20 +357,20 @@ static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) dma_queue_flush(dma_q); } -static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * data) { +static void softmax_thread_mask_dma(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; struct htp_ops_context * octx = smctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; const uint32_t src0_nrows = smctx->nrows; - const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = smctx->src0_nrows_per_hvx; - const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, smctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -384,9 +384,9 @@ static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * d const size_t dst_row_size = src0->ne[0] * sizeof(float); const size_t mask_row_size = smctx->use_f16 ? (src1->ne[0] * sizeof(__fp16)) : (src1->ne[0] * sizeof(float)); - uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); - uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_thread); - uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_hvx); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half = smctx->src0_spad_half_size; const size_t src1_vtcm_half = smctx->src1_spad_half_size; @@ -408,7 +408,7 @@ static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * d const struct fastdiv_values * div_ne12 = &smctx->div_ne12; const struct fastdiv_values * div_ne13 = &smctx->div_ne13; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { dma_addr_t cur_dst = data_dst + r * dst_row_size; @@ -492,7 +492,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { const struct htp_softmax_kernel_params * kparams = (const struct htp_softmax_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -520,8 +520,8 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); row_start = range.start; nrows = range.count; - if (nrows < octx->n_threads) { - htp_ops_context_set_n_threads(octx, nrows ? nrows : 1); + if (nrows < octx->n_hvx) { + htp_ops_context_set_n_hvx(octx, nrows ? nrows : 1); } } @@ -529,7 +529,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base; struct htp_softmax_vtcm_layout layout; @@ -538,7 +538,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { kparams->use_src1 ? octx->src[1]->ne[0] : 0, kparams->use_src1 != 0, kparams->use_f16 != 0, - kparams->n_threads); + kparams->n_hvx); struct htp_softmax_context smctx = { .octx = octx, @@ -553,9 +553,9 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_dst), .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, layout.off_src1, kparams->use_src1), - .vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread, - .vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread, - .vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread, + .vtcm_src0_size_per_hvx = (uint32_t) layout.src0_bytes_per_hvx, + .vtcm_src1_size_per_hvx = (uint32_t) layout.src1_bytes_per_hvx, + .vtcm_dst_size_per_hvx = (uint32_t) layout.dst_bytes_per_hvx, .src0_spad_half_size = (uint32_t) layout.src0_spad_half_size, .src1_spad_half_size = (uint32_t) layout.src1_spad_half_size, @@ -580,7 +580,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { .div_ne12 = kparams->div_ne12, .div_ne13 = kparams->div_ne13, - .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .row_start = row_start, .nrows = nrows, }; @@ -596,7 +596,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { } work_queue_func_t task_func = kparams->use_src1 ? softmax_thread_mask_dma : softmax_thread_dma; - work_queue_run(octx->ctx->work_queue, task_func, &smctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, task_func, &smctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.h b/ggml/src/ggml-hexagon/htp/softmax-ops.h index ef6567306..9e5f9c759 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.h +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.h @@ -18,7 +18,7 @@ enum htp_softmax_kernel_type { struct htp_softmax_kernel_params { uint8_t kernel_type; - uint32_t n_threads; + uint32_t n_hvx; uint32_t vtcm_size; uint32_t n_head; @@ -49,9 +49,9 @@ struct htp_softmax_vtcm_layout { size_t off_dst; size_t off_src1; - size_t src0_bytes_per_thread; - size_t dst_bytes_per_thread; - size_t src1_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t dst_bytes_per_hvx; + size_t src1_bytes_per_hvx; size_t src0_spad_half_size; size_t dst_spad_half_size; @@ -64,7 +64,7 @@ static inline void htp_softmax_vtcm_layout_build( uint32_t ne10, bool use_src1, bool use_f16, - uint32_t n_threads + uint32_t n_hvx ) { size_t src0_row_size = ne00 * sizeof(float); size_t dst_row_size = ne00 * sizeof(float); @@ -79,14 +79,14 @@ static inline void htp_softmax_vtcm_layout_build( L->src1_spad_half_size = src1_row_size_aligned; // Double buffering: 2 half-buffers per thread - L->src0_bytes_per_thread = src0_row_size_aligned * 2; - L->dst_bytes_per_thread = dst_row_size_aligned * 2; - L->src1_bytes_per_thread = src1_row_size_aligned * 2; + L->src0_bytes_per_hvx = src0_row_size_aligned * 2; + L->dst_bytes_per_hvx = dst_row_size_aligned * 2; + L->src1_bytes_per_hvx = src1_row_size_aligned * 2; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_thread * n_threads, use_src1); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_hvx * n_hvx, use_src1); L->total_bytes = off; } diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c index 628ca65e8..73eff927e 100644 --- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c @@ -4,7 +4,7 @@ #include #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -18,7 +18,7 @@ struct htp_solve_tri_context { struct htp_ops_context * octx; const struct htp_solve_tri_kernel_params * kparams; - uint32_t jobs_per_thread; + uint32_t jobs_per_hvx; uint32_t total_jobs; uint32_t job_start; uint32_t k_chunks; @@ -79,7 +79,7 @@ static inline void solve_tri_row_hvx(const float * A_row, } // Batch-level thread: each job is one full batch. -static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void solve_tri_batch_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data; struct htp_ops_context * octx = sctx->octx; const struct htp_solve_tri_kernel_params * kparams = sctx->kparams; @@ -96,10 +96,10 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t col_block = VLEN_FP32; const uint32_t k_full = k & ~(col_block - 1); - const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith; - const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); + const uint32_t start_batch = sctx->job_start + sctx->jobs_per_hvx * ith; + const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_hvx, sctx->job_start + sctx->total_jobs); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch); for (uint32_t batch = start_batch; batch < end_batch; ++batch) { @@ -141,7 +141,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void } // Chunk-level thread: each job is one (batch, col_chunk) pair. -static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void solve_tri_chunk_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data; struct htp_ops_context * octx = sctx->octx; const struct htp_solve_tri_kernel_params * kparams = sctx->kparams; @@ -155,10 +155,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t ne02 = src0->ne[2]; - const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith; - const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); + const uint32_t start_job = sctx->job_start + sctx->jobs_per_hvx * ith; + const uint32_t end_job = MIN(start_job + sctx->jobs_per_hvx, sctx->job_start + sctx->total_jobs); - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job); for (uint32_t job = start_job; job < end_job; ++job) { @@ -233,23 +233,23 @@ int op_solve_tri(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, njobs))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_solve_tri_context sctx = { .octx = octx, .kparams = kparams, - .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .jobs_per_hvx = fastdiv(njobs + n_hvx - 1, &octx->n_hvx_div), .total_jobs = njobs, .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, solve_tri_batch_thread_f32, &sctx, n_hvx); } else { const uint32_t total_jobs = kparams->total_jobs; @@ -267,23 +267,23 @@ int op_solve_tri(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) { + if (!htp_ops_context_set_n_hvx(octx, MIN(kparams->n_hvx, njobs))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; struct htp_solve_tri_context sctx = { .octx = octx, .kparams = kparams, - .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .jobs_per_hvx = fastdiv(njobs + n_hvx - 1, &octx->n_hvx_div), .total_jobs = njobs, .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, solve_tri_chunk_thread_f32, &sctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h index 327415814..6a46cfdc4 100644 --- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h @@ -5,7 +5,7 @@ #include struct htp_solve_tri_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t total_batches; uint32_t total_jobs; uint32_t k_chunks; diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c index 4b06b18f2..d75304fef 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.c +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c @@ -15,7 +15,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ops.h" #include "htp-tensor.h" #include "hvx-utils.h" @@ -24,7 +24,7 @@ struct htp_ssm_conv_context { struct htp_ops_context * octx; const struct htp_ssm_conv_kernel_params * kparams; - uint32_t nrows_per_thread; + uint32_t nrows_per_hvx; uint32_t d_inner_tile; uint32_t row_start; uint32_t nrows; @@ -80,8 +80,8 @@ static inline void hvx_transpose_32x32_f32(HVX_Vector m[32]) { } // HVX deinterleave for d_conv == 4: channel-major raw VTCM -> tap-major T VTCM -static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride) { - for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { +static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride) { + for (uint32_t cb = 0; cb < d_inner_per_hvx; cb += VLEN_FP32) { HVX_Vector v0 = *(const HVX_Vector *)(raw + (cb + 0) * 4); HVX_Vector v1 = *(const HVX_Vector *)(raw + (cb + 8) * 4); HVX_Vector v2 = *(const HVX_Vector *)(raw + (cb + 16) * 4); @@ -101,15 +101,15 @@ static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint } // HVX transpose for general d_conv <= 32: channel-major raw VTCM -> tap-major T VTCM -static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { +static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride, uint32_t d_conv) { uint32_t __attribute__((aligned(VLEN))) mask_buf[VLEN_FP32] = { 0 }; for (uint32_t j = 0; j < d_conv; ++j) { mask_buf[j] = 0xFFFFFFFF; } const HVX_Vector mask = *(const HVX_Vector *) mask_buf; - for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { - const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_thread - cb); + for (uint32_t cb = 0; cb < d_inner_per_hvx; cb += VLEN_FP32) { + const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_hvx - cb); HVX_Vector sub[32]; for (uint32_t r = 0; r < cb_n; ++r) { const float * ch_ptr = raw + (cb + r) * d_conv; @@ -127,11 +127,11 @@ static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, ui } } -static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { - if (d_conv == 4 && (d_inner_per_thread % VLEN_FP32 == 0)) { - hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_thread, d_inner_stride); +static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_hvx, uint32_t d_inner_stride, uint32_t d_conv) { + if (d_conv == 4 && (d_inner_per_hvx % VLEN_FP32 == 0)) { + hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_hvx, d_inner_stride); } else { - hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_thread, d_inner_stride, d_conv); + hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_hvx, d_inner_stride, d_conv); } } @@ -186,7 +186,7 @@ static inline void hvx_ssm_conv_transpose_block(const float * raw_block, } // Single-token decode worker (n_t == 1) -static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void * data) { +static void ssm_conv_thread_f32_decode(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; struct htp_ops_context * octx = scctx->octx; const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; @@ -195,14 +195,14 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void const struct htp_tensor * restrict src1 = octx->src[1]; const struct htp_tensor * restrict dst = octx->dst; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t d_conv = kparams->d_conv; const uint32_t d_inner = kparams->d_inner; const uint32_t n_s = kparams->n_s; const bool apply_silu = kparams->silu != 0; - const uint32_t dr = scctx->nrows_per_thread; + const uint32_t dr = scctx->nrows_per_hvx; const uint32_t ir0 = scctx->row_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); @@ -210,19 +210,19 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void return; } - const uint32_t d_inner_per_thread = ir1 - ir0; - const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + const uint32_t d_inner_per_hvx = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_hvx, VLEN_FP32); const uint32_t d_inner_tile = scctx->d_inner_tile; const size_t src0_stride_inner_bytes = src0->nb[1]; const size_t src0_stride_seq_bytes = src0->nb[2]; const size_t dst_stride_seq_bytes = dst->nb[2]; - uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; - uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_hvx; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; - const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_bytes = (size_t) d_inner_per_hvx * d_conv * sizeof(float); const size_t weight_raw_size = hex_round_up(weight_bytes, 128); float * src1_raw = (float *) src1_spad_base; @@ -236,12 +236,12 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) }; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; // raw_dot keeps both operands in the DMA layout, so src1 needs no prep pass - const bool raw_dot = (d_conv == 4) && (d_inner_per_thread % VLEN_FP32 == 0); + const bool raw_dot = (d_conv == 4) && (d_inner_per_hvx % VLEN_FP32 == 0); - const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile); + const uint32_t n_tiles = fastdiv(d_inner_per_hvx + d_inner_tile - 1, &kparams->div_d_inner_tile); const uint32_t n_chunks = n_s * n_tiles; const size_t row_bytes = d_conv * sizeof(float); @@ -250,14 +250,14 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void #define SSM_CONV_DECODE_PUSH_FETCH(c) \ do { \ - const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off_fetch); \ + const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off_fetch); \ const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \ (ir0 + tile_off_fetch) * src0_stride_inner_bytes; \ dma_queue_push(dma_q, \ dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \ row_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \ tile_off_fetch += d_inner_tile; \ - if (tile_off_fetch >= d_inner_per_thread) { \ + if (tile_off_fetch >= d_inner_per_hvx) { \ tile_off_fetch = 0; \ s_fetch++; \ } \ @@ -277,7 +277,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void if (!raw_dot) { // Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); - hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_hvx, d_inner_stride, d_conv); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); } @@ -285,7 +285,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void uint32_t tile_off = 0; for (uint32_t c = 0; c < n_chunks; ++c) { - const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off); + const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off); if (c >= 2) { dma_queue_pop(dma_q); // writeback of chunk c-2, frees dst_tile[c & 1] @@ -328,7 +328,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void } tile_off += d_inner_tile; - if (tile_off >= d_inner_per_thread) { + if (tile_off >= d_inner_per_hvx) { tile_off = 0; i3++; } @@ -347,7 +347,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void } // Multi-token prefill worker (n_t > 1) -static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void * data) { +static void ssm_conv_thread_f32_prefill(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; struct htp_ops_context * octx = scctx->octx; const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; @@ -356,7 +356,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void const struct htp_tensor * restrict src1 = octx->src[1]; const struct htp_tensor * restrict dst = octx->dst; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const uint32_t d_conv = kparams->d_conv; const uint32_t d_inner = kparams->d_inner; @@ -365,7 +365,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void const uint32_t ncs = src0->ne[0]; const bool apply_silu = kparams->silu != 0; - const uint32_t dr = scctx->nrows_per_thread; + const uint32_t dr = scctx->nrows_per_hvx; const uint32_t ir0 = scctx->row_start + dr * ith; const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); @@ -373,8 +373,8 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void return; } - const uint32_t d_inner_per_thread = ir1 - ir0; - const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + const uint32_t d_inner_per_hvx = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_hvx, VLEN_FP32); const uint32_t d_inner_tile = scctx->d_inner_tile; const size_t src0_stride_inner_bytes = src0->nb[1]; @@ -382,11 +382,11 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void const size_t dst_stride_token_bytes = dst->nb[1]; const size_t dst_stride_seq_bytes = dst->nb[2]; - uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; - uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_hvx; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_hvx; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_hvx; - const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_bytes = (size_t) d_inner_per_hvx * d_conv * sizeof(float); const size_t weight_raw_size = hex_round_up(weight_bytes, 128); float * src1_raw = (float *) src1_spad_base; @@ -403,9 +403,9 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) }; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; - const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile); + const uint32_t n_tiles = fastdiv(d_inner_per_hvx + d_inner_tile - 1, &kparams->div_d_inner_tile); const uint32_t n_chunks = n_s * n_tiles; const size_t row_bytes = ncs * sizeof(float); @@ -416,14 +416,14 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void // Two fetches run ahead, so the queue order is F0 F1 W0 F2 W1 ... and pops follow it. #define SSM_CONV_PUSH_FETCH(c) \ do { \ - const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off_fetch); \ + const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off_fetch); \ const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \ (ir0 + tile_off_fetch) * src0_stride_inner_bytes; \ dma_queue_push(dma_q, \ dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \ raw_stride_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \ tile_off_fetch += d_inner_tile; \ - if (tile_off_fetch >= d_inner_per_thread) { \ + if (tile_off_fetch >= d_inner_per_hvx) { \ tile_off_fetch = 0; \ s_fetch++; \ } \ @@ -442,7 +442,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void // Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); - hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_hvx, d_inner_stride, d_conv); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); const uint32_t C_TILE = VLEN_FP32; @@ -454,7 +454,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void uint32_t tile_off = 0; for (uint32_t c = 0; c < n_chunks; ++c) { - const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off); + const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_hvx - tile_off); const float * restrict raw = src0_tile_raw[c & 1]; float * restrict out = dst_tile[c & 1]; @@ -533,7 +533,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void } tile_off += d_inner_tile; - if (tile_off >= d_inner_per_thread) { + if (tile_off >= d_inner_per_hvx) { tile_off = 0; i3++; } @@ -558,7 +558,7 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { const struct htp_ssm_conv_kernel_params * kparams = (const struct htp_ssm_conv_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -586,18 +586,18 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - const uint32_t n_threads = octx->n_threads; - const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); - const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32); + const uint32_t n_hvx = octx->n_hvx; + const uint32_t raw_rpt = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); + const uint32_t d_inner_per_hvx = hex_round_up(raw_rpt, VLEN_FP32); struct htp_ssm_conv_vtcm_layout L; - htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_threads); + htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_hvx, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_hvx); uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; - octx->src0_spad.size_per_thread = L.src0_bytes_per_thread; - octx->src1_spad.size_per_thread = L.src1_bytes_per_thread; - octx->dst_spad.size_per_thread = L.dst_bytes_per_thread; + octx->src0_spad.size_per_hvx = L.src0_bytes_per_hvx; + octx->src1_spad.size_per_hvx = L.src1_bytes_per_hvx; + octx->dst_spad.size_per_hvx = L.dst_bytes_per_hvx; octx->src0_spad.size = L.src0_bytes; octx->src1_spad.size = L.src1_bytes; @@ -613,7 +613,7 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { struct htp_ssm_conv_context scctx = { .octx = octx, .kparams = kparams, - .nrows_per_thread = d_inner_per_thread, + .nrows_per_hvx = d_inner_per_hvx, .d_inner_tile = kparams->d_inner_tile, .row_start = row_start, .nrows = nrows, @@ -626,9 +626,9 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) { kparams->n_t == 1 ? "decode" : "prefill"); if (kparams->n_t == 1) { - work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_decode, &scctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, ssm_conv_thread_f32_decode, &scctx, n_hvx); } else { - work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_prefill, &scctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, ssm_conv_thread_f32_prefill, &scctx, n_hvx); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.h index cd4180857..7c3f3041f 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.h +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h @@ -9,7 +9,7 @@ #include "htp-vtcm.h" struct htp_ssm_conv_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t d_conv; uint32_t d_inner; uint32_t n_t; @@ -27,9 +27,9 @@ struct htp_ssm_conv_vtcm_layout { size_t off_src0; size_t off_src1; size_t off_dst; - size_t src0_bytes_per_thread; - size_t src1_bytes_per_thread; - size_t dst_bytes_per_thread; + size_t src0_bytes_per_hvx; + size_t src1_bytes_per_hvx; + size_t dst_bytes_per_hvx; size_t src0_bytes; size_t src1_bytes; size_t dst_bytes; @@ -37,34 +37,34 @@ struct htp_ssm_conv_vtcm_layout { static inline void htp_ssm_conv_vtcm_layout_build( struct htp_ssm_conv_vtcm_layout * L, - uint32_t d_inner_per_thread, + uint32_t d_inner_per_hvx, uint32_t d_conv, uint32_t n_t, uint32_t d_inner_tile, - uint32_t n_threads + uint32_t n_hvx ) { - const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_thread * d_conv * sizeof(float), 128) + 128; - const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_thread * sizeof(float), 128); + const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_hvx * d_conv * sizeof(float), 128) + 128; + const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_hvx * sizeof(float), 128); - L->src1_bytes_per_thread = src1_raw_bytes + src1_T_bytes; + L->src1_bytes_per_hvx = src1_raw_bytes + src1_T_bytes; if (n_t == 1) { const size_t src0_tile_raw_bytes = hex_round_up((size_t) d_inner_tile * d_conv * sizeof(float), 128); const size_t src0_T_bytes = hex_round_up((size_t) d_conv * d_inner_tile * sizeof(float), 128); - L->src0_bytes_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes; - L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128); + L->src0_bytes_per_hvx = 2 * src0_tile_raw_bytes + src0_T_bytes; + L->dst_bytes_per_hvx = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128); } else { const uint32_t ncs = d_conv - 1 + n_t; const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float); const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128); const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * raw_stride_bytes, 128); - L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T; - L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128); + L->src0_bytes_per_hvx = 2 * src0_tile_raw + src0_block_T; + L->dst_bytes_per_hvx = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128); } - L->src0_bytes = L->src0_bytes_per_thread * n_threads; - L->src1_bytes = L->src1_bytes_per_thread * n_threads; - L->dst_bytes = L->dst_bytes_per_thread * n_threads; + L->src0_bytes = L->src0_bytes_per_hvx * n_hvx; + L->src1_bytes = L->src1_bytes_per_hvx * n_hvx; + L->dst_bytes = L->dst_bytes_per_hvx * n_hvx; size_t off = 0; VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes); diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c index 3d931ed19..b3f8a7090 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c @@ -14,7 +14,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -51,18 +51,18 @@ struct sum_rows_context { uint32_t ne00; size_t src_stride; size_t dst_stride; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint32_t total_rows; }; -static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) { +static void sum_rows_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void *data) { const struct sum_rows_context * smctx = (const struct sum_rows_context *) data; - const uint32_t rows_per_thread = smctx->rows_per_thread; + const uint32_t rows_per_hvx = smctx->rows_per_hvx; const uint32_t total_rows = smctx->total_rows; - const uint32_t start_row = rows_per_thread * ith; - const uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); + const uint32_t start_row = rows_per_hvx * ith; + const uint32_t end_row = MIN(start_row + rows_per_hvx, total_rows); if (start_row >= end_row) { return; @@ -78,7 +78,7 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) // Calculate actual number of rows for this thread const uint32_t n_rows = end_row - start_row; - struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); for (uint32_t ir = 0; ir < n_rows; ir++) { @@ -109,7 +109,7 @@ int op_sum_rows(struct htp_ops_context * octx) { const struct htp_sum_rows_kernel_params * kparams = (const struct htp_sum_rows_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -131,8 +131,8 @@ int op_sum_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + const uint32_t n_hvx = octx->n_hvx; + const uint32_t rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); struct sum_rows_context smctx = { .octx = octx, @@ -141,11 +141,11 @@ int op_sum_rows(struct htp_ops_context * octx) { .ne00 = ne00, .src_stride = nb01, .dst_stride = nb1, - .rows_per_thread = rows_per_thread, + .rows_per_hvx = rows_per_hvx, .total_rows = nrows, }; - work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, sum_rows_thread_f32, &smctx, n_hvx); return HTP_STATUS_OK; } @@ -153,15 +153,15 @@ int op_sum_rows(struct htp_ops_context * octx) { struct sum_context { struct htp_ops_context * octx; const float * src_data; - float partial_sums[HTP_MAX_NTHREADS]; + float partial_sums[HTP_MAX_NHVX]; uint32_t total_elems; - uint32_t elems_per_thread; + uint32_t elems_per_hvx; }; -static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) { +static void sum_thread_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct sum_context * sctx = (struct sum_context *) data; - const uint32_t start = sctx->elems_per_thread * ith; - const uint32_t end = MIN(start + sctx->elems_per_thread, sctx->total_elems); + const uint32_t start = sctx->elems_per_hvx * ith; + const uint32_t end = MIN(start + sctx->elems_per_hvx, sctx->total_elems); if (start >= end) { sctx->partial_sums[ith] = 0.0f; @@ -171,7 +171,7 @@ static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) { const uint32_t n = end - start; const float * src = sctx->src_data + start; - struct htp_thread_trace * tr = &sctx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); hex_l2fetch_block((const void *) src, n * sizeof(float)); @@ -196,7 +196,7 @@ int op_sum(struct htp_ops_context * octx) { const struct htp_sum_rows_kernel_params * kparams = (const struct htp_sum_rows_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -210,20 +210,20 @@ int op_sum(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t elems_per_thread = kparams->elems_per_thread; + const uint32_t n_hvx = octx->n_hvx; + const uint32_t elems_per_hvx = kparams->elems_per_hvx; struct sum_context sctx = { .octx = octx, .src_data = (const float *) src0->data, .total_elems = total_elems, - .elems_per_thread = elems_per_thread, + .elems_per_hvx = elems_per_hvx, }; - work_queue_run(octx->ctx->work_queue, sum_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, sum_thread_f32, &sctx, n_hvx); float sum = 0.0f; - for (uint32_t i = 0; i < n_threads; i++) { + for (uint32_t i = 0; i < n_hvx; i++) { sum += sctx.partial_sums[i]; } ((float *) dst->data)[0] = sum; @@ -250,17 +250,17 @@ struct argmax_context { uint32_t dst_stride; uint32_t row_start; uint32_t nrows; - uint32_t rows_per_thread; - uint32_t elems_per_thread; + uint32_t rows_per_hvx; + uint32_t elems_per_hvx; - float partial_max[HTP_MAX_NTHREADS]; - int32_t partial_idx[HTP_MAX_NTHREADS]; + float partial_max[HTP_MAX_NHVX]; + int32_t partial_idx[HTP_MAX_NHVX]; }; -static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * data) { +static void argmax_thread_single_row(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct argmax_context * actx = (struct argmax_context *) data; - const uint32_t start = actx->elems_per_thread * ith; - const uint32_t end = MIN(start + actx->elems_per_thread, actx->ne00); + const uint32_t start = actx->elems_per_hvx * ith; + const uint32_t end = MIN(start + actx->elems_per_hvx, actx->ne00); if (start >= end) { actx->partial_max[ith] = -INFINITY; @@ -271,7 +271,7 @@ static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * const uint32_t n = end - start; const float * src = actx->src_data + start; - struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); hex_l2fetch_block((const void *) src, n * sizeof(float)); @@ -286,16 +286,16 @@ static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); } -static void argmax_thread_multi_row(unsigned int nth, unsigned int ith, void * data) { +static void argmax_thread_multi_row(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct argmax_context * actx = (struct argmax_context *) data; - const uint32_t r0 = actx->row_start + actx->rows_per_thread * ith; - const uint32_t r1 = MIN(r0 + actx->rows_per_thread, actx->row_start + actx->nrows); + const uint32_t r0 = actx->row_start + actx->rows_per_hvx * ith; + const uint32_t r1 = MIN(r0 + actx->rows_per_hvx, actx->row_start + actx->nrows); if (r0 >= r1) { return; } - struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r0); for (uint32_t r = r0; r < r1; r++) { @@ -329,7 +329,7 @@ int op_argmax(struct htp_ops_context * octx) { const struct htp_sum_rows_kernel_params * kparams = (const struct htp_sum_rows_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -350,8 +350,8 @@ int op_argmax(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t elems_per_thread = kparams->elems_per_thread; + const uint32_t n_hvx = octx->n_hvx; + const uint32_t elems_per_hvx = kparams->elems_per_hvx; struct argmax_context actx = { .octx = octx, @@ -362,15 +362,15 @@ int op_argmax(struct htp_ops_context * octx) { .dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t), .row_start = 0, .nrows = 1, - .rows_per_thread = 1, - .elems_per_thread = elems_per_thread, + .rows_per_hvx = 1, + .elems_per_hvx = elems_per_hvx, }; - work_queue_run(octx->ctx->work_queue, argmax_thread_single_row, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, argmax_thread_single_row, &actx, n_hvx); float best_val = actx.partial_max[0]; int32_t best_idx = actx.partial_idx[0]; - for (uint32_t i = 1; i < n_threads; i++) { + for (uint32_t i = 1; i < n_hvx; i++) { if (actx.partial_max[i] > best_val) { best_val = actx.partial_max[i]; best_idx = actx.partial_idx[i]; @@ -397,8 +397,8 @@ int op_argmax(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; - const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + const uint32_t n_hvx = octx->n_hvx; + const uint32_t rows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div); struct argmax_context actx = { .octx = octx, @@ -409,10 +409,10 @@ int op_argmax(struct htp_ops_context * octx) { .dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t), .row_start = row_start, .nrows = nrows, - .rows_per_thread = rows_per_thread, - .elems_per_thread = 0, + .rows_per_hvx = rows_per_hvx, + .elems_per_hvx = 0, }; - work_queue_run(octx->ctx->work_queue, argmax_thread_multi_row, &actx, n_threads); + work_queue_run(octx->ctx->hvx_queue, argmax_thread_multi_row, &actx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h index cc9ddd483..0b8acf030 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h @@ -4,8 +4,8 @@ #include struct htp_sum_rows_kernel_params { - uint32_t n_threads; - uint32_t elems_per_thread; + uint32_t n_hvx; + uint32_t elems_per_hvx; }; #if defined(__cplusplus) diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c index 215f01ced..9c4dc583c 100644 --- a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c +++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c @@ -9,7 +9,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "hex-common.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -25,11 +25,11 @@ struct htp_top_k_mask_context { struct htp_ops_context * octx; const struct htp_top_k_mask_kernel_params * kparams; - uint32_t rows_per_thread; + uint32_t rows_per_hvx; uint32_t row0, row1; // rows this device handles uint32_t off_extra, off_score; // parts inside one row input slot uint32_t col0, ncols_out; // PICKS: columns this device writes back - atomic_uint barrier; // PICKS: arrivals, barrier k waits for k * n_threads + atomic_uint barrier; // PICKS: arrivals, barrier k waits for k * n_hvx uint8_t * blocks; uint8_t * mask; @@ -348,7 +348,7 @@ static inline void top_k_mask_next(uint32_t * r, uint32_t * ct, uint32_t n_col_t // ROWS: the thread count follows the device rows (mdev), so the slice of an n byte copy is sized here static inline uint32_t top_k_mask_slice(const struct htp_ops_context * octx, unsigned int nth, uint32_t n) { - return hex_round_up(fastdiv(n + nth - 1, &octx->n_threads_div), HTP_TOP_K_MASK_UNIT); + return hex_round_up(fastdiv(n + nth - 1, &octx->n_hvx_div), HTP_TOP_K_MASK_UNIT); } // Push this thread's slice (128B aligned) of a contiguous n byte copy. Returns the push count. @@ -363,12 +363,12 @@ static inline uint32_t top_k_mask_push_slice(dma_queue * dma_q, unsigned int ith } // Pass 1 (ROWS): load the blocks table and the resident mask row, split across threads. Fill the tiles of this thread with -inf meanwhile. -static void top_k_mask_preload(unsigned int nth, unsigned int ith, void * data) { +static void top_k_mask_preload(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data; const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; struct htp_ops_context * octx = tctx->octx; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; uint32_t n_pushed = 0; @@ -393,12 +393,12 @@ static void top_k_mask_preload(unsigned int nth, unsigned int ith, void * data) // Pass 2 (ROWS): depth-2 DMA pipeline over the items of rows [row_first, row_last), item = (row, col tile). // The row inputs alternate between 2 slots by row. -static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, unsigned int ith, uint32_t row_first, uint32_t row_last) { +static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, const htp_thread_t * th, unsigned int ith, uint32_t row_first, uint32_t row_last) { struct htp_ops_context * octx = tctx->octx; const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; const struct htp_tensor * dst = octx->dst; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const bool is_f16 = kp->flags & HTP_TOP_K_MASK_F16; const uint32_t elem_size = kp->elem_size; @@ -480,16 +480,16 @@ static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, uns } // Pass 2 (ROWS): whole rows split across threads, the col tiles of a row in order -static void top_k_mask_thread_rows(unsigned int nth, unsigned int ith, void * data) { +static void top_k_mask_thread_rows(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { const struct htp_top_k_mask_context * tctx = (const struct htp_top_k_mask_context *) data; - const uint32_t ir0 = tctx->row0 + ith * tctx->rows_per_thread; - const uint32_t ir1 = MIN(ir0 + tctx->rows_per_thread, tctx->row1); + const uint32_t ir0 = tctx->row0 + ith * tctx->rows_per_hvx; + const uint32_t ir1 = MIN(ir0 + tctx->rows_per_hvx, tctx->row1); if (ir0 >= ir1) { return; } - top_k_mask_run_items(tctx, ith, ir0, ir1); + top_k_mask_run_items(tctx, th, ith, ir0, ir1); } static inline void top_k_mask_barrier(atomic_uint * b, uint32_t target, struct htp_thread_trace * tr, uint16_t info) { @@ -501,15 +501,15 @@ static inline void top_k_mask_barrier(atomic_uint * b, uint32_t target, struct h htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, info); } -// PICKS, one pass over n_virt = kp->n_threads virtual workers (host slices); real worker ith runs the virtual workers ith, ith + nth, ... +// PICKS, one pass over n_virt = kp->n_hvx virtual workers (host slices); real worker ith runs the virtual workers ith, ith + nth, ... // A: shared preload and a slice of the flag clear per virtual worker, the mask slice of this worker into the row tile; // barrier; B: scatter keep flags of the picks of each virtual worker; barrier; C: select mask or -inf in this worker's slice and write it back -static void top_k_mask_thread_picks(unsigned int nth, unsigned int ith, void * data) { +static void top_k_mask_thread_picks(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data; const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; struct htp_ops_context * octx = tctx->octx; - dma_queue * dma_q = octx->ctx->dma[ith]; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = th->dma_q; + struct htp_thread_trace * tr = th->trace; const struct htp_tensor * top_k = octx->src[1]; const struct htp_tensor * score = octx->src[2]; @@ -520,13 +520,13 @@ static void top_k_mask_thread_picks(unsigned int nth, unsigned int ith, void * d uint8_t * out = tctx->tiles; uint8_t * flags = tctx->tiles + tile_bytes; - const uint32_t n_virt = kp->n_threads; + const uint32_t n_virt = kp->n_hvx; // this worker's slice of the device columns: the mask goes straight into the row tile, it is popped after the scan. // The device share of the columns depends on mdev, so this slice is sized here. const uint32_t out_b0 = tctx->col0 * kp->elem_size; const uint32_t out_nb = tctx->ncols_out * kp->elem_size; - const uint32_t wslice = hex_round_up(fastdiv(out_nb + nth - 1, &octx->n_threads_div), 128); + const uint32_t wslice = hex_round_up(fastdiv(out_nb + nth - 1, &octx->n_hvx_div), 128); const uint32_t wb0 = out_b0 + ith * wslice; const uint32_t wnb = ith * wslice < out_nb ? MIN(wslice, out_nb - ith * wslice) : 0; @@ -567,8 +567,8 @@ static void top_k_mask_thread_picks(unsigned int nth, unsigned int ith, void * d HVX_Vector * tmp = (HVX_Vector *) (tctx->scratch + ith * HTP_TOP_K_MASK_SCRATCH_BYTES); for (uint32_t v = ith; v < n_virt; v += nth) { - const uint32_t j0 = MIN(v * kp->u.picks.picks_per_thread, kp->n_top); - const uint32_t j1 = MIN(j0 + kp->u.picks.picks_per_thread, kp->n_top); + const uint32_t j0 = MIN(v * kp->u.picks.picks_per_hvx, kp->n_top); + const uint32_t j1 = MIN(j0 + kp->u.picks.picks_per_hvx, kp->n_top); const bool do_extra = v == n_virt - 1; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); @@ -624,12 +624,12 @@ int op_top_k_mask(struct htp_ops_context * octx) { } } - // fewer workers than kparams->n_threads always fit the VTCM layout: ROWS runs one row range per worker, PICKS loops over the host slices - const uint32_t max_threads = MIN((uint32_t) kp->n_threads, octx->ctx->n_threads); - if (!htp_ops_context_set_n_threads(octx, picks ? max_threads : MIN(max_threads, dev_rows))) { + // fewer workers than kparams->n_hvx always fit the VTCM layout: ROWS runs one row range per worker, PICKS loops over the host slices + const uint32_t max_hvx = MIN((uint32_t) kp->n_hvx, octx->ctx->n_hvx); + if (!htp_ops_context_set_n_hvx(octx, picks ? max_hvx : MIN(max_hvx, dev_rows))) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; uint8_t * vtcm = octx->ctx->vtcm_base; @@ -652,14 +652,14 @@ int op_top_k_mask(struct htp_ops_context * octx) { if (picks) { atomic_init(&tctx.barrier, 0); - work_queue_run(octx->ctx->work_queue, top_k_mask_thread_picks, &tctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, top_k_mask_thread_picks, &tctx, n_hvx); return HTP_STATUS_OK; } - tctx.rows_per_thread = fastdiv(dev_rows + n_threads - 1, &octx->n_threads_div); + tctx.rows_per_hvx = fastdiv(dev_rows + n_hvx - 1, &octx->n_hvx_div); - work_queue_run(octx->ctx->work_queue, top_k_mask_preload, &tctx, n_threads); - work_queue_run(octx->ctx->work_queue, top_k_mask_thread_rows, &tctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, top_k_mask_preload, &tctx, n_hvx); + work_queue_run(octx->ctx->hvx_queue, top_k_mask_thread_rows, &tctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h index ba1be96b8..9dcdfaa48 100644 --- a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h +++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h @@ -53,7 +53,7 @@ static inline uint64_t htp_top_k_mask_rowin_off_score(uint32_t n_top, uint32_t n // the caller compares total_bytes against the VTCM size static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_layout * L, - const struct htp_top_k_mask_shape * S, uint32_t kernel_type, uint32_t n_threads, uint32_t tile_cols) { + const struct htp_top_k_mask_shape * S, uint32_t kernel_type, uint32_t n_hvx, uint32_t tile_cols) { memset(L, 0, sizeof(*L)); // PICKS streams the mask straight into its row tile @@ -67,10 +67,10 @@ static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_l const uint64_t tile_bytes = htp_top_k_mask_align128((uint64_t) tile_cols * S->elem_size); // ROWS streams row inputs into 2 slots per thread. PICKS: one row, a row tile and a flag tile. - const uint64_t rowin_total = (picks ? 1 : 2 * (uint64_t) n_threads) * rowin_bytes; - const uint64_t tiles_total = picks ? 2 * tile_bytes : 2 * (uint64_t) n_threads * tile_bytes; + const uint64_t rowin_total = (picks ? 1 : 2 * (uint64_t) n_hvx) * rowin_bytes; + const uint64_t tiles_total = picks ? 2 * tile_bytes : 2 * (uint64_t) n_hvx * tile_bytes; const uint64_t mtiles_total = S->mask_resident ? 0 : tiles_total; - const uint64_t scratch = (uint64_t) n_threads * HTP_TOP_K_MASK_SCRATCH_BYTES; + const uint64_t scratch = (uint64_t) n_hvx * HTP_TOP_K_MASK_SCRATCH_BYTES; size_t off = 0; VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_blocks, (size_t) blocks_bytes, blocks_bytes > 0); @@ -90,9 +90,9 @@ static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_l // Set by the host, used as is by the NPU: no tensor shape, stride or layout is derived on the NPU. // Sizes of one slot or tile, the slot parts and the VTCM offsets are in HTP_TOP_K_MASK_UNIT bytes as u16. // Row strides: byte offset of flat row r of top_k, extra, score and dst is r * stride (the host requires row flattenable inputs). -// PICKS splits the work in n_threads virtual workers; fewer real workers loop over them. +// PICKS splits the work in n_hvx virtual workers; fewer real workers loop over them. struct htp_top_k_mask_kernel_params { - uint8_t n_threads, kernel_type, flags, elem_size; + uint8_t n_hvx, kernel_type, flags, elem_size; uint32_t vtcm_size; uint32_t n_kv, n_top, n_pick, n_extra; uint32_t ninf; // -inf cell pattern of elem_size @@ -116,7 +116,7 @@ struct htp_top_k_mask_kernel_params { } m; } rows; struct { - uint32_t picks_per_thread; // per virtual worker, multiple of the lanes of one vector + uint32_t picks_per_hvx; // per virtual worker, multiple of the lanes of one vector uint16_t blocks_slice_u, top_slice_u, extra_slice_u, score_slice_u, flags_slice_u; // per virtual worker slices } picks; } u; diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 3a6840169..19ade22f3 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -21,7 +21,7 @@ #include "htp-ops.h" #include "htp-tensor.h" #include "htp-vtcm.h" -#include "hex-profile.h" +#include "htp-trace.h" struct htp_unary_context; @@ -87,7 +87,7 @@ struct htp_unary_context { uint32_t block; uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; + uint32_t src0_nrows_per_hvx; uint32_t row_start; uint32_t nc; uint32_t col_tile; // tiled mode @@ -98,10 +98,10 @@ struct htp_unary_context { uint8_t * vtcm_src2; uint8_t * vtcm_dst; - size_t vtcm_src0_size_per_thread; - size_t vtcm_src1_size_per_thread; - size_t vtcm_src2_size_per_thread; - size_t vtcm_dst_size_per_thread; + size_t vtcm_src0_size_per_hvx; + size_t vtcm_src1_size_per_hvx; + size_t vtcm_src2_size_per_hvx; + size_t vtcm_dst_size_per_hvx; }; // Convert flat row index to DDR byte offset using the tensor's actual strides. @@ -1233,25 +1233,25 @@ static void tri_apply_tile_f32(const void * restrict src, void * restrict dst, } // 1. Standard row-block unary task (F32 and F16). -static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_row_block(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1260,8 +1260,8 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; @@ -1273,7 +1273,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); @@ -1330,25 +1330,25 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * da } // 2. RMS_NORM_MUL row-block task with weight buffer. -static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_rms_norm_mul_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1363,9 +1363,9 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo const uint32_t nb12 = src1->nb[2]; const uint32_t nb13 = src1->nb[3]; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL; - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_hvx)) : NULL; + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size; @@ -1380,7 +1380,7 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; if (uctx->broadcast_weight) { dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1), @@ -1464,25 +1464,25 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo } // 2b. RMS_NORM_MUL_ADD row-block task with broadcast weight and residual add. -static void unary_thread_rms_norm_mul_add_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_rms_norm_mul_add_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1493,10 +1493,10 @@ static void unary_thread_rms_norm_mul_add_f32(unsigned int nth, unsigned int ith const dma_addr_t data_src2 = uctx->data_src2; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * src1_vtcm_data = uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread); - uint8_t * src2_vtcm_data = uctx->vtcm_src2 + (ith * uctx->vtcm_src2_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * src1_vtcm_data = uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_hvx); + uint8_t * src2_vtcm_data = uctx->vtcm_src2 + (ith * uctx->vtcm_src2_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; @@ -1508,7 +1508,7 @@ static void unary_thread_rms_norm_mul_add_f32(unsigned int nth, unsigned int ith const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1), uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1); @@ -1576,25 +1576,25 @@ static void unary_thread_rms_norm_mul_add_f32(unsigned int nth, unsigned int ith } // 3. TRI row-block task with row index ir. -static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_tri_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const size_t src0_data_row_size = uctx->src0_data_row_size; const size_t dst_data_row_size = uctx->dst_data_row_size; const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1603,14 +1603,14 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; const uint32_t BLOCK = uctx->block; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); @@ -1663,22 +1663,22 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data } // 4. Pointwise tiled unary task. -static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_tiled(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const uint32_t col_tile = uctx->col_tile; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1687,13 +1687,13 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_half = uctx->src0_vtcm_half_size; const size_t dst_half = uctx->dst_vtcm_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; @@ -1774,23 +1774,23 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) } // 5. TRI tiled task. -static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void * data) { +static void unary_thread_tiled_tri_f32(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { (void) nth; const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; struct htp_ops_context * octx = uctx->octx; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + struct htp_thread_trace * tr = th->trace; htp_unary_preamble; - const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t src0_nrows_per_hvx = uctx->src0_nrows_per_hvx; const int32_t * op_params = octx->op_params; const uint32_t col_tile = uctx->col_tile; const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_hvx * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_hvx, uctx->row_start + src0_nrows); if (src0_start_row >= src0_end_row) { return; @@ -1799,13 +1799,13 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void const dma_addr_t data_src = uctx->data_src0; const dma_addr_t data_dst = uctx->data_dst; - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_hvx); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_hvx); const size_t src0_half = uctx->src0_vtcm_half_size; const size_t dst_half = uctx->dst_vtcm_half_size; - dma_queue * dma_q = octx->ctx->dma[ith]; + dma_queue * dma_q = th->dma_q; const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; @@ -1928,7 +1928,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + if (!htp_ops_context_set_n_hvx(octx, kparams->n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } @@ -1953,7 +1953,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = octx->n_threads; + const uint32_t n_hvx = octx->n_hvx; // Always 0 for F16 - htp_unary_solve_layout() keeps F16 on the row-block path, // since only F32 has unary_task_f32_tiled_* kernels. @@ -1984,17 +1984,17 @@ static int execute_op_unary(struct htp_ops_context * octx) { struct htp_unary_vtcm_layout L; htp_unary_vtcm_layout_build(&L, octx->op, src0_data_row_size, dst_data_row_size, src1_data_row_size, - src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_threads); + src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_hvx); FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - L.src0_bytes * n_threads, L.src1_bytes * n_threads, L.dst_bytes * n_threads); + L.src0_bytes * n_hvx, L.src1_bytes * n_hvx, L.dst_bytes * n_hvx); uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; struct htp_unary_context uctx = { .octx = octx, .kparams = kparams, - .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows_per_hvx = fastdiv(nrows + n_hvx - 1, &octx->n_hvx_div), .src0_nrows = nrows, .row_start = row_start, @@ -2025,15 +2025,15 @@ static int execute_op_unary(struct htp_ops_context * octx) { .vtcm_src2 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src2, L.src2_bytes > 0), .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst), - .vtcm_src0_size_per_thread = L.src0_bytes, - .vtcm_src1_size_per_thread = L.src1_bytes, - .vtcm_src2_size_per_thread = L.src2_bytes, - .vtcm_dst_size_per_thread = L.dst_bytes, + .vtcm_src0_size_per_hvx = L.src0_bytes, + .vtcm_src1_size_per_hvx = L.src1_bytes, + .vtcm_src2_size_per_hvx = L.src2_bytes, + .vtcm_dst_size_per_hvx = L.dst_bytes, }; FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile); - worker_callback_t task_func = NULL; + work_queue_func_t task_func = NULL; void * compute_func = NULL; if (col_tile) { @@ -2132,7 +2132,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { } uctx.compute = compute_func; - work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, task_func, &uctx, n_hvx); return err; } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 8699a1403..e03b2ac73 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -8,7 +8,7 @@ // Op-specific struct for precomputed unary params struct htp_unary_kernel_params { - uint32_t n_threads; + uint32_t n_hvx; uint32_t col_tile; uint32_t block; uint32_t broadcast_weight; @@ -112,7 +112,7 @@ static inline void htp_unary_vtcm_layout_build( uint32_t col_tile, uint32_t block, bool broadcast_weight, - uint32_t n_threads + uint32_t n_hvx ) { const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); @@ -138,10 +138,10 @@ static inline void htp_unary_vtcm_layout_build( } size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_threads); - VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_threads, L->src1_bytes > 0); - VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src2, L->src2_bytes * n_threads, L->src2_bytes > 0); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_hvx); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_hvx, L->src1_bytes > 0); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src2, L->src2_bytes * n_hvx, L->src2_bytes > 0); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_hvx); L->total_bytes = off; } @@ -153,11 +153,11 @@ static inline void htp_unary_solve_layout( uint32_t ne11, uint32_t max_block, bool broadcast_weight, - uint32_t n_threads, + uint32_t n_hvx, size_t vtcm_size, size_t elem_size, uint32_t * out_col_tile, - uint32_t * out_vtcm_row_per_thread + uint32_t * out_vtcm_row_per_hvx ) { const bool has_weight = (op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_RMS_NORM_MUL_ADD); @@ -171,35 +171,35 @@ static inline void htp_unary_solve_layout( const size_t src1_row_size_aligned = has_weight ? hex_round_up(src1_data_row_size, 128) : 0; size_t vtcm_size_per_row = 0; - size_t vtcm_row_per_thread = 0; + size_t vtcm_row_per_hvx = 0; if (op == HTP_OP_RMS_NORM_MUL_ADD) { - size_t src1_vtcm_total = n_threads * src1_row_size_aligned; + size_t src1_vtcm_total = n_hvx * src1_row_size_aligned; size_t available_vtcm = (vtcm_size > src1_vtcm_total) ? (vtcm_size - src1_vtcm_total) : 0; vtcm_size_per_row = 2 * (2 * src0_row_size_aligned + dst_row_size_aligned); - vtcm_row_per_thread = available_vtcm / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = available_vtcm / (n_hvx * vtcm_size_per_row); } else if (op == HTP_OP_RMS_NORM_MUL) { if (broadcast_weight) { size_t available_vtcm = vtcm_size; - size_t src1_vtcm_total = n_threads * src1_row_size_aligned; + size_t src1_vtcm_total = n_hvx * src1_row_size_aligned; if (available_vtcm > src1_vtcm_total) { available_vtcm -= src1_vtcm_total; } else { available_vtcm = 0; } vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); - vtcm_row_per_thread = available_vtcm / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = available_vtcm / (n_hvx * vtcm_size_per_row); } else { vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned + src1_row_size_aligned); - vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = vtcm_size / (n_hvx * vtcm_size_per_row); } } else { vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); - vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row); + vtcm_row_per_hvx = vtcm_size / (n_hvx * vtcm_size_per_row); } - if (max_block > 0 && vtcm_row_per_thread > max_block) { - vtcm_row_per_thread = max_block; + if (max_block > 0 && vtcm_row_per_hvx > max_block) { + vtcm_row_per_hvx = max_block; } const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM || @@ -207,25 +207,25 @@ static inline void htp_unary_solve_layout( op == HTP_OP_L2_NORM || op == HTP_OP_HADAMARD); // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*); // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops. - // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM + // NOTE: if F16 ends up with vtcm_row_per_hvx == 0 here (row too large for the VTCM // budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op // (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known // limitation; supporting it would require adding F16 tiled kernels. const bool is_f16 = (elem_size == sizeof(_Float16)); uint32_t col_tile = 0; - if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) { - const size_t per_thread_budget = vtcm_size / n_threads; - const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128); + if (vtcm_row_per_hvx == 0 && !is_reduction && !is_f16) { + const size_t per_hvx_budget = vtcm_size / n_hvx; + const size_t col_tile_bytes = hex_align_down(per_hvx_budget / 4, 128); col_tile = (uint32_t) (col_tile_bytes / elem_size); } - const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_thread; + const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_hvx; htp_unary_vtcm_layout_build(L, op, src0_data_row_size, dst_data_row_size, src1_data_row_size, - elem_size, col_tile, block, broadcast_weight, n_threads); + elem_size, col_tile, block, broadcast_weight, n_hvx); *out_col_tile = col_tile; - *out_vtcm_row_per_thread = (uint32_t) vtcm_row_per_thread; + *out_vtcm_row_per_hvx = (uint32_t) vtcm_row_per_hvx; } #endif /* HTP_UNARY_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.c b/ggml/src/ggml-hexagon/htp/upscale-ops.c index 62070cf5d..59ab7914a 100644 --- a/ggml/src/ggml-hexagon/htp/upscale-ops.c +++ b/ggml/src/ggml-hexagon/htp/upscale-ops.c @@ -6,7 +6,7 @@ #include "hex-common.h" #include "dma-queue.h" -#include "hex-profile.h" +#include "htp-trace.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -306,7 +306,7 @@ struct upscale_context { const void * lut_y; uint32_t chunk_start; uint32_t chunk_count; - uint32_t chunks_per_thread; + uint32_t chunks_per_hvx; }; static void compute_nearest_chunk( @@ -756,31 +756,31 @@ static inline void dma_push_src_chunk( } } -static void upscale_worker(unsigned int nth, unsigned int ith, void * data) { +static void upscale_worker(const htp_thread_t * th, unsigned int nth, unsigned int ith, void * data) { struct upscale_context * ctx = (struct upscale_context *) data; const struct htp_upscale_kernel_params * p = ctx->kparams; const struct htp_upscale_vtcm_layout * L = &ctx->layout; const struct htp_tensor * src0 = ctx->octx->src[0]; const struct htp_tensor * dst = ctx->octx->dst; - const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_thread; - const uint32_t last = MIN(first + ctx->chunks_per_thread, ctx->chunk_start + ctx->chunk_count); + const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_hvx; + const uint32_t last = MIN(first + ctx->chunks_per_hvx, ctx->chunk_start + ctx->chunk_count); if (first >= last) { return; } - struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith]; - dma_queue * dma_q = ctx->octx->ctx->dma[ith]; + struct htp_thread_trace * tr = th->trace; + dma_queue * dma_q = th->dma_q; const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size; const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base; - uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread; - uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread; - float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_thread / sizeof(float)); - float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_thread / sizeof(float)); + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_hvx; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_hvx; + float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_hvx / sizeof(float)); + float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_hvx / sizeof(float)); const void * lut_y = ctx->lut_y; float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; @@ -866,12 +866,12 @@ int op_upscale(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t n_threads = MIN(p->n_threads, chunk_count); - if (!htp_ops_context_set_n_threads(octx, n_threads)) { + const uint32_t n_hvx = MIN(p->n_hvx, chunk_count); + if (!htp_ops_context_set_n_hvx(octx, n_hvx)) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t chunks_per_thread = fastdiv(chunk_count + n_threads - 1, &octx->n_threads_div); + const uint32_t chunks_per_hvx = fastdiv(chunk_count + n_hvx - 1, &octx->n_hvx_div); size_t lut_x_size = 0; size_t elem_y_sz = sizeof(uint32_t); @@ -946,13 +946,13 @@ int op_upscale(struct htp_ops_context * octx) { .lut_y = lut_y, .chunk_start = chunk_start, .chunk_count = chunk_count, - .chunks_per_thread = chunks_per_thread, + .chunks_per_hvx = chunks_per_hvx, }; htp_upscale_vtcm_layout_build(&ctx.layout, src_x, src_y, dst_x, dst_y, - p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_threads); + p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_hvx); - work_queue_run(octx->ctx->work_queue, upscale_worker, &ctx, n_threads); + work_queue_run(octx->ctx->hvx_queue, upscale_worker, &ctx, n_hvx); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.h b/ggml/src/ggml-hexagon/htp/upscale-ops.h index aff7854b5..592bf7f00 100644 --- a/ggml/src/ggml-hexagon/htp/upscale-ops.h +++ b/ggml/src/ggml-hexagon/htp/upscale-ops.h @@ -43,7 +43,7 @@ struct htp_upscale_kernel_params { float invscale0; float invscale1; - uint32_t n_threads; + uint32_t n_hvx; uint32_t chunks_per_group; uint32_t dst_nrows_per_chunk; @@ -103,10 +103,10 @@ struct htp_upscale_vtcm_layout { size_t off_dst; size_t off_r; size_t off_gather; - size_t src_bytes_per_thread; - size_t dst_bytes_per_thread; - size_t r_bytes_per_thread; - size_t gather_bytes_per_thread; + size_t src_bytes_per_hvx; + size_t dst_bytes_per_hvx; + size_t r_bytes_per_hvx; + size_t gather_bytes_per_hvx; size_t src_spad_half_size; size_t dst_spad_half_size; uint32_t max_src_chunk_rows; @@ -122,7 +122,7 @@ static inline void htp_upscale_vtcm_layout_build( uint32_t dst_chunk_rows, float inv_sf1, float filter_radius1, - uint32_t n_threads + uint32_t n_hvx ) { uint32_t max_src_rows; if (dst_chunk_rows >= dst_y) { @@ -142,16 +142,16 @@ static inline void htp_upscale_vtcm_layout_build( L->dst_chunk_rows = dst_chunk_rows; L->src_spad_half_size = hex_round_up(src_chunk_bytes + 256, 128); L->dst_spad_half_size = hex_round_up(dst_chunk_bytes, 128); - L->src_bytes_per_thread = 2 * L->src_spad_half_size; - L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; - L->r_bytes_per_thread = hex_round_up(src_x * sizeof(float), 128); - L->gather_bytes_per_thread = 256; + L->src_bytes_per_hvx = 2 * L->src_spad_half_size; + L->dst_bytes_per_hvx = 2 * L->dst_spad_half_size; + L->r_bytes_per_hvx = hex_round_up(src_x * sizeof(float), 128); + L->gather_bytes_per_hvx = 256; size_t off = 0; - VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_thread * n_threads); - VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_hvx * n_hvx); + VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_hvx * n_hvx); L->total_bytes = off; } @@ -163,13 +163,13 @@ static inline bool htp_upscale_solve_layout( uint32_t dst_y, float inv_sf1, float filter_radius1, - uint32_t n_threads, + uint32_t n_hvx, size_t vtcm_budget, uint32_t * out_dst_chunk_rows ) { // 1. Try full plane uint32_t chunk_rows = dst_y; - htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads); + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); if (layout->total_bytes <= vtcm_budget) { *out_dst_chunk_rows = chunk_rows; return true; @@ -179,7 +179,7 @@ static inline bool htp_upscale_solve_layout( chunk_rows = 64; while (chunk_rows > 1) { if (chunk_rows < dst_y) { - htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads); + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); if (layout->total_bytes <= vtcm_budget) { *out_dst_chunk_rows = chunk_rows; return true; @@ -190,7 +190,7 @@ static inline bool htp_upscale_solve_layout( // 3. Fallback to 1 row chunk_rows = 1; - htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads); + htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_hvx); if (layout->total_bytes <= vtcm_budget) { *out_dst_chunk_rows = chunk_rows; return true; diff --git a/ggml/src/ggml-hexagon/htp/work-queue.c b/ggml/src/ggml-hexagon/htp/work-queue.c index 6c74445f7..5d274274a 100644 --- a/ggml/src/ggml-hexagon/htp/work-queue.c +++ b/ggml/src/ggml-hexagon/htp/work-queue.c @@ -16,8 +16,8 @@ // internal structure kept in thread-local storage per instance of work queue typedef struct { - work_queue_t queue; - unsigned int id; + htp_thread_t th; + work_queue_t queue; } worker_context_t; struct work_queue_task_s { @@ -35,15 +35,15 @@ struct work_queue_s { uint32_t idx_mask; uint32_t capacity; - qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers - worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts - void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers - unsigned int n_threads; // total threads (workers + main) - unsigned int n_workers; // number of active threads (just workers) + qurt_thread_t thread[HTP_MAX_NTHREADS]; // thread ID's of the workers + worker_context_t context[HTP_MAX_NTHREADS]; // worker contexts (0 = main, 1..n_workers = workers) + void * stack[HTP_MAX_NTHREADS]; // thread stack pointers + unsigned int n_threads; // total threads (workers + main) + unsigned int n_workers; // number of active threads (just workers) - atomic_uint active; // workers are polling/active - atomic_bool killed; // threads need to exit - bool external_mem; // memory owned externally + atomic_uint active; // workers are polling/active + atomic_bool killed; // threads need to exit + bool external_mem; // memory owned externally struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE))); }; @@ -51,8 +51,9 @@ struct work_queue_s { static void work_queue_thread(void * context) { worker_context_t * me = (worker_context_t *) context; work_queue_t q = me->queue; + unsigned int id = (unsigned int) (me - q->context); - FARF(HIGH, "work-queue: thread %u started", me->id); + FARF(HIGH, "work-queue: thread %u (tid %u) started", id, me->th.tid); unsigned int prev_seqn = 0; @@ -62,6 +63,7 @@ static void work_queue_thread(void * context) { if (atomic_load_explicit(&q->active, memory_order_relaxed)) { hex_pause(); } else { + qurt_hvx_unlock(); qurt_futex_wait(&q->active, 0); } continue; @@ -77,9 +79,8 @@ static void work_queue_thread(void * context) { struct work_queue_task_s * task = &q->queue[ir]; unsigned int n = task->n_threads; - unsigned int i = me->id; - if (i < n) { - task->func(n, i, task->data); + if (id < n) { + task->func(&me->th, n, id, task->data); atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); } else { @@ -92,10 +93,15 @@ static void work_queue_thread(void * context) { } } - FARF(HIGH, "work-queue: thread %u stopped", me->id); + FARF(HIGH, "work-queue: thread %u (tid %u) stopped", id, me->th.tid); } -bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { +bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n <= 1) { + func(&q->context[0].th, n, 0, data); + return true; + } + if (n > q->n_threads) { FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads); return false; @@ -121,7 +127,7 @@ bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, u atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); // main thread runs job #0 - func(n, 0, data); + func(&q->context[0].th, n, 0, data); atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); @@ -148,7 +154,7 @@ size_t work_queue_alignof(void) { return 4096; } -work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size, dma_queue_t * dma, struct htp_thread_trace * trace) { capacity = hex_ceil_pow2(capacity); uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; unsigned char * mem_blob = (unsigned char *) ptr; @@ -161,11 +167,14 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, q->external_mem = true; q->capacity = capacity; + for (unsigned int i = 0; i < n_threads; i++) { + htp_thread_init(&q->context[i].th, dma, trace); + q->context[i].queue = q; + } + for (unsigned int i = 0; i < n_workers; i++) { q->stack[i] = mem_blob; mem_blob += stack_size; q->thread[i] = 0; - q->context[i].id = i + 1; - q->context[i].queue = q; } atomic_init(&q->idx_read, 0); @@ -190,7 +199,7 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, qurt_thread_attr_set_stack_size(&attr, stack_size); char thread_name[32]; - snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i); + snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i + 1); qurt_thread_attr_set_name(&attr, thread_name); // set up priority - by default, match the creating thread's prio @@ -204,7 +213,7 @@ work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, qurt_thread_attr_set_priority(&attr, prio); - int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]); + int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i + 1]); if (err) { FARF(ERROR, "Could not launch worker threads!"); work_queue_free(q); diff --git a/ggml/src/ggml-hexagon/htp/work-queue.h b/ggml/src/ggml-hexagon/htp/work-queue.h index 09ca4b1f4..446b72c46 100644 --- a/ggml/src/ggml-hexagon/htp/work-queue.h +++ b/ggml/src/ggml-hexagon/htp/work-queue.h @@ -5,34 +5,21 @@ #include #include -typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *); +#include "htp-thread.h" + +typedef void (*work_queue_func_t)(const htp_thread_t * th, unsigned int n, unsigned int i, void *); struct work_queue_s; typedef struct work_queue_s * work_queue_t; -#define WORK_QUEUE_MAX_N_THREADS 10 - size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size); size_t work_queue_alignof(void); -work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size, dma_queue_t * dma, struct htp_thread_trace * trace); void work_queue_free(work_queue_t q); void work_queue_wakeup(work_queue_t q); void work_queue_suspend(work_queue_t q); -bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); - -static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { - if (n <= 1) { - func(n, 0, data); - return true; - } - return work_queue_run_async(q, func, data, n); -} - -// Legacy compatibility -typedef work_queue_func_t worker_callback_t; -#define worker_pool_run_func work_queue_run -#define worker_pool work_queue +bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); #endif // #ifndef HTP_WORK_QUEUE_H diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.py index 05e515167..1bf35b08b 100755 --- a/scripts/snapdragon/build.py +++ b/scripts/snapdragon/build.py @@ -106,6 +106,7 @@ def main(): parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container") parser.add_argument("--preset", help="Override the CMake preset to use") parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)") + parser.add_argument("--hex-debug", "--hexagon-debug", action="store_true", help="Enable Hexagon HTP debug output (sets GGML_HEXAGON_HTP_DEBUG=ON)") parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries") parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "v85", "73", "75", "79", "81", "85"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)") @@ -197,13 +198,15 @@ def main(): else: hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81", "htp-v85"] + hex_debug_arg = f"-DGGML_HEXAGON_HTP_DEBUG={'ON' if args.hex_debug else 'OFF'}" + if args.no_docker: # Native/local host build logger.info("Running native/local CMake build...") install_prefix = os.path.join(repo_root, install_dir, "llama.cpp") # Configure - configure_cmd = ["cmake", f"--preset={preset}", "-B", build_dir] + configure_cmd = ["cmake", f"--preset={preset}", hex_debug_arg, "-B", build_dir] logger.info(f"+ {' '.join(configure_cmd)}") res = subprocess.run(configure_cmd, cwd=repo_root) if res.returncode != 0: @@ -238,7 +241,7 @@ def main(): target_args = f"--target {' '.join(hex_targets)} " if args.hex_only else "" build_sh_cmd = ( - f"cmake --preset {preset} -B /workspace/{build_dir} && " + f"cmake --preset {preset} {hex_debug_arg} -B /workspace/{build_dir} && " f"cmake --build /workspace/{build_dir} {target_args}-j {jobs} && " f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}" ) @@ -261,6 +264,17 @@ def main(): logger.error("Docker-based build failed.") sys.exit(res.returncode) + install_lib_dir = os.path.join(repo_root, install_dir, "llama.cpp", "lib") + if os.path.isdir(install_lib_dir): + if args.hex_debug: + farf_src = os.path.join(repo_root, "scripts", "snapdragon", "adb", "llama-cli.farf") + farf_apps = ["llama-server", "llama-cli", "llama-completion", "test-backend-ops", "test-llama-archs"] + for app in farf_apps: + shutil.copyfile(farf_src, os.path.join(install_lib_dir, f"{app}.farf")) + else: + for f in glob.glob(os.path.join(install_lib_dir, "*.farf")): + os.remove(f) + logger.info("\nBuild and installation completed successfully!") # Push/deploy if requested @@ -286,6 +300,9 @@ def main(): else: hex_libs += glob.glob(os.path.join(lib_dir, "libggml-htp*")) + if args.hex_debug: + hex_libs += glob.glob(os.path.join(lib_dir, "*.farf")) + if not hex_libs: logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.") sys.exit(1) @@ -307,6 +324,10 @@ def main(): if res.returncode != 0: logger.error("ADB mkdir failed.") sys.exit(res.returncode) + if not args.hex_debug: + rm_farf_cmd = adb_cmd + ["shell", f"rm -f {shlex.quote(target_lib_dir)}/*.farf"] + logger.info(f"+ {' '.join(rm_farf_cmd)}") + subprocess.run(rm_farf_cmd) for lib in hex_libs: push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"] logger.info(f"+ {' '.join(push_cmd)}") @@ -383,6 +404,10 @@ def main(): if args.hex_only: logger.info(f"\nDeploying Hexagon libraries to Linux device {ssh_host} via SCP...") target_lib_dir = f"{target_dir}/lib" + if not args.hex_debug: + rm_farf_cmd = ["ssh", ssh_host, f"rm -f {target_lib_dir}/*.farf"] + logger.info(f"+ {' '.join(rm_farf_cmd)}") + subprocess.run(rm_farf_cmd) deploy_cmd = ["scp"] + hex_libs + [f"{ssh_host}:{target_lib_dir}/"] logger.info(f"+ {' '.join(deploy_cmd)}") res = subprocess.run(deploy_cmd) diff --git a/scripts/snapdragon/ggml-hexagon-align-macros.py b/scripts/snapdragon/ggml-hexagon-align-macros.py index b64db3e65..dc1475c52 100755 --- a/scripts/snapdragon/ggml-hexagon-align-macros.py +++ b/scripts/snapdragon/ggml-hexagon-align-macros.py @@ -6,6 +6,7 @@ Usage: align-macros.py [paths...] # Check and report misaligned macros align-macros.py --diff [paths...] # Show unified diff of fixes align-macros.py --fix [paths...] # Fix misaligned macros in-place + align-macros.py --fix --reflow false ... # Disable line reflowing in mostly-aligned macros align-macros.py --fix --mode majority ... # Align to the dominant column align-macros.py --fix --pad 2 ... # Align to (max_content_len + pad) @@ -127,9 +128,227 @@ def is_macro_aligned(macro: MacroDef) -> bool: return len(set(bs_cols)) == 1 and not has_trailing_ws +def str_to_bool(val: str) -> bool: + if isinstance(val, bool): + return val + v = val.strip().lower() + if v in ("true", "1", "yes", "y", "on"): + return True + if v in ("false", "0", "no", "n", "off"): + return False + raise argparse.ArgumentTypeError(f"Boolean value expected, got '{val}'") + + +def find_paren_comma_groups(s: str) -> List[Tuple[int, int, List[int]]]: + """Find parenthesized groups (open_idx, close_idx, comma_indices) ordered from outermost to innermost.""" + stack: List[Tuple[int, List[int]]] = [] + groups: List[Tuple[int, int, List[int]]] = [] + in_str: Optional[str] = None + escape = False + + for idx, ch in enumerate(s): + if escape: + escape = False + continue + if ch == "\\": + escape = True + continue + if in_str is not None: + if ch == in_str: + in_str = None + continue + if ch in ('"', "'"): + in_str = ch + continue + if ch == "(": + stack.append((idx, [])) + elif ch == ")" and stack: + open_idx, commas = stack.pop() + if commas: + groups.append((open_idx, idx, commas)) + elif ch == "," and stack: + stack[-1][1].append(idx) + + groups.sort(key=lambda g: g[0]) + return groups + + +def find_top_level_op_splits(s: str) -> Tuple[Optional[int], List[int]]: + """Find split positions after binary operators (+, -, &&, ||) at paren depth 0.""" + depth = 0 + in_str: Optional[str] = None + escape = False + assign_idx: Optional[int] = None + splits: List[int] = [] + n = len(s) + i = 0 + + while i < n: + ch = s[i] + if escape: + escape = False + i += 1 + continue + if ch == "\\": + escape = True + i += 1 + continue + if in_str is not None: + if ch == in_str: + in_str = None + i += 1 + continue + if ch in ('"', "'"): + in_str = ch + i += 1 + continue + if ch in "([{": + depth += 1 + elif ch in ")]}": + depth = max(0, depth - 1) + elif depth == 0: + if s.startswith("&&", i) or s.startswith("||", i): + splits.append(i + 2) + i += 2 + continue + if ch == "=" and (i == 0 or s[i - 1] not in "=!<>+-*/%&|^") and (i + 1 >= n or s[i + 1] != "="): + if assign_idx is None: + assign_idx = i + elif ch in "+-" and i > 0 and s[i - 1] == " " and i + 1 < n and s[i + 1] == " ": + splits.append(i + 1) + i += 1 + + return assign_idx, splits + + +def reflow_c_line(content: str, max_len: int) -> Optional[List[str]]: + """Try to cleanly reflow a C statement/signature line so each piece has len <= max_len.""" + if len(content) <= max_len: + return [content] + + stripped = content.lstrip() + if not stripped or stripped.startswith("#") or "//" in content or "/*" in content: + return None + + base_indent = content[:len(content) - len(stripped)] + + # 1. Try splitting comma-separated arguments/parameters inside (...) + for open_idx, _, commas in find_paren_comma_groups(content): + pts = [0] + [c + 1 for c in commas] + [len(content)] + segs: List[str] = [] + for idx in range(len(pts) - 1): + seg = content[pts[idx]:pts[idx + 1]].rstrip() + if idx > 0: + seg = seg.lstrip() + segs.append(seg) + + for cont_indent in [" " * (open_idx + 1), base_indent + " ", base_indent + " "]: + lines = [segs[0]] + for seg in segs[1:]: + cand = f"{lines[-1]} {seg}" + if len(cand) <= max_len: + lines[-1] = cand + else: + lines.append(f"{cont_indent}{seg}") + if len(lines) > 1 and all(len(line) <= max_len for line in lines): + return lines + + # 2. Try splitting after top-level binary operators (+, -, &&, ||) or assignment (=) + assign_idx, op_splits = find_top_level_op_splits(content) + if not op_splits and assign_idx is not None: + op_splits = [assign_idx + 1] + if op_splits: + pts = [0] + op_splits + [len(content)] + segs = [] + for idx in range(len(pts) - 1): + seg = content[pts[idx]:pts[idx + 1]].rstrip() + if idx > 0: + seg = seg.lstrip() + segs.append(seg) + + indents = [] + if assign_idx is not None: + rhs_start = assign_idx + 1 + while rhs_start < len(content) and content[rhs_start] == " ": + rhs_start += 1 + indents.append(" " * rhs_start) + indents.extend([base_indent + " ", base_indent + " "]) + + for cont_indent in indents: + lines = [segs[0]] + for seg in segs[1:]: + cand = f"{lines[-1]} {seg}" + if len(cand) <= max_len: + lines[-1] = cand + else: + lines.append(f"{cont_indent}{seg}") + if len(lines) > 1 and all(len(line) <= max_len for line in lines): + return lines + + return None + + +def reflow_macro_if_needed( + macro: MacroDef, + pad: int, + dom_ratio_threshold: float = 0.80, + fit_ratio_threshold: float = 0.90, +) -> Tuple[MacroDef, Optional[int]]: + """ + If a macro is already mostly aligned at a dominant column (>= dom_ratio_threshold) + and >= fit_ratio_threshold of continuation lines fit within that column, reflow any + exceeding lines so the macro can keep its dominant column instead of shifting all lines. + """ + cont_lines = [ml for ml in macro.lines if ml.bs_col is not None] + if not cont_lines: + return macro, None + + counts = Counter(ml.bs_col for ml in cont_lines if ml.bs_col is not None) + dom_col, dom_count = sorted(counts.items(), key=lambda x: (-x[1], -x[0]))[0] + max_allowed_len = dom_col - pad + if max_allowed_len <= 0: + return macro, None + + dom_ratio = dom_count / len(cont_lines) + fit_count = sum(1 for ml in cont_lines if len(ml.content) <= max_allowed_len) + fit_ratio = fit_count / len(cont_lines) + + effective_dom_threshold = 0.70 if fit_count == len(cont_lines) else dom_ratio_threshold + if dom_ratio < effective_dom_threshold or fit_ratio < fit_ratio_threshold: + return macro, None + + new_macro_lines: List[MacroLine] = [] + for ml in macro.lines: + if ml.bs_col is None or len(ml.content) <= max_allowed_len: + new_macro_lines.append(ml) + else: + reflowed = reflow_c_line(ml.content, max_allowed_len) + if not reflowed: + return macro, None + nl = "\r\n" if ml.raw.endswith("\r\n") else "\n" + for piece in reflowed: + new_macro_lines.append(MacroLine( + line_num=ml.line_num, + raw=f"{piece} \\{nl}", + content=piece, + bs_col=dom_col, + trailing_ws=False, + )) + + return MacroDef( + name=macro.name, + filepath=macro.filepath, + start_line=macro.start_line, + end_line=macro.end_line, + lines=new_macro_lines, + ), dom_col + + def compute_target_column(macro: MacroDef, mode: str, pad: int, target_col: Optional[int]) -> int: """Determine the column where backslashes should be aligned.""" - max_content_len = max(len(ml.content) for ml in macro.lines) + cont_lines = [ml for ml in macro.lines if ml.bs_col is not None] + lines_for_len = cont_lines if cont_lines else macro.lines + max_content_len = max(len(ml.content) for ml in lines_for_len) min_needed = max_content_len + pad if target_col is not None: @@ -181,6 +400,7 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt misaligned_count = 0 modified = False new_file_lines = list(file_lines) + line_offset = 0 for macro in macros: aligned = is_macro_aligned(macro) @@ -188,8 +408,9 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt if not aligned: misaligned_count += 1 - bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] - max_content = max(len(ml.content) for ml in macro.lines) + cont_lines = [ml for ml in macro.lines if ml.bs_col is not None] + bs_cols = [ml.bs_col for ml in cont_lines if ml.bs_col is not None] + max_content = max(len(ml.content) for ml in (cont_lines or macro.lines)) col_counts = Counter(bs_cols) if not args.quiet: @@ -200,16 +421,28 @@ def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Opt if trailing_ws_lines: logger.warning(f" Warning: Trailing whitespace after backslash on line(s): {trailing_ws_lines}") - target_col = compute_target_column(macro, args.mode, args.pad, args.target_col) + reflow_col: Optional[int] = None + target_macro = macro + if args.reflow and args.target_col is None: + target_macro, reflow_col = reflow_macro_if_needed(macro, args.pad) + + if reflow_col is not None: + target_col = reflow_col + if not args.quiet and len(target_macro.lines) != len(macro.lines): + logger.info(f" -> Reflowed {len(target_macro.lines) - len(macro.lines)} line(s) to preserve column {target_col}") + else: + target_col = compute_target_column(target_macro, args.mode, args.pad, args.target_col) + if not args.quiet: logger.info(f" -> Target alignment column: {target_col}") - realigned = realign_macro_lines(macro, target_col) + realigned = realign_macro_lines(target_macro, target_col) - start_idx = macro.start_line - 1 + start_idx = macro.start_line - 1 + line_offset end_idx = start_idx + len(macro.lines) if new_file_lines[start_idx:end_idx] != realigned: new_file_lines[start_idx:end_idx] = realigned + line_offset += len(realigned) - len(macro.lines) modified = True diff_text = None @@ -258,6 +491,8 @@ def main(): parser.add_argument("--check", action="store_true", help="Exit with code 1 if misaligned macros exist") parser.add_argument("--mode", choices=["min", "max", "majority"], default="min", help="Alignment mode: 'min' (max_len + pad), 'max' (max existing col), 'majority' (dominant col)") + parser.add_argument("--reflow", type=str_to_bool, nargs="?", const=True, default=True, + help="Reflow long lines in mostly-aligned macros instead of shifting the whole macro (default: true)") parser.add_argument("--pad", type=int, default=2, help="Spaces between longest line and backslash (default: 2)") parser.add_argument("--target-col", type=int, default=None, help="Force alignment to an exact column") parser.add_argument("--all", action="store_true", help="Realign all macros even if already aligned (default: only misaligned)") diff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.py index 7c95ec8cb..f6a949e98 100755 --- a/scripts/snapdragon/ggml-hexagon-profile.py +++ b/scripts/snapdragon/ggml-hexagon-profile.py @@ -466,10 +466,11 @@ def print_bubbles_timeline(op): logger.info(f" Avg Thread DMA IDLE : {avg_dma_idle:.1f}%") logger.info("-" * 100) + hmx_threads = {e['thread'] for e in events if e['event'].startswith('HMX')} logger.info(" Per-Thread Idle Analysis:") for t in active_threads: stats = thread_stats[t] - thread_name = f"Thread {t:<2} (HVX)" if t != 10 else "Thread 10 (HMX)" + thread_name = f"Thread {t:<2} (HMX)" if t in hmx_threads else f"Thread {t:<2} (HVX)" logger.info(f" {thread_name} -> Compute Idle: {stats['compute_idle_pct']:.1f}% | DMA Idle: {stats['dma_idle_pct']:.1f}%") all_bubbles = [] @@ -531,8 +532,9 @@ def print_ascii_summary(op_name, dims, types, usec, cycles, events): norm_evt = normalize_event_name(evt) thread_totals[t][norm_evt] += dur + hmx_threads = {e['thread'] for e in events if e['event'].startswith('HMX')} for t in sorted(thread_totals.keys()): - thread_name = f"Thread {t} (HVX)" if t != 10 else "Thread 10 (HMX)" + thread_name = f"Thread {t} (HMX)" if t in hmx_threads else f"Thread {t} (HVX)" sorted_evts = sorted(thread_totals[t].items(), key=lambda item: item[0]) evt_strs = [] diff --git a/scripts/snapdragon/ggml-hexagon-trace.py b/scripts/snapdragon/ggml-hexagon-trace.py index 5241dca15..308591b83 100755 --- a/scripts/snapdragon/ggml-hexagon-trace.py +++ b/scripts/snapdragon/ggml-hexagon-trace.py @@ -433,7 +433,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): time_mappers = {dev: DeviceTimeMapper(dev, filtered_ops) for dev in unique_devices} # Process events - completed_events = [] + completed_events: List[Dict[str, Any]] = [] if trace_events: trace_events = sorted(trace_events, key=lambda e: e['unwrapped_cycles']) @@ -516,6 +516,9 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): dur_ns = tm.dur_cycles_to_ns(e['start_cyc'], e['end_cyc'] - e['start_cyc']) e['dur_ns'] = max(dur_ns, 1) + # Identify HMX threads dynamically from HMX* events + hmx_threads = {(e['device'], e['thread']) for e in completed_events if e['event'].startswith('HMX')} + # Allocate slots (sub-tracks) to prevent overlaps on same virtual track active_slots = defaultdict(list) for e in completed_events: @@ -528,7 +531,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): norm_evt = normalize_event_name(evt, e['info']) if norm_evt == "DMA": track_key = (dev, t, "DMA") - elif t == 10: + elif (dev, t) in hmx_threads: track_key = (dev, t, "HMX") else: track_key = (dev, t, "HVX") @@ -552,23 +555,24 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): evt = e['event'] slot = e['slot'] dev = e['device'] + is_hmx = (dev, t) in hmx_threads norm_evt = normalize_event_name(evt, e['info']) if norm_evt == "DMA": track_evt = "DMA" evt_id = 1 - elif t == 10: + elif is_hmx: track_evt = "HMX" evt_id = 3 else: track_evt = "HVX" evt_id = 2 - t_sort = 1 if t == 10 else t + 2 + t_sort = 1 if is_hmx else t + 2 dev_idx = device_to_idx[dev] # Unique UUID for each sub-track - if t == 10: + if is_hmx: uuid = dev_idx * 10000000 + 20 # HMX thread track UUID else: uuid = int(dev_idx * 10000000 + t_sort * 1000000 + evt_id * 1000 + slot) @@ -601,9 +605,9 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): hmx_packet = make_trace_packet(0, track_descriptor=make_track_descriptor(hmx_track_uuid, parent_uuid=proc_uuid, thread=hmx_thread_desc)) write_trace_packet_to_file(f, hmx_packet) - # Define Thread Tracks (T0, T1, ..., T9) for this device + # Define Thread Tracks for this device dev_used_tracks = {uuid: val for uuid, val in used_tracks.items() if val[0] == dev} - unique_threads = sorted(list(set(t for (_, t, _, _) in dev_used_tracks.values() if t != 10))) + unique_threads = sorted(list(set(t for (_, t, _, _) in dev_used_tracks.values() if (dev, t) not in hmx_threads))) for t in unique_threads: thread_uuid = dev_idx * 10000000 + 10 + t thread_name = f"T{t}" @@ -623,7 +627,7 @@ def generate_perfetto_trace(filtered_ops, trace_events, output_path): for uuid in sorted(used_tracks.keys()): dev, t, evt, slot = used_tracks[uuid] dev_idx = device_to_idx[dev] - if t == 10: + if (dev, t) in hmx_threads: continue name = f"T{t} {evt}" rank = 0 if evt == "HVX" else 1