hexagon: fix IM2COL patch-embed DMA ring overflow (#30189)

* hexagon: fix IM2COL patch-embed DMA ring overflow

The exact-tiling (stride == kernel, no pad/dilation) IM2COL DMA kernel
issues IC*KH DDR->VTCM descriptors per output row without checking the
return value of dma_queue_push(), and then pops IC*KH times. The per-thread
DMA ring holds 256 entries and a push into a full ring returns false
and drops the transfer, so for IC*KH > 255 the remaining rows of the
VTCM staging buffer were never written and stale data (often NaN/inf)
leaked into the output.

Solution is to retire the oldest descriptor when the ring is full, just as the blocked
kernel in the same file already does, and wait with dma_queue_flush().

For testing, added exact-tiling test cases with IC*KH > 256 (2D 1x1, 2D 2x2 patch
embed and 1D, F16 and F32 dst), which fail on HTP without this fix.

* Apply suggestion from @max-krasnyansky

---------

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
Mark TodorovichandMax Krasnyansky authored and GitHub committed 2026-10-09 08:04:09 +02:00
1 parent 3d65c90d04
commit d4d82d67f4
2 files changed
+15 -4

No files matched your search

+7 -4
View File
@@ -351,12 +351,15 @@ IM2COL_BLOCKED_DMA_BODY(im2col_blocked_dma_f32_thread, float, hvx_copy_f32_uu,
const dma_addr_t vsrc = ok \
? (src_data + (size_t) ((in * IC + iic) * IH + iih) * IW * sizeof(float)) \
: src_data; \
dma_queue_push(dma_q, dma_make_data(vdst, vsrc), \
IW * sizeof(float), IW * sizeof(float), IW * sizeof(float), ok ? 1 : 0); \
/* IC*KH descriptors per row can exceed the ring capacity: retire the oldest when full */ \
while (!dma_queue_push(dma_q, dma_make_data(vdst, vsrc), \
IW * sizeof(float), IW * sizeof(float), IW * sizeof(float), \
ok ? 1 : 0)) { \
dma_queue_pop(dma_q); \
} \
} \
} \
for (uint32_t i = 0; i < IC * KH; i++) \
dma_queue_pop(dma_q); \
dma_queue_flush(dma_q); \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); \
for (uint32_t iow = 0; iow < OW; iow++) { \
DST_CTYPE * dst_patch = dstb + (uint64_t) iow * patch_stride; \
+8
View File
@@ -9789,6 +9789,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 1536, 729}, {2, 2, 1536, 4096}, 1, 1, 0, 0, 1, 1, true));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 1, 2}, {32, 33, 1, 2}, 1, 1, 1, 1, 1, 1, true));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 2, 1}, {33, 34, 2, 1}, 1, 1, 1, 1, 1, 1, true));
// non-overlapping (stride == kernel, no pad/dilation) with IC*KH well above 256, e.g. SD1.5 1x1 convs
for (ggml_type dst_type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {64, 64, 320, 1}, {1, 1, 320, 1}, 1, 1, 0, 0, 1, 1, true));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {16, 16, 1280, 2}, {1, 1, 1280, 1}, 1, 1, 0, 0, 1, 1, true));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {8, 8, 2560, 1}, {1, 1, 2560, 1}, 1, 1, 0, 0, 1, 1, true));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {32, 32, 192, 1}, {2, 2, 192, 1}, 2, 2, 0, 0, 1, 1, true));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {64, 512, 1, 1}, {2, 512, 1, 1}, 2, 0, 0, 0, 1, 0, false));
}
// im2col 3D
test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));