From a30fb4924dcfec81c1b2c20634bd47fa4403061a Mon Sep 17 00:00:00 2001 From: Michael Taylor <162068037+mctylr-gh@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:28:50 -0300 Subject: [PATCH] [SYCL]: fix memory errors in mul_mat, split buffer, host pool (#29889) --- ggml/src/ggml-sycl/common.cpp | 3 +- ggml/src/ggml-sycl/cpy.cpp | 2 + ggml/src/ggml-sycl/ggml-sycl.cpp | 118 ++++++++++++++++++++++--------- ggml/src/ggml-sycl/roll.cpp | 2 + 4 files changed, 90 insertions(+), 35 deletions(-) diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp index 894006949..f01d0ff95 100644 --- a/ggml/src/ggml-sycl/common.cpp +++ b/ggml/src/ggml-sycl/common.cpp @@ -115,7 +115,8 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t #else ze_device_mem_alloc_desc_t alloc_desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0}; #endif - ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, 64, ze_dev, &ptr); + // request SYCL_BUFFER_ALIGNMENT so tensor offsets aligned to it stay aligned in absolute terms + ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, SYCL_BUFFER_ALIGNMENT, ze_dev, &ptr); if (r == ZE_RESULT_SUCCESS && ptr) { ggml_sycl_memtrace_add(type, ptr, size); return ptr; diff --git a/ggml/src/ggml-sycl/cpy.cpp b/ggml/src/ggml-sycl/cpy.cpp index ef7413abd..8e1f1d50a 100644 --- a/ggml/src/ggml-sycl/cpy.cpp +++ b/ggml/src/ggml-sycl/cpy.cpp @@ -1255,6 +1255,8 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co scope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str("\tsrc0", src0)); const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); + // the cpy kernels index in 32-bit + GGML_ASSERT(ne <= INT32_MAX); GGML_TENSOR_BINARY_OP_LOCALS01; diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 1cfc84126..b380cbedf 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -899,6 +899,10 @@ static bool ggml_sycl_is_l0_discrete_gpu(int device) { static void memcpy_host_forward(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, const void *ptr_src, size_t size) { char *host_buf = (char *)malloc(size); + if (host_buf == nullptr) { + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory for host staging\n", __func__, size); + GGML_ABORT("fatal error"); + } q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); q_dst.memcpy((char *)ptr_dst, host_buf, size).wait(); free(host_buf); @@ -1177,12 +1181,13 @@ bool is_bmg_g31_arch(int device) { } static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - size_t max_alloc_size = dpct::get_current_device().get_max_mem_alloc_size(); + ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; + int device = ctx->device; + // use the buft device, the current device may differ on multi-GPU systems + size_t max_alloc_size = dpct::get_device(device).get_max_mem_alloc_size(); if (g_ggml_sycl_host_pinned_mem_2g) { return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024); } else { - ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; - int device = ctx->device; if(is_bmg_g31_arch(device)) { //Todo, it's workaround for BMG-G31, which has a known issue with large allocations. //The max alloc size is reduced to 60% of the reported max alloc size. @@ -1398,7 +1403,12 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; ctx->tensor_extras.push_back(extra); - ctx->streams.push_back(&(dpct::get_current_device().default_queue())); + // consumers index the queues by device id, so size the vector once with one queue per device + if (ctx->streams.empty()) { + for (int i = 0; i < ggml_sycl_info().device_count; ++i) { + ctx->streams.push_back(&(dpct::get_device(i).default_queue())); + } + } for (int i = 0; i < ggml_sycl_info().device_count; ++i) { int64_t row_low, row_high; @@ -2089,8 +2099,6 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { queue_ptr qptr; int device; - inline static int counter{ 0 }; - struct ggml_sycl_buffer { void * ptr = nullptr; size_t size = 0; @@ -2100,6 +2108,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { static constexpr int MAX_POOL_SIZE{ 64 }; std::vector buffer_pool = std::vector(MAX_POOL_SIZE); size_t pool_size = 0; + int counter = 0; explicit ggml_sycl_pool_host(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {} @@ -2113,41 +2122,55 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { b.size = 0; } } - counter = 0; } void * alloc(size_t size, size_t * actual_size) override { if (counter == MAX_POOL_SIZE) { - ggml_sycl_buffer b = buffer_pool[0]; - void * ptr = b.ptr; - *actual_size = b.size; - counter = 1; - return ptr; - } - ggml_sycl_buffer & b = buffer_pool[counter]; - - if (b.ptr == nullptr) { - void * ptr; - + // the pool wrapped around, no slot is reserved for this allocation + void * ptr = nullptr; SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr))); if (!ptr) { GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); return nullptr; } - pool_size += size; + pool_size += size; *actual_size = size; - counter = counter + 1; + counter = 0; return ptr; - } else { - ++counter; - b.size = size; - return b.ptr; } + ggml_sycl_buffer & b = buffer_pool[counter]; + ++counter; + + if (b.ptr != nullptr) { + if (b.size >= size) { + // hand out the cached buffer, free() puts it back into the pool + void * ptr = b.ptr; + *actual_size = b.size; + b.ptr = nullptr; + b.size = 0; + return ptr; + } + // the cached buffer is too small, replace it + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr))); + pool_size -= b.size; + b.ptr = nullptr; + b.size = 0; + } + + void * ptr; + SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr))); + if (!ptr) { + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); + return nullptr; + } + pool_size += size; + *actual_size = size; + return ptr; } void free(void * ptr, size_t size) override { // if the pool is not completed add the pointer to it in place of the first nullptr found. - // Otherwise do nothing, pointers will be freed once the pool is deallocated. + // Otherwise free the buffer, all slots are cached. for (int i = 0; i < MAX_POOL_SIZE; ++i) { ggml_sycl_buffer & b = buffer_pool[i]; if (b.ptr == nullptr) { @@ -2156,6 +2179,8 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { return; } } + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr))); + pool_size -= size; } }; @@ -3154,8 +3179,8 @@ inline void ggml_sycl_op_mul_mat_sycl( to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream); } const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16 - ? (const sycl::half *)src1->data + src1_padded_row_size - : src1_as_f16.get(); + ? (const sycl::half *)src1_ddf_i + : src1_as_f16.get(); ggml_sycl_pool_alloc src0_as_f16(ctx.pool()); if (src0->type != GGML_TYPE_F16) { @@ -3252,6 +3277,8 @@ inline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst) float * dst_dd = static_cast(dst->data); const int64_t ne = ggml_nelements(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ne <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ne, 1, main_stream); } @@ -3266,6 +3293,8 @@ inline void ggml_sycl_op_sum_rows(ggml_backend_sycl_context & ctx, ggml_tensor * const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); } @@ -3282,6 +3311,8 @@ inline void ggml_sycl_op_mean(ggml_backend_sycl_context & ctx, ggml_tensor * dst const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); @@ -3351,6 +3382,8 @@ inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * d const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); } @@ -3366,6 +3399,8 @@ inline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_ten const int64_t ne00 = dst->src[0]->ne[0]; const int64_t ne01 = dst->src[0]->ne[1]; const int nrows0 = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ne00 * nrows0 <= INT32_MAX); const int n_past = ((int32_t *) dst->op_params)[0]; @@ -3444,6 +3479,9 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * ds memcpy(&scale, (float *) dst->op_params + 0, sizeof(float)); memcpy(&bias, (float *) dst->op_params + 1, sizeof(float)); + // the kernel indexes in 32-bit + GGML_ASSERT(ggml_nelements(dst->src[0]) <= INT32_MAX); + scale_f32_sycl(src0_dd, dst_dd, scale, bias, ggml_nelements(dst->src[0]), main_stream); /* DPCT1010:87: SYCL uses exceptions to report errors and does not use the @@ -3524,7 +3562,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten const size_t q8_1_bs = QK8_1; ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; const bool src0_is_contiguous = ggml_is_contiguous(src0); const bool src1_is_contiguous = ggml_is_contiguous(src1); @@ -3606,7 +3643,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten queue_ptr stream = ctx.stream(i, 0); if (src0_is_contiguous) { - dev[i].src0_dd = (char *) src0->data; + // a split tensor keeps its rows in the per-device slices of the tensor extra + dev[i].src0_dd = split ? (char *) src0_extra->data_device[i] : (char *) src0->data; } else { dev[i].src0_dd = dev[i].src0_dd_alloc.alloc(ctx.pool(i), ggml_nbytes(src0)); } @@ -3702,7 +3740,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten src1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs) .wait())); } else { - const char * src1_ddf_i_source = (const char *) src1_extra->data_device[ctx.device] + + // src1 is not split, on the main device it lives at dev[ctx.device].src1_ddf + const char * src1_ddf_i_source = (const char *) dev[ctx.device].src1_ddf + (i0 * ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type); SYCL_CHECK( @@ -5269,6 +5308,7 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg (int) ne11, /*stride_col_y=*/src1_padded_cols / QK8_1, /*stride_col_dst=*/(int) glu->ne[0], stream); + GGML_UNUSED(gate); } // Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx. @@ -6860,7 +6900,9 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: case GGML_GLU_OP_SWIGLU_CLAMP: - return ggml_is_contiguous_1(op->src[0]); + // the fused GLU kernels only have f32/f16 instantiations + return ggml_is_contiguous_1(op->src[0]) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); default: return false; } @@ -7123,7 +7165,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_RMS_NORM_BACK: return ggml_is_contiguous(op->src[0]); case GGML_OP_SCALE: - return true; + // the scale kernel is f32 only + return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_CONT: return true; case GGML_OP_TRI: @@ -7156,12 +7199,19 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons #endif ) && op->src[0]->type == op->type; - case GGML_OP_CONV_3D: + case GGML_OP_CONV_3D: { + const int32_t * opts = (const int32_t *) op->op_params; + // zero kernel volume, channels or output makes the im2col GEMM degenerate (k == 0 or m == 0) + // and MKL BLAS rejects it; let those fall back to the CPU backend + const int64_t knl_n_total = op->src[0]->ne[0] * op->src[0]->ne[1] * op->src[0]->ne[2] * opts[9]; + const int64_t patch_total = op->ne[0] * op->ne[1] * op->ne[2] * opts[10]; return op->type == GGML_TYPE_F32 && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && - ggml_is_contiguous(op->src[1]); + ggml_is_contiguous(op->src[1]) && + knl_n_total > 0 && patch_total > 0; + } case GGML_OP_SUM: case GGML_OP_SUM_ROWS: case GGML_OP_MEAN: diff --git a/ggml/src/ggml-sycl/roll.cpp b/ggml/src/ggml-sycl/roll.cpp index 1e0518178..a95cee9a9 100644 --- a/ggml/src/ggml-sycl/roll.cpp +++ b/ggml/src/ggml-sycl/roll.cpp @@ -73,6 +73,8 @@ void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { const ggml_tensor *src = dst->src[0]; GGML_ASSERT(src && src->type == GGML_TYPE_F32); + // the kernel indexes in 32-bit + GGML_ASSERT(dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3] <= INT32_MAX); const int ne0 = (int) dst->ne[0]; const int ne1 = (int) dst->ne[1]; const int ne2 = (int) dst->ne[2];