[SYCL]: fix memory errors in mul_mat, split buffer, host pool (#29889)
This commit is contained in:
1 parent
495c5805ce
commit
a30fb4924d
4 files changed
+90
-35
No files matched your search
@@ -115,7 +115,8 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t
|
||||
#else
|
||||
ze_device_mem_alloc_desc_t alloc_desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0};
|
||||
#endif
|
||||
ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, 64, ze_dev, &ptr);
|
||||
// request SYCL_BUFFER_ALIGNMENT so tensor offsets aligned to it stay aligned in absolute terms
|
||||
ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, SYCL_BUFFER_ALIGNMENT, ze_dev, &ptr);
|
||||
if (r == ZE_RESULT_SUCCESS && ptr) {
|
||||
ggml_sycl_memtrace_add(type, ptr, size);
|
||||
return ptr;
|
||||
|
||||
@@ -1255,6 +1255,8 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co
|
||||
scope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str("\tsrc0", src0));
|
||||
const int64_t ne = ggml_nelements(src0);
|
||||
GGML_ASSERT(ne == ggml_nelements(src1));
|
||||
// the cpy kernels index in 32-bit
|
||||
GGML_ASSERT(ne <= INT32_MAX);
|
||||
|
||||
GGML_TENSOR_BINARY_OP_LOCALS01;
|
||||
|
||||
|
||||
@@ -899,6 +899,10 @@ static bool ggml_sycl_is_l0_discrete_gpu(int device) {
|
||||
static void memcpy_host_forward(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst,
|
||||
const void *ptr_src, size_t size) {
|
||||
char *host_buf = (char *)malloc(size);
|
||||
if (host_buf == nullptr) {
|
||||
GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory for host staging\n", __func__, size);
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
q_src.memcpy(host_buf, (const char *)ptr_src, size).wait();
|
||||
q_dst.memcpy((char *)ptr_dst, host_buf, size).wait();
|
||||
free(host_buf);
|
||||
@@ -1177,12 +1181,13 @@ bool is_bmg_g31_arch(int device) {
|
||||
}
|
||||
|
||||
static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
|
||||
size_t max_alloc_size = dpct::get_current_device().get_max_mem_alloc_size();
|
||||
ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;
|
||||
int device = ctx->device;
|
||||
// use the buft device, the current device may differ on multi-GPU systems
|
||||
size_t max_alloc_size = dpct::get_device(device).get_max_mem_alloc_size();
|
||||
if (g_ggml_sycl_host_pinned_mem_2g) {
|
||||
return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024);
|
||||
} else {
|
||||
ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;
|
||||
int device = ctx->device;
|
||||
if(is_bmg_g31_arch(device)) {
|
||||
//Todo, it's workaround for BMG-G31, which has a known issue with large allocations.
|
||||
//The max alloc size is reduced to 60% of the reported max alloc size.
|
||||
@@ -1398,7 +1403,12 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer,
|
||||
ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
|
||||
|
||||
ctx->tensor_extras.push_back(extra);
|
||||
ctx->streams.push_back(&(dpct::get_current_device().default_queue()));
|
||||
// consumers index the queues by device id, so size the vector once with one queue per device
|
||||
if (ctx->streams.empty()) {
|
||||
for (int i = 0; i < ggml_sycl_info().device_count; ++i) {
|
||||
ctx->streams.push_back(&(dpct::get_device(i).default_queue()));
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < ggml_sycl_info().device_count; ++i) {
|
||||
int64_t row_low, row_high;
|
||||
@@ -2089,8 +2099,6 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
|
||||
queue_ptr qptr;
|
||||
int device;
|
||||
|
||||
inline static int counter{ 0 };
|
||||
|
||||
struct ggml_sycl_buffer {
|
||||
void * ptr = nullptr;
|
||||
size_t size = 0;
|
||||
@@ -2100,6 +2108,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
|
||||
static constexpr int MAX_POOL_SIZE{ 64 };
|
||||
std::vector<ggml_sycl_buffer> buffer_pool = std::vector<ggml_sycl_buffer>(MAX_POOL_SIZE);
|
||||
size_t pool_size = 0;
|
||||
int counter = 0;
|
||||
|
||||
explicit ggml_sycl_pool_host(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {}
|
||||
|
||||
@@ -2113,41 +2122,55 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
|
||||
b.size = 0;
|
||||
}
|
||||
}
|
||||
counter = 0;
|
||||
}
|
||||
|
||||
void * alloc(size_t size, size_t * actual_size) override {
|
||||
if (counter == MAX_POOL_SIZE) {
|
||||
ggml_sycl_buffer b = buffer_pool[0];
|
||||
void * ptr = b.ptr;
|
||||
*actual_size = b.size;
|
||||
counter = 1;
|
||||
return ptr;
|
||||
}
|
||||
ggml_sycl_buffer & b = buffer_pool[counter];
|
||||
|
||||
if (b.ptr == nullptr) {
|
||||
void * ptr;
|
||||
|
||||
// the pool wrapped around, no slot is reserved for this allocation
|
||||
void * ptr = nullptr;
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));
|
||||
if (!ptr) {
|
||||
GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
|
||||
return nullptr;
|
||||
}
|
||||
pool_size += size;
|
||||
pool_size += size;
|
||||
*actual_size = size;
|
||||
counter = counter + 1;
|
||||
counter = 0;
|
||||
return ptr;
|
||||
} else {
|
||||
++counter;
|
||||
b.size = size;
|
||||
return b.ptr;
|
||||
}
|
||||
ggml_sycl_buffer & b = buffer_pool[counter];
|
||||
++counter;
|
||||
|
||||
if (b.ptr != nullptr) {
|
||||
if (b.size >= size) {
|
||||
// hand out the cached buffer, free() puts it back into the pool
|
||||
void * ptr = b.ptr;
|
||||
*actual_size = b.size;
|
||||
b.ptr = nullptr;
|
||||
b.size = 0;
|
||||
return ptr;
|
||||
}
|
||||
// the cached buffer is too small, replace it
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr)));
|
||||
pool_size -= b.size;
|
||||
b.ptr = nullptr;
|
||||
b.size = 0;
|
||||
}
|
||||
|
||||
void * ptr;
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));
|
||||
if (!ptr) {
|
||||
GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
|
||||
return nullptr;
|
||||
}
|
||||
pool_size += size;
|
||||
*actual_size = size;
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void free(void * ptr, size_t size) override {
|
||||
// if the pool is not completed add the pointer to it in place of the first nullptr found.
|
||||
// Otherwise do nothing, pointers will be freed once the pool is deallocated.
|
||||
// Otherwise free the buffer, all slots are cached.
|
||||
for (int i = 0; i < MAX_POOL_SIZE; ++i) {
|
||||
ggml_sycl_buffer & b = buffer_pool[i];
|
||||
if (b.ptr == nullptr) {
|
||||
@@ -2156,6 +2179,8 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
|
||||
return;
|
||||
}
|
||||
}
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr)));
|
||||
pool_size -= size;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3154,8 +3179,8 @@ inline void ggml_sycl_op_mul_mat_sycl(
|
||||
to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream);
|
||||
}
|
||||
const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16
|
||||
? (const sycl::half *)src1->data + src1_padded_row_size
|
||||
: src1_as_f16.get();
|
||||
? (const sycl::half *)src1_ddf_i
|
||||
: src1_as_f16.get();
|
||||
|
||||
ggml_sycl_pool_alloc<sycl::half> src0_as_f16(ctx.pool());
|
||||
if (src0->type != GGML_TYPE_F16) {
|
||||
@@ -3252,6 +3277,8 @@ inline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst)
|
||||
float * dst_dd = static_cast<float *>(dst->data);
|
||||
|
||||
const int64_t ne = ggml_nelements(dst->src[0]);
|
||||
// the kernel indexes in 32-bit
|
||||
GGML_ASSERT(ne <= INT32_MAX);
|
||||
|
||||
sum_rows_f32_sycl(src0_dd, dst_dd, ne, 1, main_stream);
|
||||
}
|
||||
@@ -3266,6 +3293,8 @@ inline void ggml_sycl_op_sum_rows(ggml_backend_sycl_context & ctx, ggml_tensor *
|
||||
|
||||
const int64_t ncols = dst->src[0]->ne[0];
|
||||
const int64_t nrows = ggml_nrows(dst->src[0]);
|
||||
// the kernel indexes in 32-bit
|
||||
GGML_ASSERT(ncols * nrows <= INT32_MAX);
|
||||
|
||||
sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
|
||||
}
|
||||
@@ -3282,6 +3311,8 @@ inline void ggml_sycl_op_mean(ggml_backend_sycl_context & ctx, ggml_tensor * dst
|
||||
|
||||
const int64_t ncols = dst->src[0]->ne[0];
|
||||
const int64_t nrows = ggml_nrows(dst->src[0]);
|
||||
// the kernel indexes in 32-bit
|
||||
GGML_ASSERT(ncols * nrows <= INT32_MAX);
|
||||
|
||||
sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
|
||||
|
||||
@@ -3351,6 +3382,8 @@ inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * d
|
||||
|
||||
const int64_t ncols = dst->src[0]->ne[0];
|
||||
const int64_t nrows = ggml_nrows(dst->src[0]);
|
||||
// the kernel indexes in 32-bit
|
||||
GGML_ASSERT(ncols * nrows <= INT32_MAX);
|
||||
|
||||
argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
|
||||
}
|
||||
@@ -3366,6 +3399,8 @@ inline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_ten
|
||||
const int64_t ne00 = dst->src[0]->ne[0];
|
||||
const int64_t ne01 = dst->src[0]->ne[1];
|
||||
const int nrows0 = ggml_nrows(dst->src[0]);
|
||||
// the kernel indexes in 32-bit
|
||||
GGML_ASSERT(ne00 * nrows0 <= INT32_MAX);
|
||||
|
||||
const int n_past = ((int32_t *) dst->op_params)[0];
|
||||
|
||||
@@ -3444,6 +3479,9 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * ds
|
||||
memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
|
||||
memcpy(&bias, (float *) dst->op_params + 1, sizeof(float));
|
||||
|
||||
// the kernel indexes in 32-bit
|
||||
GGML_ASSERT(ggml_nelements(dst->src[0]) <= INT32_MAX);
|
||||
|
||||
scale_f32_sycl(src0_dd, dst_dd, scale, bias, ggml_nelements(dst->src[0]), main_stream);
|
||||
/*
|
||||
DPCT1010:87: SYCL uses exceptions to report errors and does not use the
|
||||
@@ -3524,7 +3562,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
|
||||
const size_t q8_1_bs = QK8_1;
|
||||
|
||||
ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;
|
||||
ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;
|
||||
|
||||
const bool src0_is_contiguous = ggml_is_contiguous(src0);
|
||||
const bool src1_is_contiguous = ggml_is_contiguous(src1);
|
||||
@@ -3606,7 +3643,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
|
||||
queue_ptr stream = ctx.stream(i, 0);
|
||||
|
||||
if (src0_is_contiguous) {
|
||||
dev[i].src0_dd = (char *) src0->data;
|
||||
// a split tensor keeps its rows in the per-device slices of the tensor extra
|
||||
dev[i].src0_dd = split ? (char *) src0_extra->data_device[i] : (char *) src0->data;
|
||||
} else {
|
||||
dev[i].src0_dd = dev[i].src0_dd_alloc.alloc(ctx.pool(i), ggml_nbytes(src0));
|
||||
}
|
||||
@@ -3702,7 +3740,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
|
||||
src1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs)
|
||||
.wait()));
|
||||
} else {
|
||||
const char * src1_ddf_i_source = (const char *) src1_extra->data_device[ctx.device] +
|
||||
// src1 is not split, on the main device it lives at dev[ctx.device].src1_ddf
|
||||
const char * src1_ddf_i_source = (const char *) dev[ctx.device].src1_ddf +
|
||||
(i0 * ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type);
|
||||
|
||||
SYCL_CHECK(
|
||||
@@ -5269,6 +5308,7 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
|
||||
(int) ne11,
|
||||
/*stride_col_y=*/src1_padded_cols / QK8_1,
|
||||
/*stride_col_dst=*/(int) glu->ne[0], stream);
|
||||
GGML_UNUSED(gate);
|
||||
}
|
||||
|
||||
// Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx.
|
||||
@@ -6860,7 +6900,9 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
case GGML_GLU_OP_GEGLU_ERF:
|
||||
case GGML_GLU_OP_GEGLU_QUICK:
|
||||
case GGML_GLU_OP_SWIGLU_CLAMP:
|
||||
return ggml_is_contiguous_1(op->src[0]);
|
||||
// the fused GLU kernels only have f32/f16 instantiations
|
||||
return ggml_is_contiguous_1(op->src[0]) &&
|
||||
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
@@ -7123,7 +7165,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
case GGML_OP_RMS_NORM_BACK:
|
||||
return ggml_is_contiguous(op->src[0]);
|
||||
case GGML_OP_SCALE:
|
||||
return true;
|
||||
// the scale kernel is f32 only
|
||||
return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_CONT:
|
||||
return true;
|
||||
case GGML_OP_TRI:
|
||||
@@ -7156,12 +7199,19 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
#endif
|
||||
) &&
|
||||
op->src[0]->type == op->type;
|
||||
case GGML_OP_CONV_3D:
|
||||
case GGML_OP_CONV_3D: {
|
||||
const int32_t * opts = (const int32_t *) op->op_params;
|
||||
// zero kernel volume, channels or output makes the im2col GEMM degenerate (k == 0 or m == 0)
|
||||
// and MKL BLAS rejects it; let those fall back to the CPU backend
|
||||
const int64_t knl_n_total = op->src[0]->ne[0] * op->src[0]->ne[1] * op->src[0]->ne[2] * opts[9];
|
||||
const int64_t patch_total = op->ne[0] * op->ne[1] * op->ne[2] * opts[10];
|
||||
return op->type == GGML_TYPE_F32 &&
|
||||
(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
|
||||
op->src[1]->type == GGML_TYPE_F32 &&
|
||||
ggml_is_contiguous(op->src[0]) &&
|
||||
ggml_is_contiguous(op->src[1]);
|
||||
ggml_is_contiguous(op->src[1]) &&
|
||||
knl_n_total > 0 && patch_total > 0;
|
||||
}
|
||||
case GGML_OP_SUM:
|
||||
case GGML_OP_SUM_ROWS:
|
||||
case GGML_OP_MEAN:
|
||||
|
||||
@@ -73,6 +73,8 @@ void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
|
||||
const ggml_tensor *src = dst->src[0];
|
||||
GGML_ASSERT(src && src->type == GGML_TYPE_F32);
|
||||
|
||||
// the kernel indexes in 32-bit
|
||||
GGML_ASSERT(dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3] <= INT32_MAX);
|
||||
const int ne0 = (int) dst->ne[0];
|
||||
const int ne1 = (int) dst->ne[1];
|
||||
const int ne2 = (int) dst->ne[2];
|
||||
|
||||
Reference in new issue
Block a user