diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 20775a921..995037f1b 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -5314,9 +5314,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_UNARY_OP_CEIL: case GGML_UNARY_OP_ROUND: case GGML_UNARY_OP_TRUNC: - // TODO: should become: - //return ggml_is_contiguous_rows(op->src[0]); - return ggml_is_contiguous(op->src[0]); + if (op->src[0]->type == GGML_TYPE_BF16 && ggml_get_unary_op(op) == GGML_UNARY_OP_XIELU) { + return false; + } + return op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16; default: return false; } diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu index c9a8cf6b0..436fdfffb 100644 --- a/ggml/src/ggml-cuda/unary.cu +++ b/ggml/src/ggml-cuda/unary.cu @@ -134,24 +134,67 @@ static void unary_cuda(const T * x, T * dst, const int k, cudaStream_t stream) { ggml_cuda_kernel_launch(unary_op_kernel, launch_params, x, dst, k); } +template +static __global__ void unary_op_kernel_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03) { + ggml_cuda_pdl_lc(); + const int i = blockDim.x*blockIdx.x + threadIdx.x; + + if (i >= k) { + return; + } + + int64_t rem = i; + const int64_t i0 = rem % ne00; rem /= ne00; + const int64_t i1 = rem % ne01; rem /= ne01; + const int64_t i2 = rem % ne02; + const int64_t i3 = rem / ne02; + const size_t src_byte_offset = i0 * nb00 + i1 * nb01 + i2 * nb02 + i3 * nb03; + const T * src_ptr = (const T *)((const char *)x + src_byte_offset); + + ggml_cuda_pdl_sync(); + dst[i] = ggml_cuda_cast(op(ggml_cuda_cast(*src_ptr))); +} + +template +static void unary_cuda_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03, cudaStream_t stream) { + const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE; + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_NEG_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(unary_op_kernel_strided, launch_params, x, dst, k, ne00,ne01,ne02,nb00,nb01,nb02,nb03); +} + template void ggml_cuda_op_unary(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const void * src0_d = src0->data; void * dst_d = dst->data; - cudaStream_t stream = ctx.stream(); - GGML_ASSERT(ggml_is_contiguous(src0)); + cudaStream_t stream = ctx.stream(); GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); GGML_ASSERT(src0->type == dst->type); - if (src0->type == GGML_TYPE_F16) { - unary_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream); - } else if (src0->type == GGML_TYPE_BF16) { - unary_cuda((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream); + if (ggml_is_contiguous(src0)) { + if (src0->type == GGML_TYPE_F16) { + unary_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream); + } else if (src0->type == GGML_TYPE_BF16) { + unary_cuda((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream); + } else { + unary_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream); + } } else { - unary_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream); + if (src0->type == GGML_TYPE_F16) { + unary_cuda_strided((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } else if (src0->type == GGML_TYPE_BF16) { + unary_cuda_strided((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } else { + unary_cuda_strided((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } } }