cuda : support arbitrary striding for unary ops on f16, f32, and bf16 (#29781)
* cuda : support arbitrary striding for unary ops on f16, f32, and bf16 * Remove added newline --------- Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
This commit is contained in:
1 parent
46baf1f1fe
commit
08246a28f6
2 files changed
+54
-10
No files matched your search
@@ -5314,9 +5314,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_UNARY_OP_CEIL:
|
||||
case GGML_UNARY_OP_ROUND:
|
||||
case GGML_UNARY_OP_TRUNC:
|
||||
// TODO: should become:
|
||||
//return ggml_is_contiguous_rows(op->src[0]);
|
||||
return ggml_is_contiguous(op->src[0]);
|
||||
if (op->src[0]->type == GGML_TYPE_BF16 && ggml_get_unary_op(op) == GGML_UNARY_OP_XIELU) {
|
||||
return false;
|
||||
}
|
||||
return op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -134,24 +134,67 @@ static void unary_cuda(const T * x, T * dst, const int k, cudaStream_t stream) {
|
||||
ggml_cuda_kernel_launch(unary_op_kernel<op, T>, launch_params, x, dst, k);
|
||||
}
|
||||
|
||||
template <float (*op)(float), typename T>
|
||||
static __global__ void unary_op_kernel_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03) {
|
||||
ggml_cuda_pdl_lc();
|
||||
const int i = blockDim.x*blockIdx.x + threadIdx.x;
|
||||
|
||||
if (i >= k) {
|
||||
return;
|
||||
}
|
||||
|
||||
int64_t rem = i;
|
||||
const int64_t i0 = rem % ne00; rem /= ne00;
|
||||
const int64_t i1 = rem % ne01; rem /= ne01;
|
||||
const int64_t i2 = rem % ne02;
|
||||
const int64_t i3 = rem / ne02;
|
||||
const size_t src_byte_offset = i0 * nb00 + i1 * nb01 + i2 * nb02 + i3 * nb03;
|
||||
const T * src_ptr = (const T *)((const char *)x + src_byte_offset);
|
||||
|
||||
ggml_cuda_pdl_sync();
|
||||
dst[i] = ggml_cuda_cast<T>(op(ggml_cuda_cast<float>(*src_ptr)));
|
||||
}
|
||||
|
||||
template <float (*op)(float), typename T>
|
||||
static void unary_cuda_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03, cudaStream_t stream) {
|
||||
const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE;
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_NEG_BLOCK_SIZE, 0, stream);
|
||||
ggml_cuda_kernel_launch(unary_op_kernel_strided<op, T>, launch_params, x, dst, k, ne00,ne01,ne02,nb00,nb01,nb02,nb03);
|
||||
}
|
||||
|
||||
template <float (*op)(float)>
|
||||
void ggml_cuda_op_unary(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const void * src0_d = src0->data;
|
||||
void * dst_d = dst->data;
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
GGML_ASSERT(ggml_is_contiguous(src0));
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
|
||||
GGML_ASSERT(src0->type == dst->type);
|
||||
|
||||
if (src0->type == GGML_TYPE_F16) {
|
||||
unary_cuda<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16) {
|
||||
unary_cuda<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream);
|
||||
if (ggml_is_contiguous(src0)) {
|
||||
if (src0->type == GGML_TYPE_F16) {
|
||||
unary_cuda<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16) {
|
||||
unary_cuda<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream);
|
||||
} else {
|
||||
unary_cuda<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream);
|
||||
}
|
||||
} else {
|
||||
unary_cuda<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream);
|
||||
if (src0->type == GGML_TYPE_F16) {
|
||||
unary_cuda_strided<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0),
|
||||
src0->ne[0], src0->ne[1], src0->ne[2],
|
||||
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16) {
|
||||
unary_cuda_strided<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0),
|
||||
src0->ne[0], src0->ne[1], src0->ne[2],
|
||||
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream);
|
||||
} else {
|
||||
unary_cuda_strided<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0),
|
||||
src0->ne[0], src0->ne[1], src0->ne[2],
|
||||
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user