From 43bd7a2a42210726fa77ae1c97a33c9151d7fbb3 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Wed, 18 Nov 2026 11:26:03 +0100 Subject: [PATCH] vulkan: add bfloat16 support to various ops (#32202) * vulkan: expand bf16 support for unary and binary ops and rms_norm * cpu: add bf16 support for scale, clamp, leaky_relu, swiglu_oai, swiglu_clamp, tri and diag * fix sycl fill supports_op condition * restore CUDA op order --- ggml/src/ggml-cpu/ops.cpp | 136 +++--- ggml/src/ggml-cpu/vec.h | 6 - ggml/src/ggml-cuda/ggml-cuda.cu | 6 +- ggml/src/ggml-metal/ggml-metal-device.m | 2 +- ggml/src/ggml-opencl/ggml-opencl.cpp | 2 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 2 + ggml/src/ggml-vulkan/ggml-vulkan-types.h | 84 ++-- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 402 ++++++++---------- ggml/src/ggml-vulkan/vulkan-shaders/add.comp | 4 +- ggml/src/ggml-vulkan/vulkan-shaders/diag.comp | 3 +- ggml/src/ggml-vulkan/vulkan-shaders/div.comp | 2 +- .../ggml-vulkan/vulkan-shaders/glu_head.glsl | 1 + .../ggml-vulkan/vulkan-shaders/glu_main.glsl | 6 +- ggml/src/ggml-vulkan/vulkan-shaders/log.comp | 4 +- ggml/src/ggml-vulkan/vulkan-shaders/mul.comp | 2 +- .../ggml-vulkan/vulkan-shaders/rms_norm.comp | 18 +- .../src/ggml-vulkan/vulkan-shaders/scale.comp | 2 +- ggml/src/ggml-vulkan/vulkan-shaders/sub.comp | 2 +- ggml/src/ggml-vulkan/vulkan-shaders/tri.comp | 3 +- .../src/ggml-vulkan/vulkan-shaders/types.glsl | 21 + .../src/ggml-vulkan/vulkan-shaders/unary.comp | 6 +- .../vulkan-shaders/vulkan-shaders-gen.cpp | 127 ++---- ggml/src/ggml.c | 2 +- tests/test-backend-ops.cpp | 28 +- 24 files changed, 410 insertions(+), 461 deletions(-) diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 7c2542741..72efbc6e9 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -2240,7 +2240,8 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor * // ggml_compute_tri -static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggml_tensor * dst) { +template +static void ggml_compute_forward_tri_t(const ggml_compute_params * params, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0); @@ -2266,11 +2267,11 @@ static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggm const int64_t i02 = (ir - i03*ne02*ne01)/ne01; const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01); - const float * src_ptr = (const float *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01); - float * dst_ptr = ( float *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1); + const T * src_ptr = (const T *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01); + T * dst_ptr = ( T *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1); for (int i0 = 0; i0 < ne0; ++i0) { - dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : 0.0f; + dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : T(0); } } } @@ -2281,7 +2282,12 @@ void ggml_compute_forward_tri(const ggml_compute_params * params, ggml_tensor * switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_tri_f32(params, dst); + ggml_compute_forward_tri_t(params, dst); + } break; + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + { + ggml_compute_forward_tri_t(params, dst); } break; default: { @@ -2708,7 +2714,8 @@ static void ggml_compute_forward_leaky_relu_f32( } } -static void ggml_compute_forward_leaky_relu_f16( +template +static void ggml_compute_forward_leaky_relu_half( const ggml_compute_params * params, ggml_tensor * dst) { @@ -2728,13 +2735,16 @@ static void ggml_compute_forward_leaky_relu_f16( float negative_slope; memcpy(&negative_slope, dst->op_params, sizeof(float)); - assert(dst->nb[0] == sizeof(ggml_fp16_t)); - assert(src0->nb[0] == sizeof(ggml_fp16_t)); + assert(dst->nb[0] == sizeof(T)); + assert(src0->nb[0] == sizeof(T)); for (int i = 0; i < n; i++) { - ggml_vec_leaky_relu_f16(nc, - (ggml_fp16_t *) ((char *) dst->data + i*( dst->nb[1])), - (ggml_fp16_t *) ((char *) src0->data + i*(src0->nb[1])), negative_slope); + T * y = (T *) ((char *) dst->data + i*( dst->nb[1])); + const T * x = (const T *) ((const char *) src0->data + i*(src0->nb[1])); + for (int k = 0; k < nc; k++) { + const float v = type_conversion_table::to_f32(x[k]); + y[k] = type_conversion_table::from_f32(((v > 0.f) ? v : 0.f) + negative_slope * ((v < 0.0f) ? v : 0.f)); + } } } @@ -2751,7 +2761,11 @@ void ggml_compute_forward_leaky_relu( } break; case GGML_TYPE_F16: { - ggml_compute_forward_leaky_relu_f16(params, dst); + ggml_compute_forward_leaky_relu_half(params, dst); + } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_leaky_relu_half(params, dst); } break; default: { @@ -3452,7 +3466,8 @@ static void ggml_compute_forward_swiglu_oai_f32( } } -static void ggml_compute_forward_swiglu_oai_f16( +template +static void ggml_compute_forward_swiglu_oai_half( const ggml_compute_params * params, ggml_tensor * dst) { @@ -3492,9 +3507,9 @@ static void ggml_compute_forward_swiglu_oai_f16( const int ir1 = MIN(ir0 + dr, nr); for (int i1 = ir0; i1 < ir1; i1++) { - ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o); - ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o); - ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])); + T * src0_p = (T *) (src0_d + i1*src0_o); + T * src1_p = (T *) (src1_d + i1*src1_o); + T * dst_p = (T *) ((char *) dst->data + i1*(dst->nb[1])); if (!src1) { src0_p += swapped ? nc : 0; @@ -3502,15 +3517,15 @@ static void ggml_compute_forward_swiglu_oai_f16( } for (int k = 0; k < nc; k++) { - const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit); - const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit); + const float x = std::min(type_conversion_table::to_f32(src0_p[k]), limit); + const float y = std::clamp(type_conversion_table::to_f32(src1_p[k]), -limit, limit); const float out_glu = x / (1.f + expf(alpha * (-x))); - dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f)); + dst_p[k] = type_conversion_table::from_f32(out_glu * (y + 1.f)); } #ifndef NDEBUG for (int k = 0; k < nc; k++) { - const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]); + const float x = type_conversion_table::to_f32(dst_p[k]); GGML_UNUSED(x); assert(!isnan(x)); assert(!isinf(x)); @@ -3532,7 +3547,11 @@ static void ggml_compute_forward_swiglu_oai( } break; case GGML_TYPE_F16: { - ggml_compute_forward_swiglu_oai_f16(params, dst); + ggml_compute_forward_swiglu_oai_half(params, dst); + } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_swiglu_oai_half(params, dst); } break; default: { @@ -3602,7 +3621,8 @@ static void ggml_compute_forward_swiglu_clamp_f32(const ggml_compute_params * pa } } -static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * params, ggml_tensor * dst) { +template +static void ggml_compute_forward_swiglu_clamp_half(const ggml_compute_params * params, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; char * src0_d = (char *) src0->data; @@ -3635,9 +3655,9 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa const int ir1 = MIN(ir0 + dr, nr); for (int i1 = ir0; i1 < ir1; i1++) { - ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1 * src0_o); - ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1 * src1_o); - ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1 * (dst->nb[1])); + T * src0_p = (T *) (src0_d + i1 * src0_o); + T * src1_p = (T *) (src1_d + i1 * src1_o); + T * dst_p = (T *) ((char *) dst->data + i1 * (dst->nb[1])); if (!src1) { src0_p += swapped ? nc : 0; @@ -3645,14 +3665,14 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa } for (int k = 0; k < nc; k++) { - const float gate = std::min(GGML_FP16_TO_FP32(src0_p[k]), limit); - const float up = std::clamp(GGML_FP16_TO_FP32(src1_p[k]), -limit, limit); - dst_p[k] = GGML_FP32_TO_FP16(gate / (1.f + expf(-gate)) * up); + const float gate = std::min(type_conversion_table::to_f32(src0_p[k]), limit); + const float up = std::clamp(type_conversion_table::to_f32(src1_p[k]), -limit, limit); + dst_p[k] = type_conversion_table::from_f32(gate / (1.f + expf(-gate)) * up); } #ifndef NDEBUG for (int k = 0; k < nc; k++) { - const float x = GGML_FP16_TO_FP32(dst_p[k]); + const float x = type_conversion_table::to_f32(dst_p[k]); GGML_UNUSED(x); assert(!isnan(x)); assert(!isinf(x)); @@ -3667,7 +3687,10 @@ static void ggml_compute_forward_swiglu_clamp(const ggml_compute_params * params ggml_compute_forward_swiglu_clamp_f32(params, dst); break; case GGML_TYPE_F16: - ggml_compute_forward_swiglu_clamp_f16(params, dst); + ggml_compute_forward_swiglu_clamp_half(params, dst); + break; + case GGML_TYPE_BF16: + ggml_compute_forward_swiglu_clamp_half(params, dst); break; default: GGML_ABORT("fatal error"); @@ -4953,7 +4976,8 @@ static void ggml_compute_forward_scale_f32( } } -static void ggml_compute_forward_scale_bf16( +template +static void ggml_compute_forward_scale_half( const ggml_compute_params * params, ggml_tensor * dst) { @@ -4989,10 +5013,10 @@ static void ggml_compute_forward_scale_bf16( const int64_t i2 = (i / ne01) % ne02; const int64_t i3 = i / (ne01*ne02); - const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); - ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); + const T * x = (const T *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); + T * y = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); for (int k = 0; k < nc; k++) { - y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b); + y[k] = type_conversion_table::from_f32(s*type_conversion_table::to_f32(x[k]) + b); } } } @@ -5008,9 +5032,13 @@ void ggml_compute_forward_scale( { ggml_compute_forward_scale_f32(params, dst); } break; + case GGML_TYPE_F16: + { + ggml_compute_forward_scale_half(params, dst); + } break; case GGML_TYPE_BF16: { - ggml_compute_forward_scale_bf16(params, dst); + ggml_compute_forward_scale_half(params, dst); } break; default: { @@ -5694,7 +5722,8 @@ void ggml_compute_forward_get_rows_back( // ggml_compute_forward_diag -static void ggml_compute_forward_diag_f32( +template +static void ggml_compute_forward_diag_t( const ggml_compute_params * params, ggml_tensor * dst) { @@ -5714,14 +5743,14 @@ static void ggml_compute_forward_diag_f32( GGML_ASSERT(ne02 == ne2); GGML_ASSERT(ne03 == ne3); - GGML_ASSERT(nb00 == sizeof(float)); - GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(nb00 == sizeof(T)); + GGML_ASSERT(nb0 == sizeof(T)); for (int i3 = 0; i3 < ne3; i3++) { for (int i2 = 0; i2 < ne2; i2++) { for (int i1 = 0; i1 < ne1; i1++) { - float * d = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1); - float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02); + T * d = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1); + T * s = (T *)((char *) src0->data + i3*nb03 + i2*nb02); for (int i0 = 0; i0 < i1; i0++) { d[i0] = 0; } @@ -5743,7 +5772,12 @@ void ggml_compute_forward_diag( switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_diag_f32(params, dst); + ggml_compute_forward_diag_t(params, dst); + } break; + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + { + ggml_compute_forward_diag_t(params, dst); } break; default: { @@ -6026,7 +6060,8 @@ static void ggml_compute_forward_clamp_f32( } } -static void ggml_compute_forward_clamp_f16( +template +static void ggml_compute_forward_clamp_half( const ggml_compute_params * params, ggml_tensor * dst) { @@ -6045,20 +6080,20 @@ static void ggml_compute_forward_clamp_f16( GGML_TENSOR_UNARY_OP_LOCALS - GGML_ASSERT( nb0 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); + GGML_ASSERT( nb0 == sizeof(T)); + GGML_ASSERT(nb00 == sizeof(T)); for (int j = ith; j < n; j += nth) { const int64_t i1 = j % ne01; const int64_t i2 = (j / ne01) % ne02; const int64_t i3 = j / (ne01*ne02); - ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); - ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); + T * dst_ptr = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); + T * src0_ptr = (T *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); for (int i = 0; i < nc; i++) { - float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]); - dst_ptr[i] = GGML_CPU_FP32_TO_FP16(MAX(MIN(v, max), min)); + float v = type_conversion_table::to_f32(src0_ptr[i]); + dst_ptr[i] = type_conversion_table::from_f32(MAX(MIN(v, max), min)); } } } @@ -6076,9 +6111,12 @@ void ggml_compute_forward_clamp( } break; case GGML_TYPE_F16: { - ggml_compute_forward_clamp_f16(params, dst); + ggml_compute_forward_clamp_half(params, dst); } break; case GGML_TYPE_BF16: + { + ggml_compute_forward_clamp_half(params, dst); + } break; case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index c54ed3f07..4aac5e966 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -962,12 +962,6 @@ inline static void ggml_vec_relu_f16 (const int n, ggml_fp16_t * y, const ggml_f } } inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); } -inline static void ggml_vec_leaky_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float ns) { - for (int i = 0; i < n; ++i) { - float v = GGML_CPU_FP16_TO_FP32(x[i]); - y[i] = GGML_CPU_FP32_TO_FP16(((v > 0.f) ? v : 0.f) + ns * ((v < 0.0f) ? v : 0.f)); - } -} inline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); } inline static void ggml_vec_sigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { for (int i = 0; i < n; ++i) { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index cc3ce73d0..b7842d289 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -6564,7 +6564,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_CLAMP: - return ggml_is_contiguous_rows(op->src[0]); + return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); case GGML_OP_ADD: case GGML_OP_SUB: case GGML_OP_MUL: @@ -6664,7 +6664,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_PAD_REFLECT_1D: case GGML_OP_ARANGE: case GGML_OP_TIMESTEP_EMBEDDING: + return true; case GGML_OP_LEAKY_RELU: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; case GGML_OP_RWKV_WKV6: case GGML_OP_GATED_LINEAR_ATTN: case GGML_OP_RWKV_WKV7: @@ -6690,7 +6692,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_FILL: case GGML_OP_CUMSUM: case GGML_OP_TRI: + return true; case GGML_OP_DIAG: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; case GGML_OP_SOLVE_TRI: return true; case GGML_OP_LIGHTNING_INDEXER: diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 0ea4a821a..f4c4991e9 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -2019,7 +2019,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te }; } case GGML_OP_DIAG: - return true; + return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_OPT_STEP_ADAMW: case GGML_OP_OPT_STEP_SGD: return has_simdgroup_reduction; diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 8cd38f382..3a15efabf 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -12283,7 +12283,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_TRANSPOSE: return true; case GGML_OP_DIAG: - return true; + return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_ROPE: { const int mode = ((const int32_t *) op->op_params)[2]; if (!ggml_rope_get_truncate(op)) { diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 3bcbad6f8..4fc90e543 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -7700,8 +7700,10 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons return false; } case GGML_OP_FILL: + return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16; case GGML_OP_CUMSUM: case GGML_OP_DIAG: + return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_CROSS_ENTROPY_LOSS: case GGML_OP_CROSS_ENTROPY_LOSS_BACK: return true; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index e19e2d029..54c7a3690 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -838,15 +838,16 @@ struct vk_device_struct { vk_pipeline pipeline_acc_f32; vk_pipeline pipeline_set_f32; + // [src0][src1][dst], 0=fp32,1=fp16,2=bf16, null where unsupported + vk_pipeline pipeline_add[3][3][3]; + vk_pipeline pipeline_add_norepeat[3][3][3]; + vk_pipeline pipeline_sub[3][3][3]; + vk_pipeline pipeline_sub_norepeat[3][3][3]; + vk_pipeline pipeline_mul[3][3][3]; + vk_pipeline pipeline_mul_norepeat[3][3][3]; + vk_pipeline pipeline_div[3][3][3]; + vk_pipeline pipeline_div_norepeat[3][3][3]; // [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16] - vk_pipeline pipeline_add[2][2][2]; - vk_pipeline pipeline_add_norepeat[2][2][2]; - vk_pipeline pipeline_sub[2][2][2]; - vk_pipeline pipeline_sub_norepeat[2][2][2]; - vk_pipeline pipeline_mul[2][2][2]; - vk_pipeline pipeline_mul_norepeat[2][2][2]; - vk_pipeline pipeline_div[2][2][2]; - vk_pipeline pipeline_div_norepeat[2][2][2]; vk_pipeline pipeline_add_rms[2][2][2]; vk_pipeline pipeline_add_rms_norepeat[2][2][2]; @@ -859,11 +860,17 @@ struct vk_device_struct { vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64; vk_pipeline pipeline_concat_transpose_i8, pipeline_concat_transpose_i16, pipeline_concat_transpose_i32; vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32; - vk_pipeline pipeline_scale_f32; - vk_pipeline pipeline_log[2]; - vk_pipeline pipeline_tri[2]; - vk_pipeline pipeline_diag[2]; - vk_pipeline pipeline_clamp[2]; + // [src/dst 0=fp32,1=fp16,2=bf16] + vk_pipeline pipeline_scale[3]; + vk_pipeline pipeline_log[3]; + vk_pipeline pipeline_tri[3]; + vk_pipeline pipeline_diag[3]; + vk_pipeline pipeline_clamp[3]; + vk_pipeline pipeline_sqr[3]; + vk_pipeline pipeline_sqrt[3]; + vk_pipeline pipeline_sin[3]; + vk_pipeline pipeline_cos[3]; + vk_pipeline pipeline_leaky_relu[3]; vk_pipeline pipeline_pad_f32; vk_pipeline pipeline_pad_reflect_1d_f32; vk_pipeline pipeline_roll_f32; @@ -880,10 +887,10 @@ struct vk_device_struct { vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT]; vk_pipeline pipeline_norm_f32; vk_pipeline pipeline_group_norm_f32; - std::array pipeline_rms_norm_f32; - std::array pipeline_rms_norm_mul_f32; - std::array pipeline_rms_norm_mul_add_f32; - std::array pipeline_rms_norm_mul_add_mul_f32; + // [src/dst 0=fp32,1=bf16][mul] + std::array pipeline_rms_norm[2][2]; + // [src/dst 0=fp32,1=bf16][residual 0=fp32,1=bf16][post mul] + std::array pipeline_rms_norm_mul_add[2][2][2]; vk_pipeline pipeline_rms_norm_mul_add_partials_f32; vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32; vk_pipeline pipeline_rms_norm_set_rows_f32_f32; @@ -895,36 +902,11 @@ struct vk_device_struct { vk_pipeline pipeline_rms_norm_back_f32; vk_pipeline pipeline_l2_norm_f32; - // [src/dst 0=fp32,1=fp16] - vk_pipeline pipeline_exp[2]; - vk_pipeline pipeline_expm1[2]; - vk_pipeline pipeline_elu[2]; - vk_pipeline pipeline_gelu[2]; - vk_pipeline pipeline_gelu_erf[2]; - vk_pipeline pipeline_gelu_quick[2]; - vk_pipeline pipeline_silu[2]; - vk_pipeline pipeline_relu[2]; - vk_pipeline pipeline_sqr[2]; - vk_pipeline pipeline_sqrt[2]; - vk_pipeline pipeline_sin[2]; - vk_pipeline pipeline_cos[2]; - vk_pipeline pipeline_xielu[2]; - vk_pipeline pipeline_neg[2]; - vk_pipeline pipeline_tanh[2]; - vk_pipeline pipeline_sigmoid[2]; - vk_pipeline pipeline_hardsigmoid[2]; - vk_pipeline pipeline_hardswish[2]; - vk_pipeline pipeline_abs[2]; - vk_pipeline pipeline_softplus[2]; - vk_pipeline pipeline_step[2]; - vk_pipeline pipeline_round[2]; - vk_pipeline pipeline_ceil[2]; - vk_pipeline pipeline_floor[2]; - vk_pipeline pipeline_trunc[2]; - vk_pipeline pipeline_sgn[2]; + // [src/dst 0=fp32,1=fp16,2=bf16] + vk_pipeline pipeline_unary[GGML_UNARY_OP_COUNT][3]; - // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b] - vk_pipeline pipeline_unary_mul[4][2][2][2]; + // fused UNARY+MUL pipelines: [op][0=fp32,1=fp16,2=bf16][norepeat][op_on_b] + vk_pipeline pipeline_unary_mul[4][3][2][2]; vk_pipeline pipeline_add1_f16_f16; vk_pipeline pipeline_add1_f16_f32; @@ -935,15 +917,9 @@ struct vk_device_struct { vk_pipeline pipeline_fill_f32; vk_pipeline pipeline_fill_f16; - vk_pipeline pipeline_geglu[2]; - vk_pipeline pipeline_reglu[2]; - vk_pipeline pipeline_swiglu[2]; - vk_pipeline pipeline_swiglu_oai[2]; - vk_pipeline pipeline_swiglu_clamp[2]; - vk_pipeline pipeline_geglu_erf[2]; - vk_pipeline pipeline_geglu_quick[2]; + // [src/dst 0=fp32,1=fp16,2=bf16] + vk_pipeline pipeline_glu[GGML_GLU_OP_COUNT][3]; - vk_pipeline pipeline_leaky_relu[2]; vk_pipeline pipeline_silu_back_f32; vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16; vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 45bfe52f4..616a17c4c 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -3600,10 +3600,16 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { for (uint32_t i = 0; i < 3; ++i) { const uint32_t block_size = 128u << i; const std::string suffix = "_" + std::to_string(block_size); - ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_f32[i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups); - ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_f32[i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups); - ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_f32[i], "rms_norm_mul_add_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups); - ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_mul_f32[i], "rms_norm_mul_add_mul_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1, block_size, rms_unroll}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[0][0][i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[0][1][i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[1][0][i], "rms_norm_bf16" + suffix, RMS_SHADER(rms_norm_bf16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[1][1][i], "rms_norm_mul_bf16" + suffix, RMS_SHADER(rms_norm_bf16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups); + for (uint32_t pm : {0, 1}) { + const std::string name = pm ? "rms_norm_mul_add_mul" : "rms_norm_mul_add"; + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[0][0][pm][i], name + "_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[1][0][pm][i], name + "_bf16_f32" + suffix, RMS_SHADER(rms_norm_mul_add_bf16_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[1][1][pm][i], name + "_bf16_bf16" + suffix, RMS_SHADER(rms_norm_mul_add_bf16_bf16), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups); + } } ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_partials_f32, "rms_norm_mul_add_mul_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true); @@ -3714,6 +3720,22 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_BINARY(add_rms, _norepeat, {1}, 4) #undef CREATE_BINARY +#define CREATE_BINARY_BF16(name, namemod, spec, bindings) \ + ggml_vk_create_pipeline2(device, device->pipeline_ ## name ## namemod[2][0][2], #name "_bf16_f32_bf16" #namemod, name ## _bf16_f32_bf16_len, name ## _bf16_f32_bf16_data, \ + "main", (bindings), sizeof(vk_op_binary_push_constants), {512, 1, 1}, spec, 1); \ + ggml_vk_create_pipeline2(device, device->pipeline_ ## name ## namemod[2][2][2], #name "_bf16_bf16_bf16" #namemod, name ## _bf16_bf16_bf16_len, name ## _bf16_bf16_bf16_data, \ + "main", (bindings), sizeof(vk_op_binary_push_constants), {512, 1, 1}, spec, 1); + + CREATE_BINARY_BF16(add, , {0}, 4) + CREATE_BINARY_BF16(add, _norepeat, {1}, 4) + CREATE_BINARY_BF16(sub, , {0}, 3) + CREATE_BINARY_BF16(sub, _norepeat, {1}, 3) + CREATE_BINARY_BF16(mul, , {0}, 3) + CREATE_BINARY_BF16(mul, _norepeat, {1}, 3) + CREATE_BINARY_BF16(div, , {0}, 3) + CREATE_BINARY_BF16(div, _norepeat, {1}, 3) +#undef CREATE_BINARY_BF16 + if (device->multi_add) { for (uint32_t i = 0; i < MAX_FUSED_ADDS; ++i) { ggml_vk_create_pipeline2(device, device->pipeline_multi_add[i], "multi_add_f32_" + std::to_string(i+1), multi_add_f32_len, multi_add_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1); @@ -3739,17 +3761,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_upscale_bicubic_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BICUBIC}, 1); ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_antialias_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS}, 1); - ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - - ggml_vk_create_pipeline(device, device->pipeline_log[0], "log_f32", log_f32_len, log_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_log[1], "log_f16", log_f16_len, log_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - - ggml_vk_create_pipeline(device, device->pipeline_tri[0], "tri_f32", tri_f32_len, tri_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_tri[1], "tri_f16", tri_f16_len, tri_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - - ggml_vk_create_pipeline(device, device->pipeline_diag[0], "diag_f32", diag_f32_len, diag_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_diag[1], "diag_f16", diag_f16_len, diag_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_pad_f32, "pad_f32", pad_f32_len, pad_f32_data, "main", 2, sizeof(vk_op_pad_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_pad_reflect_1d_f32, "pad_reflect_1d_f32", pad_reflect_1d_f32_len, pad_reflect_1d_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); @@ -3762,44 +3773,63 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #define CREATE_UNARY(name) \ ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \ - ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \ + ggml_vk_create_pipeline(device, device->pipeline_ ## name [2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - CREATE_UNARY(elu) - CREATE_UNARY(gelu) - CREATE_UNARY(gelu_erf) - CREATE_UNARY(gelu_quick) - CREATE_UNARY(silu) - CREATE_UNARY(relu) + CREATE_UNARY(scale) + CREATE_UNARY(log) CREATE_UNARY(sqr) CREATE_UNARY(sqrt) CREATE_UNARY(sin) CREATE_UNARY(cos) CREATE_UNARY(clamp) CREATE_UNARY(leaky_relu) - CREATE_UNARY(xielu) - CREATE_UNARY(neg) - CREATE_UNARY(tanh) - CREATE_UNARY(sigmoid) - CREATE_UNARY(hardsigmoid) - CREATE_UNARY(hardswish) - CREATE_UNARY(abs) - CREATE_UNARY(softplus) - CREATE_UNARY(step) - CREATE_UNARY(round) - CREATE_UNARY(ceil) - CREATE_UNARY(floor) - CREATE_UNARY(trunc) - CREATE_UNARY(sgn) - CREATE_UNARY(exp) - CREATE_UNARY(expm1) #undef CREATE_UNARY +#define CREATE_COPY_OP(name) \ + ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_32", name ## _32_len, name ## _32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \ + ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_16", name ## _16_len, name ## _16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \ + device->pipeline_ ## name [2] = device->pipeline_ ## name [1]; + + CREATE_COPY_OP(tri) + CREATE_COPY_OP(diag) +#undef CREATE_COPY_OP + +#define CREATE_UNARY_OP(name, op) \ + ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \ + ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \ + ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + + CREATE_UNARY_OP(elu, ELU) + CREATE_UNARY_OP(gelu, GELU) + CREATE_UNARY_OP(gelu_erf, GELU_ERF) + CREATE_UNARY_OP(gelu_quick, GELU_QUICK) + CREATE_UNARY_OP(silu, SILU) + CREATE_UNARY_OP(relu, RELU) + CREATE_UNARY_OP(xielu, XIELU) + CREATE_UNARY_OP(neg, NEG) + CREATE_UNARY_OP(tanh, TANH) + CREATE_UNARY_OP(sigmoid, SIGMOID) + CREATE_UNARY_OP(hardsigmoid, HARDSIGMOID) + CREATE_UNARY_OP(hardswish, HARDSWISH) + CREATE_UNARY_OP(abs, ABS) + CREATE_UNARY_OP(softplus, SOFTPLUS) + CREATE_UNARY_OP(step, STEP) + CREATE_UNARY_OP(round, ROUND) + CREATE_UNARY_OP(ceil, CEIL) + CREATE_UNARY_OP(floor, FLOOR) + CREATE_UNARY_OP(trunc, TRUNC) + CREATE_UNARY_OP(sgn, SGN) + CREATE_UNARY_OP(exp, EXP) + CREATE_UNARY_OP(expm1, EXPM1) +#undef CREATE_UNARY_OP + // spec constants: {norepeat, op_on_b} #define CREATE_UNARY_MUL(name, idx) \ - for (int dt = 0; dt < 2; ++dt) { \ - const size_t len_ = dt ? name ## _mul_f16_len : name ## _mul_f32_len; \ - const unsigned char * data_ = dt ? name ## _mul_f16_data : name ## _mul_f32_data; \ - const std::string dts_ = dt ? "f16" : "f32"; \ + for (int dt = 0; dt < 3; ++dt) { \ + const size_t len_ = dt == 2 ? name ## _mul_bf16_len : dt ? name ## _mul_f16_len : name ## _mul_f32_len; \ + const unsigned char * data_ = dt == 2 ? name ## _mul_bf16_data : dt ? name ## _mul_f16_data : name ## _mul_f32_data; \ + const std::string dts_ = dt == 2 ? "bf16" : dt ? "f16" : "f32"; \ for (int ob = 0; ob < 2; ++ob) \ for (int nr = 0; nr < 2; ++nr) \ ggml_vk_create_pipeline(device, device->pipeline_unary_mul[(idx)][dt][nr][ob], \ @@ -3819,17 +3849,18 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); -#define CREATE_GLU(name) \ - ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); +#define CREATE_GLU(name, op) \ + ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); - CREATE_GLU(geglu) - CREATE_GLU(reglu) - CREATE_GLU(swiglu) - CREATE_GLU(swiglu_oai) - CREATE_GLU(swiglu_clamp) - CREATE_GLU(geglu_erf) - CREATE_GLU(geglu_quick) + CREATE_GLU(geglu, GEGLU) + CREATE_GLU(reglu, REGLU) + CREATE_GLU(swiglu, SWIGLU) + CREATE_GLU(swiglu_oai, SWIGLU_OAI) + CREATE_GLU(swiglu_clamp, SWIGLU_CLAMP) + CREATE_GLU(geglu_erf, GEGLU_ERF) + CREATE_GLU(geglu_quick, GEGLU_QUICK) #undef CREATE_GLU ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); @@ -9374,6 +9405,33 @@ static bool ggml_vk_concat_is_transposed(const vk_device & device, const ggml_te return src0->ne[1] == src1->ne[1] && dst->ne[1] == src1->ne[1]; } +static bool ggml_vk_is_float_type(ggml_type type) { + return type == GGML_TYPE_F32 || type == GGML_TYPE_F16 || type == GGML_TYPE_BF16; +} + +static uint32_t ggml_vk_float_type_idx(ggml_type type) { + return type == GGML_TYPE_BF16 ? 2 : type == GGML_TYPE_F16 ? 1 : 0; +} + +static bool ggml_vk_is_float_elementwise(const ggml_tensor * op) { + return ggml_vk_is_float_type(op->type) && op->src[0]->type == op->type; +} + +// bf16: bf16 src0/dst, f32 or bf16 src1 +static bool ggml_vk_binary_types_supported(ggml_type src0, ggml_type src1, ggml_type dst) { + if (src0 == GGML_TYPE_BF16 || src1 == GGML_TYPE_BF16 || dst == GGML_TYPE_BF16) { + return src0 == GGML_TYPE_BF16 && dst == GGML_TYPE_BF16 && (src1 == GGML_TYPE_BF16 || src1 == GGML_TYPE_F32); + } + return ggml_vk_is_float_type(src0) && ggml_vk_is_float_type(src1) && ggml_vk_is_float_type(dst); +} + +static vk_pipeline ggml_vk_float_pipeline(const vk_pipeline (&pipelines)[3], const ggml_tensor * src0, const ggml_tensor * dst) { + if (src0->type != dst->type || !ggml_vk_is_float_type(dst->type)) { + return nullptr; + } + return pipelines[ggml_vk_float_type_idx(dst->type)]; +} + static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_GET_ROWS: @@ -9410,14 +9468,19 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const case GGML_OP_SUB: case GGML_OP_MUL: case GGML_OP_DIV: - if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || - (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) || - (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16)) { + { + if (!ggml_vk_binary_types_supported(src0->type, src1->type, dst->type)) { return nullptr; } + const uint32_t s0 = ggml_vk_float_type_idx(src0->type); + const uint32_t s1 = ggml_vk_float_type_idx(src1->type); + const uint32_t d = ggml_vk_float_type_idx(dst->type); switch (op) { case GGML_OP_ADD: { + if ((ctx->num_additional_fused_ops > 0 || ctx->do_add_rms_partials) && (s0 == 2 || s1 == 2 || d == 2)) { + return nullptr; + } if (ctx->num_additional_fused_ops > 0) { if (ctx->do_add_rms_partials) { return ctx->device->pipeline_multi_add_rms[ctx->num_additional_fused_ops]; @@ -9430,28 +9493,29 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; } else { auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_add_norepeat : ctx->device->pipeline_add; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; + return pipelines[s0][s1][d]; } } case GGML_OP_SUB: { auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_sub_norepeat : ctx->device->pipeline_sub; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; + return pipelines[s0][s1][d]; } case GGML_OP_MUL: { auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_mul_norepeat : ctx->device->pipeline_mul; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; + return pipelines[s0][s1][d]; } case GGML_OP_DIV: { auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_div_norepeat : ctx->device->pipeline_div; - return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16]; + return pipelines[s0][s1][d]; } default: break; } return nullptr; + } case GGML_OP_ADD_ID: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_add_id_f32; @@ -9509,58 +9573,23 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const } return nullptr; case GGML_OP_SCALE: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_scale_f32; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_scale, src0, dst); case GGML_OP_SQR: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_sqr[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_sqr, src0, dst); case GGML_OP_SQRT: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_sqrt[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_sqrt, src0, dst); case GGML_OP_SIN: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_sin[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_sin, src0, dst); case GGML_OP_COS: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_cos[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_cos, src0, dst); case GGML_OP_LOG: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_log[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_log, src0, dst); case GGML_OP_TRI: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_tri[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_tri, src0, dst); case GGML_OP_DIAG: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_diag[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_diag, src0, dst); case GGML_OP_CLAMP: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_clamp[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_clamp, src0, dst); case GGML_OP_PAD: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_pad_f32; @@ -9626,7 +9655,10 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (ctx->do_add_rms_partials) { return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32; } - return ggml_vk_rms_norm_pipeline(ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32, src0->ne[0]); + return ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm[0][ctx->fused_rms_norm_mode == RMS_NORM_MUL], src0->ne[0]); + } + if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16 && !ctx->do_add_rms_partials) { + return ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm[1][ctx->fused_rms_norm_mode == RMS_NORM_MUL], src0->ne[0]); } return nullptr; case GGML_OP_RMS_NORM_BACK: @@ -9640,87 +9672,9 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const } return nullptr; case GGML_OP_UNARY: - if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || - (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) || - (src0->type != dst->type)) { - return nullptr; - } - - switch (ggml_get_unary_op(dst)) { - case GGML_UNARY_OP_EXP: - return ctx->device->pipeline_exp[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_EXPM1: - return ctx->device->pipeline_expm1[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_ELU: - return ctx->device->pipeline_elu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_SILU: - return ctx->device->pipeline_silu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_GELU: - return ctx->device->pipeline_gelu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_GELU_ERF: - return ctx->device->pipeline_gelu_erf[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_GELU_QUICK: - return ctx->device->pipeline_gelu_quick[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_RELU: - return ctx->device->pipeline_relu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_XIELU: - return ctx->device->pipeline_xielu[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_NEG: - return ctx->device->pipeline_neg[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_TANH: - return ctx->device->pipeline_tanh[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_SIGMOID: - return ctx->device->pipeline_sigmoid[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_HARDSIGMOID: - return ctx->device->pipeline_hardsigmoid[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_HARDSWISH: - return ctx->device->pipeline_hardswish[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_ABS: - return ctx->device->pipeline_abs[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_SOFTPLUS: - return ctx->device->pipeline_softplus[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_STEP: - return ctx->device->pipeline_step[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_ROUND: - return ctx->device->pipeline_round[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_CEIL: - return ctx->device->pipeline_ceil[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_FLOOR: - return ctx->device->pipeline_floor[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_TRUNC: - return ctx->device->pipeline_trunc[dst->type == GGML_TYPE_F16]; - case GGML_UNARY_OP_SGN: - return ctx->device->pipeline_sgn[dst->type == GGML_TYPE_F16]; - default: - break; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_unary[ggml_get_unary_op(dst)], src0, dst); case GGML_OP_GLU: - if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || - (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) || - (src0->type != dst->type)) { - return nullptr; - } - - switch (ggml_get_glu_op(dst)) { - case GGML_GLU_OP_GEGLU: - return ctx->device->pipeline_geglu[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_REGLU: - return ctx->device->pipeline_reglu[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_SWIGLU: - return ctx->device->pipeline_swiglu[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_SWIGLU_OAI: - return ctx->device->pipeline_swiglu_oai[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_SWIGLU_CLAMP: - return ctx->device->pipeline_swiglu_clamp[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_GEGLU_ERF: - return ctx->device->pipeline_geglu_erf[dst->type == GGML_TYPE_F16]; - case GGML_GLU_OP_GEGLU_QUICK: - return ctx->device->pipeline_geglu_quick[dst->type == GGML_TYPE_F16]; - default: - break; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_glu[ggml_get_glu_op(dst)], src0, dst); case GGML_OP_SOFT_MAX: GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F32); @@ -9965,11 +9919,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const } return nullptr; case GGML_OP_LEAKY_RELU: - if (src0->type == dst->type && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) { - return ctx->device->pipeline_leaky_relu[dst->type == GGML_TYPE_F16]; - } - return nullptr; + return ggml_vk_float_pipeline(ctx->device->pipeline_leaky_relu, src0, dst); case GGML_OP_CONV_2D: case GGML_OP_CONV_TRANSPOSE_2D: if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { @@ -10805,13 +10755,12 @@ void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const const ggml_tensor * src1 = op_on_b ? unary->src[0] : ((mul->src[0] == unary) ? mul->src[1] : mul->src[0]); - const bool f16 = src0->type == GGML_TYPE_F16; const bool norepeat = ggml_are_same_shape(src0, src1); const int oi = ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)); if (oi < 0) { GGML_ABORT("fatal error"); } - vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][f16][norepeat][op_on_b]; + vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][ggml_vk_float_type_idx(src0->type)][norepeat][op_on_b]; const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t src1_type_size = ggml_type_size(src1->type); @@ -11592,11 +11541,13 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s vk_pipeline pipeline; if (ctx->do_add_rms_partials) { + GGML_ASSERT(src0->type == GGML_TYPE_F32); pipeline = do_post_multiply ? ctx->device->pipeline_rms_norm_mul_add_mul_partials_f32 : ctx->device->pipeline_rms_norm_mul_add_partials_f32; } else { - pipeline = ggml_vk_rms_norm_pipeline(do_post_multiply ? - ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32, src0->ne[0]); + const bool src0_bf16 = src0->type == GGML_TYPE_BF16; + const bool residual_bf16 = residual->type == GGML_TYPE_BF16; + pipeline = ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm_mul_add[src0_bf16][residual_bf16][do_post_multiply], src0->ne[0]); } ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); if (ctx->do_add_rms_partials) { @@ -12994,6 +12945,8 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in if (next_node_idx < cgraph->n_nodes && cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM && cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] && + cgraph->nodes[next_node_idx]->type == GGML_TYPE_F32 && + node->type == GGML_TYPE_F32 && ggml_nrows(cgraph->nodes[next_node_idx]) == 1 && ctx->device->add_rms_fusion && !ctx->device->disable_fusion) { uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]); @@ -14292,7 +14245,7 @@ static bool ggml_vk_can_fuse_unary_mul(const struct ggml_cgraph * cgraph, int un if (ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) { return false; } - if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) { + if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16 && unary->type != GGML_TYPE_BF16) { return false; } if (unary->type != mul->type) { @@ -14350,12 +14303,24 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; const ggml_tensor *mul = cgraph->nodes[node_idx + 1]; - GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); - // rms_norm only supports f32 - if (mul->src[0]->type != GGML_TYPE_F32 || - mul->src[1]->type != GGML_TYPE_F32 || - mul->type != GGML_TYPE_F32) { + const ggml_type t = rms_norm->type; + if (t == GGML_TYPE_F32) { + if (mul->src[0]->type != GGML_TYPE_F32 || + mul->src[1]->type != GGML_TYPE_F32 || + mul->type != GGML_TYPE_F32) { + return false; + } + } else if (t == GGML_TYPE_BF16) { + // bf16 activations, f32 weights + if (mul->src[0] != rms_norm || + mul->src[1]->type != GGML_TYPE_F32 || + mul->type != GGML_TYPE_BF16) { + return false; + } + if (ops.size() >= 3 && ops.begin()[2] != GGML_OP_ADD) { + return false; + } + } else { return false; } // if rms_norm is the B operand, then we don't handle broadcast @@ -14374,7 +14339,7 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr if (add->src[0] != mul && add->src[1] != mul) { return false; } - if (residual->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 || + if ((residual->type != GGML_TYPE_F32 && residual->type != t) || add->type != t || !ggml_are_same_shape(add, residual) || !ggml_is_contiguous(residual) || !ggml_is_contiguous(add) || get_misalign_bytes(ctx, residual) != 0) { return false; @@ -14392,7 +14357,7 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr return false; } // The shader reads data_e[0], so the final multiply must use a scalar. - if (scale->type != GGML_TYPE_F32 || post_mul->type != GGML_TYPE_F32 || + if (scale->type != GGML_TYPE_F32 || post_mul->type != t || ggml_nelements(scale) != 1 || !ggml_is_contiguous(post_mul) || get_misalign_bytes(ctx, scale) != 0) { return false; @@ -14933,6 +14898,10 @@ bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const str const int mode = ((const int32_t *) rope->op_params)[2]; + if (rms->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { + return false; + } + // noncontig tensors aren't tested, and don't seem common in practice if (!ggml_is_contiguous(rms) || !ggml_is_contiguous(mul) || @@ -16257,9 +16226,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_UNARY_OP_FLOOR: case GGML_UNARY_OP_TRUNC: case GGML_UNARY_OP_SGN: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && - (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && - (op->src[0]->type == op->type); + return ggml_vk_is_float_elementwise(op); default: return false; } @@ -16272,10 +16239,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: case GGML_GLU_OP_SWIGLU_CLAMP: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && - (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && - (op->src[0]->type == op->type) && - (!op->src[1] || op->src[1]->type == op->src[0]->type); + return ggml_vk_is_float_elementwise(op) && (!op->src[1] || op->src[1]->type == op->type); default: return false; } @@ -16548,16 +16512,15 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_GROUP_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_NORM: - case GGML_OP_RMS_NORM: case GGML_OP_L2_NORM: return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_RMS_NORM: + return ggml_vk_is_float_elementwise(op) && op->type != GGML_TYPE_F16; case GGML_OP_ADD: case GGML_OP_SUB: case GGML_OP_MUL: case GGML_OP_DIV: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && - (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && - (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); + return ggml_vk_binary_types_supported(op->src[0]->type, op->src[1]->type, op->type); case GGML_OP_ADD_ID: return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->src[2]->type == GGML_TYPE_I32 && op->type == GGML_TYPE_F32; @@ -16568,10 +16531,12 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_SQRT: case GGML_OP_SIN: case GGML_OP_COS: + case GGML_OP_LOG: case GGML_OP_CLAMP: case GGML_OP_LEAKY_RELU: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && - op->type == op->src[0]->type; + case GGML_OP_TRI: + case GGML_OP_DIAG: + return ggml_vk_is_float_elementwise(op); case GGML_OP_OPT_STEP_ADAMW: case GGML_OP_OPT_STEP_SGD: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; @@ -16579,11 +16544,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; - case GGML_OP_LOG: - case GGML_OP_TRI: - case GGML_OP_DIAG: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && - op->type == op->src[0]->type; case GGML_OP_ARGSORT: { if (!ggml_is_contiguous(op) || !ggml_is_contiguous(op->src[0])) { @@ -16629,7 +16589,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_FILL: return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16; case GGML_OP_SCALE: - return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; + return ggml_is_contiguous(op->src[0]) && ggml_vk_is_float_elementwise(op); case GGML_OP_PAD: case GGML_OP_PAD_REFLECT_1D: case GGML_OP_ROLL: diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/add.comp b/ggml/src/ggml-vulkan/vulkan-shaders/add.comp index 3bcfe6908..2d93792ae 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/add.comp @@ -36,10 +36,10 @@ void main() { uint i00, i01, i02, i03; get_indices(idx, i00, i01, i02, i03); - FLOAT_TYPE sum = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]); + FLOAT_TYPE sum = A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]); sum_sq += sum*sum; - data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(sum); + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(sum); idx += num_threads; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp b/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp index 249e6b16e..790a5421d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp @@ -20,8 +20,7 @@ void main() { const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10; if (i10 == i11) { - const float val = float(data_a[get_aoffset() + i13*p.nb03 + i12*p.nb02 + 0*p.nb01 + i10*p.nb00]); - data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val); + data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + i13*p.nb03 + i12*p.nb02 + 0*p.nb01 + i10*p.nb00]); } else { data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/div.comp b/ggml/src/ggml-vulkan/vulkan-shaders/div.comp index 572472f8a..781f2004d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/div.comp @@ -20,7 +20,7 @@ void main() { uint i00, i01, i02, i03; get_indices(idx, i00, i01, i02, i03); - data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) / FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) / B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); idx += num_threads; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl index fc2951ec2..aed5a65a6 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl @@ -1,5 +1,6 @@ #extension GL_EXT_shader_16bit_storage : require +#include "types.glsl" #include "utils.glsl" diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl index 14c5e7a54..327191d51 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl @@ -21,15 +21,15 @@ void main() { const uint offset = (p.ne00 / 2) * p.nb00; const uint idx = src_idx_a; - data_d[dst_idx] = D_TYPE(op(float(data_a[idx]), float(data_a[idx + offset]))); + data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[idx]), A_TO_FLOAT(data_a[idx + offset]))); } else if (p.mode == 1) { // Swapped const uint offset = (p.ne00 / 2) * p.nb00; const uint idx = src_idx_a; - data_d[dst_idx] = D_TYPE(op(float(data_a[idx + offset]), float(data_a[idx]))); + data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[idx + offset]), A_TO_FLOAT(data_a[idx]))); } else { // Split - data_d[dst_idx] = D_TYPE(op(float(data_a[src_idx_a]), float(data_b[src_idx_b]))); + data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[src_idx_a]), A_TO_FLOAT(data_b[src_idx_b]))); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/log.comp b/ggml/src/ggml-vulkan/vulkan-shaders/log.comp index 3cda6a63c..7eb1e7fd8 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/log.comp @@ -12,6 +12,6 @@ void main() { return; } - const float val = float(data_a[get_aoffset() + src0_idx(idx)]); - data_d[get_doffset() + dst_idx(idx)] = D_TYPE(log(val)); + const float val = A_TO_FLOAT(data_a[get_aoffset() + src0_idx(idx)]); + data_d[get_doffset() + dst_idx(idx)] = FLOAT_TO_D(log(val)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp index 02ef1eace..eca78e33e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp @@ -20,7 +20,7 @@ void main() { uint i00, i01, i02, i03; get_indices(idx, i00, i01, i02, i03); - data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) * B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); idx += num_threads; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp index 31e4483b2..b9f0a1841 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp @@ -41,7 +41,7 @@ layout (binding = 6) readonly buffer R_I {uvec2 rope_data_i[];}; // indices for #elif RMS_NORM_ADD_FUSION -layout (binding = 3) readonly buffer C {float data_c[];}; +layout (binding = 3) readonly buffer C {C_TYPE data_c[];}; layout (binding = 4) readonly buffer E {float data_e[];}; #elif RMS_NORM_SET_ROWS_FUSION @@ -90,7 +90,7 @@ void rms_norm(uint num_iters) { [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { FLOAT_TYPE xi = FLOAT_TYPE(0); if (col < ncols) { - xi = FLOAT_TYPE(data_a[a_offset + col]); + xi = FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])); } sum += xi * xi; } @@ -137,28 +137,28 @@ void rms_norm(uint num_iters) { if (col >= ncols) { continue; } - FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]); + FLOAT_TYPE value = scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])) * FLOAT_TYPE(B_TO_FLOAT(data_b[b_offset + fastmod(col, p.ne10)])); #if RMS_NORM_ADD_FUSION - value += FLOAT_TYPE(data_c[d_offset + col]); + value += FLOAT_TYPE(C_TO_FLOAT(data_c[d_offset + col])); if (do_post_multiply) { value *= FLOAT_TYPE(data_e[0]); } #endif - data_d[d_offset + col] = D_TYPE(value); + data_d[d_offset + col] = FLOAT_TO_D(value); } } else { [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { if (col >= ncols) { continue; } - FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]); + FLOAT_TYPE value = scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])) * FLOAT_TYPE(B_TO_FLOAT(data_b[b_offset + col])); #if RMS_NORM_ADD_FUSION - value += FLOAT_TYPE(data_c[d_offset + col]); + value += FLOAT_TYPE(C_TO_FLOAT(data_c[d_offset + col])); if (do_post_multiply) { value *= FLOAT_TYPE(data_e[0]); } #endif - data_d[d_offset + col] = D_TYPE(value); + data_d[d_offset + col] = FLOAT_TO_D(value); } } } else { @@ -166,7 +166,7 @@ void rms_norm(uint num_iters) { if (col >= ncols) { continue; } - data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col])); + data_d[d_offset + col] = FLOAT_TO_D(scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col]))); } } #if RMS_NORM_ROPE_FUSION diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp b/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp index 35ec726a0..83ee31a72 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp @@ -18,7 +18,7 @@ void main() { continue; } - data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2)); + data_d[get_doffset() + idx] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2)); idx += num_threads; } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp b/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp index bc924b520..ebecb2c0f 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp @@ -22,7 +22,7 @@ void main() { uint i00, i01, i02, i03; get_indices(idx, i00, i01, i02, i03); - data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) - FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); + data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) - B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); idx += num_threads; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp b/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp index 9def5dbc9..2585540af 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp @@ -34,8 +34,7 @@ void main() { } if (pass) { - const float val = float(data_a[get_aoffset() + src0_idx(idx)]); - data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val); + data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + src0_idx(idx)]); } else { data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index 99ce1458a..b7e5a31aa 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -1976,6 +1976,27 @@ vec4 bf16_to_fp32(uvec4 u) return vec4(bf16_to_fp32(u.x), bf16_to_fp32(u.y), bf16_to_fp32(u.z), bf16_to_fp32(u.w)); } +#if defined(DATA_A_BF16) +#define A_TO_FLOAT(x) bf16_to_fp32(uint32_t(x)) +#else +#define A_TO_FLOAT(x) float(x) +#endif +#if defined(DATA_B_BF16) +#define B_TO_FLOAT(x) bf16_to_fp32(uint32_t(x)) +#else +#define B_TO_FLOAT(x) float(x) +#endif +#if defined(DATA_C_BF16) +#define C_TO_FLOAT(x) bf16_to_fp32(uint32_t(x)) +#else +#define C_TO_FLOAT(x) float(x) +#endif +#if defined(DATA_D_BF16) +#define FLOAT_TO_D(x) D_TYPE(fp32_to_bf16(x)) +#else +#define FLOAT_TO_D(x) D_TYPE(x) +#endif + float e8m0_to_fp32(uint8_t x) { uint32_t bits; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp index 9ee7769ba..04a3756bd 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp @@ -184,10 +184,10 @@ void main() { if (op_on_b) { data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = - D_TYPE(FLOAT_TYPE(OP(float(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])); + FLOAT_TO_D(OP(B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])) * A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])); } else { data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = - D_TYPE(FLOAT_TYPE(OP(float(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); + FLOAT_TO_D(OP(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])) * B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])); } idx += num_threads; @@ -200,6 +200,6 @@ void main() { const uint a_idx = get_aoffset() + src0_idx(idx); const uint d_idx = get_doffset() + dst_idx(idx); - data_d[d_idx] = D_TYPE(OP(float(data_a[a_idx]))); + data_d[d_idx] = FLOAT_TO_D(OP(A_TO_FLOAT(data_a[a_idx]))); #endif } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 3952030fb..86b69d15f 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -863,9 +863,13 @@ void process_shaders() { string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); for (bool use_subgroups : {false, true}) { const std::string suffix = use_subgroups ? "_subgroup" : ""; - const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}}); + const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"C_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}}); + const auto rms_bf16_dict = merge_maps(base_dict, {{"A_TYPE", "uint16_t"}, {"B_TYPE", "float"}, {"C_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}}); string_to_spv("rms_norm_f32" + suffix, "rms_norm.comp", rms_dict); + string_to_spv("rms_norm_bf16" + suffix, "rms_norm.comp", rms_bf16_dict); string_to_spv("rms_norm_mul_add_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_ADD_FUSION", "1"}})); + string_to_spv("rms_norm_mul_add_bf16_f32" + suffix, "rms_norm.comp", merge_maps(rms_bf16_dict, {{"RMS_NORM_ADD_FUSION", "1"}})); + string_to_spv("rms_norm_mul_add_bf16_bf16" + suffix, "rms_norm.comp", merge_maps({{"C_TYPE", "uint16_t"}, {"DATA_C_BF16", "1"}, {"RMS_NORM_ADD_FUSION", "1"}}, rms_bf16_dict)); string_to_spv("rms_norm_set_rows_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SET_ROWS_FUSION", "1"}})); string_to_spv("rms_norm_set_rows_f32_f16" + suffix, "rms_norm.comp", merge_maps({{"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}, rms_dict)); // keep rms_dict last so 'merge_maps' overwrites the value for key 'D_TYPE' string_to_spv("rms_norm_mul_rope_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}})); @@ -934,6 +938,10 @@ void process_shaders() { } } } + for (std::string op : {"add", "sub", "mul", "div"}) { + string_to_spv(op + "_bf16_f32_bf16", op + ".comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"FLOAT_TYPE", "float"}, {"ADD_RMS" , "0"}}); + string_to_spv(op + "_bf16_bf16_bf16", op + ".comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_B_BF16", "1"}, {"DATA_D_BF16", "1"}, {"FLOAT_TYPE", "float"}, {"ADD_RMS" , "0"}}); + } string_to_spv("sub_f32", "sub.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); @@ -969,8 +977,6 @@ void process_shaders() { string_to_spv("repeat_i16", "repeat.comp", {{"A_TYPE", "int16_t"}, {"D_TYPE", "int16_t"}}); - string_to_spv("scale_f32", "scale.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); - string_to_spv("pad_f32", "pad.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("pad_reflect_1d_f32", "pad_reflect_1d.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); @@ -984,70 +990,36 @@ void process_shaders() { string_to_spv("upscale_f32", "upscale.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("exp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_exp"}}); - string_to_spv("exp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_exp"}}); - string_to_spv("expm1_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_expm1"}}); - string_to_spv("expm1_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_expm1"}}); + const std::vector>> float_types = { + {"f32", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}}, + {"f16", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}}, + {"bf16", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}}}, + }; + for (const auto & [t, dict] : float_types) { + for (const std::string op : {"exp", "expm1", "gelu", "gelu_erf", "gelu_quick", "silu", "relu", "sqr", "sqrt", "sin", + "cos", "clamp", "leaky_relu", "neg", "tanh", "sigmoid", "hardsigmoid", "hardswish", "abs", + "elu", "xielu", "sgn", "softplus", "step", "round", "ceil", "floor", "trunc"}) { + string_to_spv(op + "_" + t, "unary.comp", merge_maps(dict, {{"OP", "op_" + op}})); + } + for (const std::string op : {"geglu", "reglu", "swiglu", "swiglu_oai", "swiglu_clamp", "geglu_erf", "geglu_quick"}) { + string_to_spv(op + "_" + t, op + ".comp", dict); + } + string_to_spv("log_" + t, "log.comp", dict); + string_to_spv("scale_" + t, "scale.comp", merge_maps(dict, {{"FLOAT_TYPE", "float"}})); - string_to_spv("log_f16", "log.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("log_f32", "log.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("gelu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu"}}); - string_to_spv("gelu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu"}}); - string_to_spv("gelu_erf_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu_erf"}}); - string_to_spv("gelu_erf_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu_erf"}}); - string_to_spv("gelu_quick_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu_quick"}}); - string_to_spv("gelu_quick_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu_quick"}}); - string_to_spv("silu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_silu"}}); - string_to_spv("silu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_silu"}}); - string_to_spv("relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_relu"}}); - string_to_spv("relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_relu"}}); - string_to_spv("sqr_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqr"}}); - string_to_spv("sqr_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqr"}}); - string_to_spv("sqrt_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqrt"}}); - string_to_spv("sqrt_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqrt"}}); - string_to_spv("sin_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sin"}}); - string_to_spv("sin_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sin"}}); - string_to_spv("cos_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_cos"}}); - string_to_spv("cos_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_cos"}}); - string_to_spv("clamp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_clamp"}}); - string_to_spv("clamp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_clamp"}}); - string_to_spv("leaky_relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_leaky_relu"}}); - string_to_spv("leaky_relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_leaky_relu"}}); - string_to_spv("neg_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_neg"}}); - string_to_spv("neg_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_neg"}}); - string_to_spv("tanh_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_tanh"}}); - string_to_spv("tanh_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_tanh"}}); - string_to_spv("sigmoid_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sigmoid"}}); - string_to_spv("sigmoid_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sigmoid"}}); - string_to_spv("hardsigmoid_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_hardsigmoid"}}); - string_to_spv("hardsigmoid_f32","unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_hardsigmoid"}}); - string_to_spv("hardswish_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_hardswish"}}); - string_to_spv("hardswish_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_hardswish"}}); - string_to_spv("abs_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_abs"}}); - string_to_spv("abs_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_abs"}}); - string_to_spv("elu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_elu"}}); - string_to_spv("elu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_elu"}}); - string_to_spv("xielu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_xielu"}}); - string_to_spv("xielu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_xielu"}}); - string_to_spv("sgn_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sgn"}}); - string_to_spv("sgn_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sgn"}}); + auto mul_dict = merge_maps(dict, {{"B_TYPE", dict.at("A_TYPE")}, {"FLOAT_TYPE", "float"}, {"UNARY_MUL_FUSION", "1"}}); + if (t == "bf16") { + mul_dict["DATA_B_BF16"] = "1"; + } + for (const std::string op : {"gelu", "sigmoid", "silu", "softplus"}) { + string_to_spv(op + "_mul_" + t, "unary.comp", merge_maps(mul_dict, {{"OP", "op_" + op}})); + } + } - string_to_spv("tri_f16", "tri.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("tri_f32", "tri.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("diag_f16", "diag.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("diag_f32", "diag.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - - string_to_spv("softplus_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_softplus"}}); - string_to_spv("softplus_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_softplus"}}); - - string_to_spv("gelu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}}); - string_to_spv("gelu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}}); - string_to_spv("sigmoid_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}}); - string_to_spv("sigmoid_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}}); - string_to_spv("silu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}}); - string_to_spv("silu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}}); - string_to_spv("softplus_mul_f32","unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}}); - string_to_spv("softplus_mul_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}}); + string_to_spv("tri_16", "tri.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); + string_to_spv("tri_32", "tri.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}}); + string_to_spv("diag_16", "diag.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); + string_to_spv("diag_32", "diag.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}}); string_to_spv("add1_f16_f16", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}}); string_to_spv("add1_f16_f32", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}}); @@ -1055,31 +1027,6 @@ void process_shaders() { string_to_spv("arange_f32", "arange.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); string_to_spv("fill_f32", "fill.comp", {{"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); string_to_spv("fill_f16", "fill.comp", {{"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}}); - string_to_spv("step_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_step"}}); - string_to_spv("step_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_step"}}); - string_to_spv("round_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_round"}}); - string_to_spv("round_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_round"}}); - string_to_spv("ceil_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_ceil"}}); - string_to_spv("ceil_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_ceil"}}); - string_to_spv("floor_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_floor"}}); - string_to_spv("floor_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_floor"}}); - string_to_spv("trunc_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_trunc"}}); - string_to_spv("trunc_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_trunc"}}); - - string_to_spv("geglu_f16", "geglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("geglu_f32", "geglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("reglu_f16", "reglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("reglu_f32", "reglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("swiglu_f16", "swiglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("swiglu_f32", "swiglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("swiglu_oai_f16", "swiglu_oai.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("swiglu_oai_f32", "swiglu_oai.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("swiglu_clamp_f16", "swiglu_clamp.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("swiglu_clamp_f32", "swiglu_clamp.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("geglu_erf_f16", "geglu_erf.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("geglu_erf_f32", "geglu_erf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); - string_to_spv("geglu_quick_f16","geglu_quick.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); - string_to_spv("geglu_quick_f32","geglu_quick.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 1fca258a5..7132abc51 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -5280,7 +5280,7 @@ struct ggml_tensor * ggml_tri( struct ggml_context * ctx, struct ggml_tensor * a, enum ggml_tri_type type) { - GGML_ASSERT(a->type == GGML_TYPE_F32); + GGML_ASSERT(a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16); GGML_ASSERT(ggml_is_contiguous(a)); GGML_ASSERT(a->ne[0] == a->ne[1]); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f2ede4290..cddc2dbaa 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10770,7 +10770,7 @@ static std::vector> make_test_cases_eval() { } } - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { for (int v : {0, 1}) { for (float alpha : {.5f, 1.702f}) { for (float limit : {2.0f, 7.0f}) { @@ -10780,7 +10780,7 @@ static std::vector> make_test_cases_eval() { } } - for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { for (int v : {0, 1}) { for (float limit : {2.0f, 10.0f}) { test_cases.emplace_back(new test_swiglu_clamp(type, { 128, 2, 2, 2 }, v, limit)); @@ -11574,6 +11574,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f)); test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {100, 10, 10, 10}, 0.5f, 0.0f)); + test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {10, 10, 10, 10}, 2.0f, 1.0f)); + test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test + test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {100, 10, 10, 10}, 0.5f, 0.0f)); test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 0.0f, false, 1)); // view test test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace @@ -11684,6 +11687,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 1536, 1, 1, 1 }, 1e-6f, false, false, false, true)); test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 256, 4, 1, 1 }, 1e-6f, false, false, false, true)); test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, true, false, false, GGML_TYPE_F32)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, true, false, false, GGML_TYPE_F32, GGML_TYPE_BF16)); // the ADD operand is the BF16 input itself, added in place test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, false, true, false, GGML_TYPE_F32)); @@ -12471,7 +12475,7 @@ static std::vector> make_test_cases_eval() { } } - for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) { test_cases.emplace_back(new test_sqr (type)); test_cases.emplace_back(new test_sqrt (type)); test_cases.emplace_back(new test_log (type)); @@ -12946,10 +12950,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_xielu(GGML_TYPE_F16, { 512, 16, 1, 1 })); test_cases.emplace_back(new test_xielu(GGML_TYPE_BF16, { 512, 16, 1, 1 })); - test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER)); - test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG)); - test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER)); - test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG)); + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { + test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER, type)); + test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG, type)); + test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER, type)); + test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG, type)); + } test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER)); test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER_DIAG)); test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_UPPER)); @@ -12963,9 +12969,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_fill(-152.0f, GGML_TYPE_F16, { 800, 600, 4, 4 })); test_cases.emplace_back(new test_fill(3.5f, GGML_TYPE_F16, { 2048, 512, 2, 2 })); - test_cases.emplace_back(new test_diag()); - test_cases.emplace_back(new test_diag(GGML_TYPE_F32, { 79, 1, 19, 13 })); - test_cases.emplace_back(new test_diag(GGML_TYPE_F32, { 256, 1, 8, 16 })); + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { + test_cases.emplace_back(new test_diag(type)); + test_cases.emplace_back(new test_diag(type, { 79, 1, 19, 13 })); + test_cases.emplace_back(new test_diag(type, { 256, 1, 8, 16 })); + } test_cases.emplace_back(new test_solve_tri()); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 11, 11, 1, 1 }, { 5, 11, 1, 1 }));