diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index c7e7c47a7..3b39683d5 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -3548,6 +3548,73 @@ static void ggml_compute_forward_swiglu_oai_f32( } } +static void ggml_compute_forward_swiglu_oai_f16( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + char * src0_d = (char *) src0->data; + char * src1_d = (char *) (src1 ? src1->data : src0->data); + const size_t src0_o = src0->nb[1]; + const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(ggml_is_contiguous_1(dst)); + + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src0->type == src1->type); + } + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + const int nr = ggml_nrows(src0); + + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == nr); + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + const float alpha = ggml_get_op_params_f32(dst, 2); + const float limit = ggml_get_op_params_f32(dst, 3); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o); + ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o); + ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])); + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + for (int k = 0; k < nc; k++) { + const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit); + const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit); + const float out_glu = x / (1.f + expf(alpha * (-x))); + dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f)); + } + +#ifndef NDEBUG + for (int k = 0; k < nc; k++) { + const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]); + GGML_UNUSED(x); + assert(!isnan(x)); + assert(!isinf(x)); + } +#endif // NDEBUG + } +} + static void ggml_compute_forward_swiglu_oai( const ggml_compute_params * params, ggml_tensor * dst) { @@ -3559,6 +3626,10 @@ static void ggml_compute_forward_swiglu_oai( { ggml_compute_forward_swiglu_oai_f32(params, dst); } break; + case GGML_TYPE_F16: + { + ggml_compute_forward_swiglu_oai_f16(params, dst); + } break; default: { GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 9fe618f85..7c45d0d1f 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -5474,6 +5474,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g (ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_CLAMP)) { return false; } + if (op->src[0]->type == GGML_TYPE_F16 && ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI) { + return false; + } return ggml_is_contiguous_1(op->src[0]); default: return false; diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp index 7eea81d96..2c206a597 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -75,6 +75,15 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) { const float alpha = reinterpret_cast(params)[2]; const float limit = reinterpret_cast(params)[3]; + // Compute in f32: alpha_const/one are f32, and Swish/Add need matching input types. + auto output_type = context.get_output_type(); + if (src0.get_element_type() != ov::element::f32) { + src0 = std::make_shared(src0, ov::element::f32); + } + if (src1.get_element_type() != ov::element::f32) { + src1 = std::make_shared(src1, ov::element::f32); + } + auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha}); auto out_glu = std::make_shared(gate, alpha_const); @@ -82,7 +91,10 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) { auto up = std::make_shared(src1, -limit, limit); auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f}); auto up_plus_one = std::make_shared(up, one); - auto res = std::make_shared(out_glu, up_plus_one); + ov::Output res = std::make_shared(out_glu, up_plus_one); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e61823084..93b30d8d2 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2308,6 +2308,12 @@ struct test_swiglu_oai : public test_case { init_tensor_uniform(t, -150.f, 150.f); } } + + double max_nmse_err() override { + // F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the + // natural roundoff drift from the exp()/clamp() chain computed in F16. + return type == GGML_TYPE_F16 ? 5e-6 : 1e-7; + } }; struct test_swiglu_clamp : public test_case { @@ -2365,6 +2371,12 @@ struct test_swiglu_clamp : public test_case { init_tensor_uniform(t, -150.f, 150.f); } } + + double max_nmse_err() override { + // F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the + // natural roundoff drift from the sigmoid()/clamp() chain computed in F16. + return type == GGML_TYPE_F16 ? 5e-6 : 1e-7; + } }; // GGML_OP_GET_ROWS @@ -9747,10 +9759,12 @@ static std::vector> make_test_cases_eval() { } } - for (int v : {0, 1}) { - for (float alpha : {.5f, 1.702f}) { - for (float limit : {2.0f, 7.0f}) { - test_cases.emplace_back(new test_swiglu_oai(GGML_TYPE_F32, { 128, 2, 2, 2 }, v, alpha, limit)); + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (int v : {0, 1}) { + for (float alpha : {.5f, 1.702f}) { + for (float limit : {2.0f, 7.0f}) { + test_cases.emplace_back(new test_swiglu_oai(type, { 128, 2, 2, 2 }, v, alpha, limit)); + } } } }