cpu: add F16 SWIGLU_OAI reference implementation and validation (#29200)
* cpu: add F16 SWIGLU_OAI reference implementation and F16 activation op test coverage
Adds ggml_compute_forward_swiglu_oai_f16() to the CPU backend, mirroring
the existing F32 path, and extends test-backend-ops with F16 test cases
for SWIGLU_OAI so downstream backends(e.g. Hexagon) can be CPU-diffed
for correctness.
* ggml-openvino: fix f16 SWIGLU_OAI type mismatch in alpha/one constants
translate_glu_swiglu_oai creates alpha_const and one as f32 constants
unconditionally, so f16 inputs hit a type mismatch in Swish/Add on GPU
("Swish inputs must have the same type but they are: f16 and f32").
translate_glu_swiglu_clamp already upcasts to f32 and converts back at
the end; apply the same pattern here.
---------
Co-authored-by: ravi9 <ravi.panchumarthy@intel.com>
This commit is contained in:
1 parent
6e66cb5d4e
commit
5a638aedb7
4 files changed
+105
-5
No files matched your search
@@ -3548,6 +3548,73 @@ static void ggml_compute_forward_swiglu_oai_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_swiglu_oai_f16(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
char * src0_d = (char *) src0->data;
|
||||
char * src1_d = (char *) (src1 ? src1->data : src0->data);
|
||||
const size_t src0_o = src0->nb[1];
|
||||
const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
|
||||
|
||||
GGML_ASSERT(ggml_is_contiguous_1(src0));
|
||||
GGML_ASSERT(ggml_is_contiguous_1(dst));
|
||||
|
||||
if (src1) {
|
||||
GGML_ASSERT(ggml_is_contiguous_1(src1));
|
||||
GGML_ASSERT(src0->type == src1->type);
|
||||
}
|
||||
|
||||
const int ith = params->ith;
|
||||
const int nth = params->nth;
|
||||
|
||||
const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
|
||||
const int nr = ggml_nrows(src0);
|
||||
|
||||
GGML_ASSERT(dst->ne[0] == nc);
|
||||
GGML_ASSERT(ggml_nrows(dst) == nr);
|
||||
|
||||
const int32_t swapped = ggml_get_op_params_i32(dst, 1);
|
||||
const float alpha = ggml_get_op_params_f32(dst, 2);
|
||||
const float limit = ggml_get_op_params_f32(dst, 3);
|
||||
|
||||
// rows per thread
|
||||
const int dr = (nr + nth - 1)/nth;
|
||||
|
||||
// row range for this thread
|
||||
const int ir0 = dr*ith;
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int i1 = ir0; i1 < ir1; i1++) {
|
||||
ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
|
||||
ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
|
||||
ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1]));
|
||||
|
||||
if (!src1) {
|
||||
src0_p += swapped ? nc : 0;
|
||||
src1_p += swapped ? 0 : nc;
|
||||
}
|
||||
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit);
|
||||
const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit);
|
||||
const float out_glu = x / (1.f + expf(alpha * (-x)));
|
||||
dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f));
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]);
|
||||
GGML_UNUSED(x);
|
||||
assert(!isnan(x));
|
||||
assert(!isinf(x));
|
||||
}
|
||||
#endif // NDEBUG
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_swiglu_oai(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
@@ -3559,6 +3626,10 @@ static void ggml_compute_forward_swiglu_oai(
|
||||
{
|
||||
ggml_compute_forward_swiglu_oai_f32(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_swiglu_oai_f16(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
GGML_ABORT("fatal error");
|
||||
|
||||
@@ -5474,6 +5474,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
(ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_CLAMP)) {
|
||||
return false;
|
||||
}
|
||||
if (op->src[0]->type == GGML_TYPE_F16 && ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI) {
|
||||
return false;
|
||||
}
|
||||
return ggml_is_contiguous_1(op->src[0]);
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -75,6 +75,15 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {
|
||||
const float alpha = reinterpret_cast<const float *>(params)[2];
|
||||
const float limit = reinterpret_cast<const float *>(params)[3];
|
||||
|
||||
// Compute in f32: alpha_const/one are f32, and Swish/Add need matching input types.
|
||||
auto output_type = context.get_output_type();
|
||||
if (src0.get_element_type() != ov::element::f32) {
|
||||
src0 = std::make_shared<ov::op::v0::Convert>(src0, ov::element::f32);
|
||||
}
|
||||
if (src1.get_element_type() != ov::element::f32) {
|
||||
src1 = std::make_shared<ov::op::v0::Convert>(src1, ov::element::f32);
|
||||
}
|
||||
|
||||
auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit);
|
||||
auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha});
|
||||
auto out_glu = std::make_shared<ov::op::v4::Swish>(gate, alpha_const);
|
||||
@@ -82,7 +91,10 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {
|
||||
auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit);
|
||||
auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f});
|
||||
auto up_plus_one = std::make_shared<ov::op::v1::Add>(up, one);
|
||||
auto res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);
|
||||
ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);
|
||||
if (res.get_element_type() != output_type) {
|
||||
res = std::make_shared<ov::op::v0::Convert>(res, output_type);
|
||||
}
|
||||
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
@@ -2308,6 +2308,12 @@ struct test_swiglu_oai : public test_case {
|
||||
init_tensor_uniform(t, -150.f, 150.f);
|
||||
}
|
||||
}
|
||||
|
||||
double max_nmse_err() override {
|
||||
// F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the
|
||||
// natural roundoff drift from the exp()/clamp() chain computed in F16.
|
||||
return type == GGML_TYPE_F16 ? 5e-6 : 1e-7;
|
||||
}
|
||||
};
|
||||
|
||||
struct test_swiglu_clamp : public test_case {
|
||||
@@ -2365,6 +2371,12 @@ struct test_swiglu_clamp : public test_case {
|
||||
init_tensor_uniform(t, -150.f, 150.f);
|
||||
}
|
||||
}
|
||||
|
||||
double max_nmse_err() override {
|
||||
// F16 epsilon ~ 9.7e-4: relax tolerance vs the F32 default to absorb the
|
||||
// natural roundoff drift from the sigmoid()/clamp() chain computed in F16.
|
||||
return type == GGML_TYPE_F16 ? 5e-6 : 1e-7;
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_GET_ROWS
|
||||
@@ -9747,10 +9759,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
|
||||
for (int v : {0, 1}) {
|
||||
for (float alpha : {.5f, 1.702f}) {
|
||||
for (float limit : {2.0f, 7.0f}) {
|
||||
test_cases.emplace_back(new test_swiglu_oai(GGML_TYPE_F32, { 128, 2, 2, 2 }, v, alpha, limit));
|
||||
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
|
||||
for (int v : {0, 1}) {
|
||||
for (float alpha : {.5f, 1.702f}) {
|
||||
for (float limit : {2.0f, 7.0f}) {
|
||||
test_cases.emplace_back(new test_swiglu_oai(type, { 128, 2, 2, 2 }, v, alpha, limit));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user