Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ggml : add support for CPU f16->f16 GGML_OP_SET_ROWS (#25344)

Browse Source 
* ggml : add support for CPU f16->f16 GGML_OP_SET_ROWS

* ggml : add missing type checks in f16 GGML_OP_SET_ROWS

* ggml : merge ggml_compute_forward_set_rows_f32() and ggml_compute_forward_set_rows_f16() into ggml_compute_forward_set_rows_impl()

* chore : replace assert() with GGML_ASSERT()

---------

Co-authored-by: Stanisław Szymczyk <sszymczy@gmail.com>
This commit is contained in:
fairydreaming
2026-07-08 11:46:28 +08:00
committed by GitHub
co-authored by Stanisław Szymczyk
parent 931ca30bef
commit 68a521b591
3 changed files with 59 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-cpu/ops.cpp
+33 -8
View File
@@ -5025,8 +5025,8 @@ void ggml_compute_forward_get_rows(
//} //}
} }
template<typename idx_t> template<typename src_t, typename idx_t>
static void ggml_compute_forward_set_rows_f32( static void ggml_compute_forward_set_rows_impl(
const ggml_compute_params * params, const ggml_compute_params * params,
ggml_tensor * dst) { ggml_tensor * dst) {
@@ -5041,7 +5041,7 @@ static void ggml_compute_forward_set_rows_f32(
assert(ne0 == nc); assert(ne0 == nc);
assert(ne2 == ne02); assert(ne2 == ne02);
assert(ne3 == ne03); assert(ne3 == ne03);
assert(src0->type == GGML_TYPE_F32); GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
assert(ne02 % ne11 == 0); assert(ne02 % ne11 == 0);
assert(ne03 % ne12 == 0); assert(ne03 % ne12 == 0);
@@ -5055,6 +5055,8 @@ static void ggml_compute_forward_set_rows_f32(
const int64_t ir0 = dr*ith; const int64_t ir0 = dr*ith;
const int64_t ir1 = std::min(ir0 + dr, nr); const int64_t ir1 = std::min(ir0 + dr, nr);
const size_t rs = ggml_row_size(src0->type, nc);
ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float; ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float;
for (int64_t i03 = 0; i03 < ne03; ++i03) { for (int64_t i03 = 0; i03 < ne03; ++i03) {
@@ -5068,9 +5070,18 @@ static void ggml_compute_forward_set_rows_f32(
GGML_ASSERT(i1 >= 0 && i1 < ne1); GGML_ASSERT(i1 >= 0 && i1 < ne1);
from_float( if constexpr (std::is_same_v<src_t, float>) {
(const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), from_float(
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
} else if constexpr (std::is_same_v<src_t, ggml_fp16_t>) {
memcpy(
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3),
((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
rs);
} else {
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
}
} }
} }
} }
@@ -5087,13 +5098,27 @@ void ggml_compute_forward_set_rows(
case GGML_TYPE_F32: case GGML_TYPE_F32:
{ {
if (src1->type == GGML_TYPE_I64) { if (src1->type == GGML_TYPE_I64) {
ggml_compute_forward_set_rows_f32<int64_t>(params, dst); ggml_compute_forward_set_rows_impl<float, int64_t>(params, dst);
} else if (src1->type == GGML_TYPE_I32) { } else if (src1->type == GGML_TYPE_I32) {
ggml_compute_forward_set_rows_f32<int32_t>(params, dst); ggml_compute_forward_set_rows_impl<float, int32_t>(params, dst);
} else { } else {
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
} }
} break; } break;
case GGML_TYPE_F16:
{
if (dst->type == GGML_TYPE_F16) {
if (src1->type == GGML_TYPE_I64) {
ggml_compute_forward_set_rows_impl<ggml_fp16_t, int64_t>(params, dst);
} else if (src1->type == GGML_TYPE_I32) {
ggml_compute_forward_set_rows_impl<ggml_fp16_t, int32_t>(params, dst);
} else {
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
}
} else {
GGML_ABORT("dst->type = %d (%s) not supported with src0->type = %d (%s)", dst->type, ggml_type_name(dst->type), src0->type, ggml_type_name(src0->type));
}
} break;
default: default:
{ {
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type)); GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
ggml/src/ggml.c
+1 -1
View File
@@ -3926,7 +3926,7 @@ struct ggml_tensor * ggml_set_rows(
GGML_ASSERT(b->ne[2] % c->ne[1] == 0); GGML_ASSERT(b->ne[2] % c->ne[1] == 0);
GGML_ASSERT(b->ne[3] % c->ne[2] == 0); GGML_ASSERT(b->ne[3] % c->ne[2] == 0);
GGML_ASSERT(c->ne[3] == 1); GGML_ASSERT(c->ne[3] == 1);
GGML_ASSERT(b->type == GGML_TYPE_F32); GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16);
GGML_ASSERT(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32); GGML_ASSERT(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32);
GGML_ASSERT(ggml_is_contiguous_rows(a)); GGML_ASSERT(ggml_is_contiguous_rows(a));
tests/test-backend-ops.cpp
+25 -19
View File
@@ -2393,7 +2393,8 @@ static void init_set_rows_row_ids(ggml_tensor * t, int num_rows) {
// GGML_OP_SET_ROWS // GGML_OP_SET_ROWS
struct test_set_rows : public test_case { struct test_set_rows : public test_case {
const ggml_type type; const ggml_type type_src;
const ggml_type type_dst;
const ggml_type type_idx; const ggml_type type_idx;
const std::array<int64_t, 4> ne; const std::array<int64_t, 4> ne;
const std::array<int, 2> nr23; // broadcast only dims 2 and 3 const std::array<int, 2> nr23; // broadcast only dims 2 and 3
@@ -2401,21 +2402,22 @@ struct test_set_rows : public test_case {
const bool v; // view (non-contiguous src1) const bool v; // view (non-contiguous src1)
std::string vars() override { std::string vars() override {
return VARS_TO_STR6(type, type_idx, ne, nr23, r, v); return VARS_TO_STR7(type_src, type_dst, type_idx, ne, nr23, r, v);
} }
test_set_rows(ggml_type type, test_set_rows(ggml_type type_src,
ggml_type type_dst,
ggml_type type_idx, ggml_type type_idx,
std::array<int64_t, 4> ne, std::array<int64_t, 4> ne,
std::array<int, 2> nr23, std::array<int, 2> nr23,
int r, bool v = false) int r, bool v = false)
: type(type), type_idx(type_idx), ne(ne), nr23(nr23), r(r), v(v) {} : type_src(type_src), type_dst(type_dst), type_idx(type_idx), ne(ne), nr23(nr23), r(r), v(v) {}
ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * dst = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2]*nr23[0], ne[3]*nr23[1]); ggml_tensor * dst = ggml_new_tensor_4d(ctx, type_dst, ne[0], ne[1], ne[2]*nr23[0], ne[3]*nr23[1]);
ggml_set_name(dst, "dst"); ggml_set_name(dst, "dst");
ggml_tensor * src = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], r, ne[2]*nr23[0], ne[3]*nr23[1]); ggml_tensor * src = ggml_new_tensor_4d(ctx, type_src, ne[0], r, ne[2]*nr23[0], ne[3]*nr23[1]);
ggml_set_name(src, "src"); ggml_set_name(src, "src");
ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type_idx, r, ne[2], ne[3]); ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type_idx, r, ne[2], ne[3]);
@@ -2448,17 +2450,17 @@ struct test_set_rows : public test_case {
} }
double max_nmse_err() override { double max_nmse_err() override {
if (type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_IQ4_NL || if (type_dst == GGML_TYPE_Q4_0 || type_dst == GGML_TYPE_Q4_1 || type_dst == GGML_TYPE_IQ4_NL ||
type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1 || type == GGML_TYPE_Q8_0) { type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1 || type_dst == GGML_TYPE_Q8_0) {
// estimate what the max nmse error would be if one quantized value is // estimate what the max nmse error would be if one quantized value is
// off by one. The test values are distributed in [-1,1], so it'll be // off by one. The test values are distributed in [-1,1], so it'll be
// roughly (2.0 / 2^bits)^2, divided by the mean square value of the reference, // roughly (2.0 / 2^bits)^2, divided by the mean square value of the reference,
// which is roughly 0.25 times the number of elements. // which is roughly 0.25 times the number of elements.
double err_estimate = 1.0f/8.0f; double err_estimate = 1.0f/8.0f;
if (type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) { if (type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1) {
err_estimate /= 2.0f; err_estimate /= 2.0f;
} }
if (type == GGML_TYPE_Q8_0) { if (type_dst == GGML_TYPE_Q8_0) {
err_estimate /= 8.0f; err_estimate /= 8.0f;
} }
err_estimate *= err_estimate; err_estimate *= err_estimate;
@@ -2471,7 +2473,7 @@ struct test_set_rows : public test_case {
// See dicussion here: https://github.com/ggml-org/llama.cpp/pull/23760#issuecomment-4566312209 // See dicussion here: https://github.com/ggml-org/llama.cpp/pull/23760#issuecomment-4566312209
double max_nmse_err(ggml_backend_t backend) override { double max_nmse_err(ggml_backend_t backend) override {
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
if (type == GGML_TYPE_Q8_0 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) { if (type_dst == GGML_TYPE_Q8_0 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) {
return std::max(test_case::max_nmse_err(backend), 2e-7); return std::max(test_case::max_nmse_err(backend), 2e-7);
} }
return test_case::max_nmse_err(backend); return test_case::max_nmse_err(backend);
@@ -7873,24 +7875,28 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_I32, 256, 5, 4, 1, v)); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_I32, 256, 5, 4, 1, v));
} }
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false));
for (ggml_type type : all_types) { for (ggml_type type : all_types) {
for (int b : {1, 7}) { for (int b : {1, 7}) {
for (bool v : {false, true}) { for (bool v : {false, true}) {
test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 256, 5, b, 3 }, { 1, 1, }, 1, v)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 5, b, 3 }, { 1, 1, }, 1, v));
test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v));
test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v));
if (ggml_blck_size(type) == 1) { if (ggml_blck_size(type) == 1) {
test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v));
test_cases.emplace_back(new test_set_rows(type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v));
} }
} }
} }
} }
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, true));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, true));
for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION }) { for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION }) {
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {