ggml: support non-contiguous SUM (CPU, CUDA, Vulkan, WebGPU) (#28851)
* ggml: support non-contiguous SUM (CPU, CUDA) * tests: add non-contiguous SUM cases * tests: SUM over non-contiguous views * vulkan: support non-contiguous SUM * webgpu: support non-contiguous SUM
This commit is contained in:
1 parent
5aeac37a35
commit
0878f7c304
5 files changed
+53
-14
No files matched your search
+23
-10
@@ -1292,24 +1292,37 @@ static void ggml_compute_forward_sum_f32(
|
||||
}
|
||||
|
||||
assert(ggml_is_scalar(dst));
|
||||
assert(src0->nb[0] == sizeof(float));
|
||||
|
||||
GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
|
||||
GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
|
||||
|
||||
ggml_float sum = 0;
|
||||
ggml_float row_sum = 0;
|
||||
ggml_float sum = 0;
|
||||
|
||||
for (int64_t i03 = 0; i03 < ne03; i03++) {
|
||||
for (int64_t i02 = 0; i02 < ne02; i02++) {
|
||||
for (int64_t i01 = 0; i01 < ne01; i01++) {
|
||||
ggml_vec_sum_f32_ggf(ne00,
|
||||
&row_sum,
|
||||
(float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
|
||||
sum += row_sum;
|
||||
if (nb00 == sizeof(float)) {
|
||||
ggml_float row_sum = 0;
|
||||
for (int64_t i03 = 0; i03 < ne03; i03++) {
|
||||
for (int64_t i02 = 0; i02 < ne02; i02++) {
|
||||
for (int64_t i01 = 0; i01 < ne01; i01++) {
|
||||
ggml_vec_sum_f32_ggf(ne00,
|
||||
&row_sum,
|
||||
(float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
|
||||
sum += row_sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int64_t i03 = 0; i03 < ne03; i03++) {
|
||||
for (int64_t i02 = 0; i02 < ne02; i02++) {
|
||||
for (int64_t i01 = 0; i01 < ne01; i01++) {
|
||||
const char * row = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
|
||||
for (int64_t i00 = 0; i00 < ne00; i00++) {
|
||||
sum += (ggml_float) *(const float *) (row + i00*nb00);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
((float *) dst->data)[0] = sum;
|
||||
}
|
||||
|
||||
|
||||
@@ -5897,7 +5897,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
//return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]);
|
||||
return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
|
||||
case GGML_OP_SUM:
|
||||
return ggml_is_contiguous_rows(op->src[0]);
|
||||
return ggml_is_contiguously_allocated(op->src[0]);
|
||||
case GGML_OP_TOP_K:
|
||||
return op->src[0]->ne[0] <= INT_MAX;
|
||||
case GGML_OP_ARGSORT:
|
||||
|
||||
@@ -15872,6 +15872,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32
|
||||
&& ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_SUM:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguously_allocated(op->src[0]);
|
||||
case GGML_OP_SUM_ROWS:
|
||||
case GGML_OP_MEAN:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]);
|
||||
|
||||
@@ -4781,6 +4781,8 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type;
|
||||
break;
|
||||
case GGML_OP_SUM:
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguously_allocated(src0);
|
||||
break;
|
||||
case GGML_OP_SUM_ROWS:
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguous_rows(src0);
|
||||
break;
|
||||
|
||||
@@ -7804,24 +7804,43 @@ struct test_sum : public test_case {
|
||||
const std::array<int64_t, 4> ne;
|
||||
const std::array<int64_t, 4> permute;
|
||||
bool _use_permute;
|
||||
const bool slice;
|
||||
|
||||
std::string vars() override {
|
||||
std::string v = VARS_TO_STR2(type, ne);
|
||||
if (_use_permute) v += "," + VAR_TO_STR(permute);
|
||||
if (slice) v += "," + VAR_TO_STR(slice);
|
||||
return v;
|
||||
}
|
||||
|
||||
test_sum(ggml_type type = GGML_TYPE_F32,
|
||||
std::array<int64_t, 4> ne = {10, 5, 4, 3},
|
||||
std::array<int64_t, 4> permute = {0, 0, 0, 0})
|
||||
std::array<int64_t, 4> permute = {0, 0, 0, 0},
|
||||
bool slice = false)
|
||||
: type(type), ne(ne), permute(permute),
|
||||
_use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {}
|
||||
_use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0),
|
||||
slice(slice) {}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
|
||||
ggml_set_param(a);
|
||||
|
||||
// the backward pass of a sliced view scatters the gradient via GGML_OP_ACC,
|
||||
// whose CPU implementation requires contiguous rows; a row-breaking permute
|
||||
// (permute[0] != 0) on top of a slice violates that, so skip grad there
|
||||
const bool grad_supported = !(slice && _use_permute && permute[0] != 0);
|
||||
if (grad_supported) {
|
||||
ggml_set_param(a);
|
||||
}
|
||||
|
||||
ggml_set_name(a, "a");
|
||||
|
||||
if (slice) {
|
||||
a = ggml_view_4d(ctx, a,
|
||||
ne[0], ne[1], ne[2] / 2, ne[3] - 1,
|
||||
a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]);
|
||||
ggml_set_name(a, "a_sliced");
|
||||
}
|
||||
|
||||
if (_use_permute) {
|
||||
a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]);
|
||||
ggml_set_name(a, "a_permuted");
|
||||
@@ -11540,6 +11559,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 }));
|
||||
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }));
|
||||
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }, { 1, 0, 2, 3 })); // sum dst not-contiguous
|
||||
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 10, 5, 4, 3 }, { 1, 0, 2, 3 })); // non-contiguous rows, 4D
|
||||
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 11, 5, 6, 3 }, { 2, 0, 1, 3 })); // rotated dims, nb[0] != sizeof(float)
|
||||
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 0, 0, 0}, true)); // sliced view (gaps)
|
||||
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {1, 0, 2, 3}, true)); // sliced + non-contiguous rows
|
||||
test_cases.emplace_back(new test_sum_rows());
|
||||
test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false));
|
||||
test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true));
|
||||
|
||||
Reference in new issue
Block a user