diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 7a73e138e..7dff1d11b 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -1292,24 +1292,37 @@ static void ggml_compute_forward_sum_f32( } assert(ggml_is_scalar(dst)); - assert(src0->nb[0] == sizeof(float)); GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) - ggml_float sum = 0; - ggml_float row_sum = 0; + ggml_float sum = 0; - for (int64_t i03 = 0; i03 < ne03; i03++) { - for (int64_t i02 = 0; i02 < ne02; i02++) { - for (int64_t i01 = 0; i01 < ne01; i01++) { - ggml_vec_sum_f32_ggf(ne00, - &row_sum, - (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03)); - sum += row_sum; + if (nb00 == sizeof(float)) { + ggml_float row_sum = 0; + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = 0; i01 < ne01; i01++) { + ggml_vec_sum_f32_ggf(ne00, + &row_sum, + (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03)); + sum += row_sum; + } + } + } + } else { + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = 0; i01 < ne01; i01++) { + const char * row = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + for (int64_t i00 = 0; i00 < ne00; i00++) { + sum += (ggml_float) *(const float *) (row + i00*nb00); + } + } } } } + ((float *) dst->data)[0] = sum; } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index cb85321f5..dc30492fa 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -5897,7 +5897,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g //return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]); return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); case GGML_OP_SUM: - return ggml_is_contiguous_rows(op->src[0]); + return ggml_is_contiguously_allocated(op->src[0]); case GGML_OP_TOP_K: return op->src[0]->ne[0] <= INT_MAX; case GGML_OP_ARGSORT: diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index fa038bb5b..e603cbcb1 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -15872,6 +15872,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32; case GGML_OP_SUM: + return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguously_allocated(op->src[0]); case GGML_OP_SUM_ROWS: case GGML_OP_MEAN: return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 15e6c784b..77fbb4f47 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -4781,6 +4781,8 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type; break; case GGML_OP_SUM: + supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguously_allocated(src0); + break; case GGML_OP_SUM_ROWS: supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguous_rows(src0); break; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 011f93865..8fc90e037 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7804,24 +7804,43 @@ struct test_sum : public test_case { const std::array ne; const std::array permute; bool _use_permute; + const bool slice; std::string vars() override { std::string v = VARS_TO_STR2(type, ne); if (_use_permute) v += "," + VAR_TO_STR(permute); + if (slice) v += "," + VAR_TO_STR(slice); return v; } test_sum(ggml_type type = GGML_TYPE_F32, std::array ne = {10, 5, 4, 3}, - std::array permute = {0, 0, 0, 0}) + std::array permute = {0, 0, 0, 0}, + bool slice = false) : type(type), ne(ne), permute(permute), - _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {} + _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0), + slice(slice) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_set_param(a); + + // the backward pass of a sliced view scatters the gradient via GGML_OP_ACC, + // whose CPU implementation requires contiguous rows; a row-breaking permute + // (permute[0] != 0) on top of a slice violates that, so skip grad there + const bool grad_supported = !(slice && _use_permute && permute[0] != 0); + if (grad_supported) { + ggml_set_param(a); + } + ggml_set_name(a, "a"); + if (slice) { + a = ggml_view_4d(ctx, a, + ne[0], ne[1], ne[2] / 2, ne[3] - 1, + a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]); + ggml_set_name(a, "a_sliced"); + } + if (_use_permute) { a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]); ggml_set_name(a, "a_permuted"); @@ -11540,6 +11559,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 })); test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }, { 1, 0, 2, 3 })); // sum dst not-contiguous + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 10, 5, 4, 3 }, { 1, 0, 2, 3 })); // non-contiguous rows, 4D + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 11, 5, 6, 3 }, { 2, 0, 1, 3 })); // rotated dims, nb[0] != sizeof(float) + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 0, 0, 0}, true)); // sliced view (gaps) + test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {1, 0, 2, 3}, true)); // sliced + non-contiguous rows test_cases.emplace_back(new test_sum_rows()); test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false)); test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true));