diff --git a/ggml/src/ggml-zdnn/fattn.cpp b/ggml/src/ggml-zdnn/fattn.cpp index e877b7a75..e6a0c24ca 100644 --- a/ggml/src/ggml-zdnn/fattn.cpp +++ b/ggml/src/ggml-zdnn/fattn.cpp @@ -246,10 +246,4 @@ void ggml_zdnn_flash_attn_ext(ggml_backend_zdnn_context * ctx, ggml_tensor * dst memcpy((char *) dst->data + h*dst->nb[1] + iq*dst->nb[2], o_host + (h*n_q + iq)*d_v, d_v*sizeof(float)); } } - - // the result is written to dst->data directly, so the dst ztensor is stale - ggml_backend_zdnn_buffer * dst_extra = (ggml_backend_zdnn_buffer *) dst->extra; - if (dst_extra != nullptr) { - dst_extra->ztensor.is_transformed = false; - } } diff --git a/ggml/src/ggml-zdnn/ggml-zdnn.cpp b/ggml/src/ggml-zdnn/ggml-zdnn.cpp index 27415935b..533aefd9c 100644 --- a/ggml/src/ggml-zdnn/ggml-zdnn.cpp +++ b/ggml/src/ggml-zdnn/ggml-zdnn.cpp @@ -95,6 +95,10 @@ static int64_t ggml_zdnn_mul_mat_min_batch(ggml_type type, int32_t n_threads) { } } +static bool ggml_zdnn_is_tensor_own(const ggml_tensor * tensor) { + return tensor->buffer != nullptr && tensor->buffer->buft == ggml_backend_zdnn_buffer_type(); +} + static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_dev, const ggml_tensor * op) { switch (op->op) { case GGML_OP_NONE: @@ -127,14 +131,24 @@ static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_d return false; } + // tensors with no buffer yet get a zDNN buffer, computed or graph input weights and tensors in other buffers have no ztensor + if (weights->op != GGML_OP_NONE || (weights->flags & GGML_TENSOR_FLAG_INPUT) || + (weights->buffer != nullptr && !ggml_zdnn_is_tensor_own(weights)) || + (op->buffer != nullptr && !ggml_zdnn_is_tensor_own(op))) { + return false; + } + + // training writes allocated weights through ->data, test-backend-ops sets PARAM before it allocates and uses set_tensor + if ((weights->flags & GGML_TENSOR_FLAG_PARAM) && weights->buffer != nullptr) { + return false; + } + switch (weights->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_BF16: - // the inputs ztensor is used, and views have none - return inputs->view_src == nullptr; + return true; case GGML_TYPE_Q8_0: - // the inputs are quantized from their data, so views work as well return ctx_dev->has_parmblkformat_1 && inputs->type == GGML_TYPE_F32; default: return false; @@ -274,6 +288,11 @@ static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer return GGML_STATUS_SUCCESS; } + // ops read other compute tensors through ->data + if (tensor->op != GGML_OP_NONE && tensor->op != GGML_OP_MUL_MAT) { + return GGML_STATUS_SUCCESS; + } + ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context; const int64_t tsize = ggml_nbytes(tensor); @@ -324,23 +343,27 @@ static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer GGML_UNUSED(buffer_idx); } +static void ggml_zdnn_reset_tensor(const ggml_tensor * tensor) { + const ggml_tensor * src = tensor->view_src != nullptr ? tensor->view_src : tensor; + + ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)src->extra; + if (extra != nullptr) { + zdnn_reset_ztensor(&extra->ztensor); + } +} + static void ggml_backend_zdnn_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { memset((char *)tensor->data + offset, value, size); + ggml_zdnn_reset_tensor(tensor); + GGML_UNUSED(buffer); } static void ggml_backend_zdnn_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { memcpy((char *)tensor->data + offset, data, size); - // views share the ztensor of their source, which is now stale, and empty tensors have none - if (tensor->view_src != nullptr) { - ggml_backend_zdnn_buffer * src_extra = (ggml_backend_zdnn_buffer *)tensor->view_src->extra; - if (src_extra != nullptr && src_extra->ztensor.is_transformed) { - zdnn_reset_ztensor(&src_extra->ztensor); - } - return; - } + ggml_zdnn_reset_tensor(tensor); ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)tensor->extra; if (extra == nullptr) { @@ -355,12 +378,10 @@ static void ggml_backend_zdnn_buffer_set_tensor(ggml_backend_buffer_t buffer, gg // Fixes the LLAMA_SET_ROWS bug // see: https://github.com/ggml-org/llama.cpp/issues/15414 - // only weights are transformed here, others on first use, as not every op reads the ztensor and some data (masks) is not finite - if (tensor->buffer->usage != GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - zdnn_reset_ztensor(&extra->ztensor); - return; + // only complete writes to weights are transformed here, others on first use, as not every op reads the ztensor and some data (masks) is not finite + if (tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && size == ggml_nbytes(tensor)) { + ggml_zdnn_load_tensor(extra->ztensor, tensor->data); } - if (extra->ztensor.is_transformed == false) ggml_zdnn_load_tensor(extra->ztensor, tensor->data); GGML_UNUSED(buffer); } @@ -375,6 +396,17 @@ static void ggml_backend_zdnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context; memset(ctx->all_data, value, ctx->all_size); + + for (const auto & buf_ptr : ctx->buffers) { + zdnn_reset_ztensor(&buf_ptr->ztensor); + } + + // the bias has its own data, and no op transforms it on first use + for (const auto & buf_ptr : ctx->buffers) { + if (buf_ptr->extra != nullptr) { + ggml_zdnn_load_tensor(buf_ptr->extra->ztensor, buf_ptr->extra->data); + } + } } static void ggml_backend_zdnn_buffer_reset(ggml_backend_buffer_t buffer) { @@ -620,8 +652,7 @@ static bool ggml_backend_zdnn_device_supports_op(ggml_backend_dev_t dev, const g } static bool ggml_backend_zdnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - return - buft->iface.get_name == ggml_backend_zdnn_buffer_type_get_name; + return ggml_backend_buft_is_host(buft); GGML_UNUSED(dev); } diff --git a/ggml/src/ggml-zdnn/mmf.cpp b/ggml/src/ggml-zdnn/mmf.cpp index 0551fb88b..acd443e5b 100644 --- a/ggml/src/ggml-zdnn/mmf.cpp +++ b/ggml/src/ggml-zdnn/mmf.cpp @@ -3,7 +3,7 @@ #include "utils.hpp" void ggml_zdnn_mul_mat_f( - const ggml_backend_zdnn_context * ctx, + ggml_backend_zdnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -31,10 +31,14 @@ void ggml_zdnn_mul_mat_f( ggml_tensor * output = dst; ggml_backend_zdnn_buffer * weights_extra = (ggml_backend_zdnn_buffer *)weights->extra; - ggml_backend_zdnn_buffer * inputs_extra = (ggml_backend_zdnn_buffer *)inputs->extra; ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra; ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra; + zdnn_tensor_desc inputs_pre_tfm_desc; + zdnn_tensor_desc inputs_tfm_desc; + zdnn_ztensor inputs_ztensor; + ggml_zdnn_pool_alloc inputs_ztensor_alloc; + const int64_t weights_rows = ne01; const int64_t weights_cols = ne00; const int64_t inputs_rows = ne11; @@ -53,37 +57,29 @@ void ggml_zdnn_mul_mat_f( // weights_extra->pre_tfm_desc.dim3, // weights_extra->pre_tfm_desc.dim4); - // GGML_LOG_INFO("%s: tensor '%s' tensor dimensions: [%ld, %ld, %ld, %ld] pre_tfm_desc dimensions: [%ld, %ld, %ld, %ld]\n", - // __func__, inputs_extra->name, - // inputs->ne[3], inputs->ne[2], inputs->ne[1], inputs->ne[0], - // inputs_extra->pre_tfm_desc.dim1, - // inputs_extra->pre_tfm_desc.dim2, - // inputs_extra->pre_tfm_desc.dim3, - // inputs_extra->pre_tfm_desc.dim4); - GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[0] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[0]"); GGML_ASSERT(weights_extra->pre_tfm_desc.dim2 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim2 must match weights->ne[1]"); - GGML_ASSERT(inputs_extra->pre_tfm_desc.dim1 == inputs->ne[0] && "inputs_extra->pre_tfm_desc.dim1 must match inputs->ne[0]"); - GGML_ASSERT(inputs_extra->pre_tfm_desc.dim2 == inputs->ne[1] && "inputs_extra->pre_tfm_desc.dim2 must match inputs->ne[1]"); - // compute tensors are transformed on first use + // weights read straight into a host buffer skip set_tensor, so transform them on first use if (!weights_extra->ztensor.is_transformed) { ggml_zdnn_load_tensor(weights_extra->ztensor, weights->data); } - if (!inputs_extra->ztensor.is_transformed) { - ggml_zdnn_load_tensor(inputs_extra->ztensor, inputs->data); + + // the inputs can be in any host buffer or be a view, so transform them from their data on every call + { + zdnn_init_pre_transformed_desc(ZDNN_2D, ggml_zdnn_type_mapping(inputs->type), &inputs_pre_tfm_desc, inputs_rows, inputs_cols); + ZDNN_CHECK(zdnn_generate_transformed_desc(&inputs_pre_tfm_desc, &inputs_tfm_desc)); + ggml_zdnn_init_scratch_ztensor_dlf16(ctx, &inputs_pre_tfm_desc, &inputs_tfm_desc, &inputs_ztensor, inputs_ztensor_alloc); + ggml_zdnn_load_tensor(inputs_ztensor, inputs->data); } - ZDNN_CHECK(zdnn_matmul_transpose_op(&inputs_extra->ztensor, &weights_extra->ztensor, &bias_extra->ztensor, + ZDNN_CHECK(zdnn_matmul_transpose_op(&inputs_ztensor, &weights_extra->ztensor, &bias_extra->ztensor, false, true, MATMUL_OP_ADDITION, &output_extra->ztensor)); // TODO: Remove in the future as we are currently DLF16 -> FP32 then in the next op, FP32 -> DLF16 again. Inefficient. ZDNN_CHECK(zdnn_transform_origtensor(&output_extra->ztensor, output->data)); - GGML_UNUSED(ctx); GGML_UNUSED(weights_rows); GGML_UNUSED(weights_cols); - GGML_UNUSED(inputs_rows); - GGML_UNUSED(inputs_cols); GGML_UNUSED(output_rows); GGML_UNUSED(output_cols); } diff --git a/ggml/src/ggml-zdnn/mmf.hpp b/ggml/src/ggml-zdnn/mmf.hpp index a12f1b8f8..afc02bc0e 100644 --- a/ggml/src/ggml-zdnn/mmf.hpp +++ b/ggml/src/ggml-zdnn/mmf.hpp @@ -4,7 +4,7 @@ #include "common.hpp" void ggml_zdnn_mul_mat_f( - const ggml_backend_zdnn_context * ctx, + ggml_backend_zdnn_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); diff --git a/ggml/src/ggml-zdnn/mmq.cpp b/ggml/src/ggml-zdnn/mmq.cpp index 0c4472845..285479b41 100644 --- a/ggml/src/ggml-zdnn/mmq.cpp +++ b/ggml/src/ggml-zdnn/mmq.cpp @@ -188,7 +188,6 @@ void ggml_zdnn_mul_mat_q( GGML_ASSERT(inputs->type == GGML_TYPE_F32); ggml_backend_zdnn_buffer * weights_extra = (ggml_backend_zdnn_buffer *)weights->extra; - ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra; // weights read straight into a host buffer skip set_tensor, so transform them on first use if (!weights_extra->ztensor.is_transformed) { @@ -202,13 +201,11 @@ void ggml_zdnn_mul_mat_q( GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[1]"); ggml_zdnn_mul_mat_q_grouped(ctx, n_threads, inputs, weights_extra, output); - - // the result is written to output->data directly, so the output ztensor is stale - output_extra->ztensor.is_transformed = false; return; } - ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra; + ggml_backend_zdnn_buffer * output_extra = (ggml_backend_zdnn_buffer *)output->extra; + ggml_backend_zdnn_buffer * bias_extra = (ggml_backend_zdnn_buffer *)output_extra->extra; GGML_ASSERT(weights_extra->pre_tfm_desc.dim2 == weights->ne[0] && "weights_extra->pre_tfm_desc.dim2 must match weights->ne[0]"); GGML_ASSERT(weights_extra->pre_tfm_desc.dim1 == weights->ne[1] && "weights_extra->pre_tfm_desc.dim1 must match weights->ne[1]"); @@ -271,9 +268,6 @@ void ggml_zdnn_mul_mat_q( } }); - // the result is written to output->data directly, so the output ztensor is stale - output_extra->ztensor.is_transformed = false; - const float * weights_scales = weights_extra->scales.data(); ggml_zdnn_parallel_for(n_threads, ne1, [&](int64_t i1_start, int64_t i1_end) { for (int64_t i1 = i1_start; i1 < i1_end; i1++) {