diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 3c2e6734b..e6ac1af1b 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -1477,6 +1477,16 @@ extern "C" { enum ggml_prec prec, int idx); + // [TAG_GGML_PREC] + // get the accumulator / src[idx] precision set by the functions above + // return GGML_PREC_UNDEFINED when the op does not carry the requested precision + GGML_API enum ggml_prec ggml_prec_get_acc( + const struct ggml_tensor * a); + + GGML_API enum ggml_prec ggml_prec_get_src( + const struct ggml_tensor * a, + int idx); + // A: k columns, n rows => [ne03, ne02, n, k] // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index e603cbcb1..0ea4e063b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -6005,20 +6005,49 @@ void ggml_vk_matmul( ctx->prealloc_split_k_need_sync = true; } -static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { +static bool ggml_vk_src1_prec_allows_q8(const ggml_tensor * dst) { + switch (ggml_prec_get_src(dst, 1)) { + case GGML_PREC_UNDEFINED: + case GGML_PREC_Q8: + case GGML_PREC_Q4: + return true; + case GGML_PREC_F32: + case GGML_PREC_BF16: + case GGML_PREC_F16: + return false; + } + return false; +} + +static bool ggml_vk_prec_allows_f16acc(ggml_prec prec) { + switch (prec) { + case GGML_PREC_F32: + case GGML_PREC_BF16: + return false; + case GGML_PREC_UNDEFINED: + case GGML_PREC_F16: + case GGML_PREC_Q8: + case GGML_PREC_Q4: + return true; + } + return false; +} + +static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec acc_prec) { if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false; if (src1_type == GGML_TYPE_Q8_1) return false; + const bool prec_allows_f16 = ggml_vk_prec_allows_f16acc(acc_prec); if (src0_type == GGML_TYPE_F16) { - return prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support); + return prec_allows_f16 && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support); } // quant types if (ctx->device->coopmat2) { - return prec == GGML_PREC_DEFAULT; + return prec_allows_f16; } if (ctx->device->coopmat_support) { - return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT; + return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec_allows_f16; } - return ctx->device->fp16 && prec == GGML_PREC_DEFAULT; + return ctx->device->fp16 && prec_allows_f16; } static const std::vector* ggml_vk_get_mul_mat_mat_pipeline_map( @@ -6386,26 +6415,26 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub const bool is_coopmat = ctx->device->coopmat_support || ctx->device->coopmat2; const bool can_quantize_y = (ctx->device->integer_dot_product || ctx->device->coopmat_int_support) && - src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0; + src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst); // coopmat shaders are fast enough that memory bandwidth reduction from f16 activations helps const bool prefer_f16_b = is_coopmat && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32; const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type, - (ggml_prec)dst->op_params[0], false, can_quantize_y, prefer_f16_b); + ggml_prec_get_acc(dst), false, can_quantize_y, prefer_f16_b); const bool quantize_y = y_kernel_type == GGML_TYPE_Q8_1; const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32; const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type); const std::vector* mmp_map = - ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, (ggml_prec)dst->op_params[0]); + ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst)); const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; const bool qy_needs_dequant = y_needs_reformat; if (qx_needs_dequant) { // dequant src0 to f16 - mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst)); } GGML_ASSERT(mmp_map != nullptr); @@ -6736,7 +6765,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& const ggml_type y_kernel_type = widen_bf16 ? GGML_TYPE_F32 : src1->type; const bool f16_f32_kernel = y_kernel_type == GGML_TYPE_F32; - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type); + bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type); vk_pipeline to_fp16_vk_0 = nullptr; vk_pipeline to_fp16_vk_1 = nullptr; @@ -7435,26 +7464,26 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& const bool y_decode_vector_staging = false; #endif const bool can_quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && - ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0; + ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst); // Intel coopmat1 matrix cores prefer f16 activations for quantized weights const bool prefer_f16_b = ctx->device->coopmat_support && !ctx->device->coopmat2 && ctx->device->vendor_id == VK_VENDOR_ID_INTEL && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32; const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type, - (ggml_prec)dst->op_params[0], true, can_quantize_y, prefer_f16_b); + ggml_prec_get_acc(dst), true, can_quantize_y, prefer_f16_b); const bool quantize_y = y_kernel_type == GGML_TYPE_Q8_1; const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32; const std::vector* mmp_map = - ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, (ggml_prec)dst->op_params[0], true); + ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst), true); const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; if (qx_needs_dequant) { // dequant src0 to f16 - mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, (ggml_prec)dst->op_params[0], true); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst), true); } GGML_ASSERT(mmp_map != nullptr); @@ -7749,7 +7778,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte const bool y_non_contig = !ggml_vk_dim01_contiguous(src1); const bool f16_f32_kernel = src1->type == GGML_TYPE_F32; - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type); + bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type); vk_pipeline to_fp16_vk_0 = nullptr; vk_pipeline to_fp16_vk_1 = nullptr; @@ -8128,7 +8157,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const uint32_t workgroups_y = (uint32_t)neq2; uint32_t workgroups_z = (uint32_t)neq3; - const bool f32acc = !ctx->device->fp16 || dst->op_params[3] == GGML_PREC_F32 || k->type == GGML_TYPE_BF16; + const bool f32acc = !ctx->device->fp16 || !ggml_vk_prec_allows_f16acc(ggml_prec_get_acc(dst)) || k->type == GGML_TYPE_BF16; // dequant K/V once into an f16 scratch, reordered KV layout so FA can read without a stride auto is_dense_kv_cache = [](const ggml_tensor * t) { @@ -15516,9 +15545,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm // If there's not enough shared memory for row_ids and the result tile, fallback to CPU return false; } - if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { - return false; - } + } + if (ggml_prec_get_src(op, 1) == GGML_PREC_F32) { + return false; } switch (src0_type) { case GGML_TYPE_F32: diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 0b2adf6e4..f33e3f2e4 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -3348,6 +3348,36 @@ bool ggml_prec_set_src( return true; } +enum ggml_prec ggml_prec_get_acc( + const struct ggml_tensor * a) { + switch (a->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + return (enum ggml_prec) ggml_get_op_params_i32(a, 0); + case GGML_OP_FLASH_ATTN_EXT: + return (enum ggml_prec) ggml_get_op_params_i32(a, 3); + default: + return GGML_PREC_UNDEFINED; + }; +} + +enum ggml_prec ggml_prec_get_src( + const struct ggml_tensor * a, + int idx) { + GGML_ASSERT(idx >= 0 && idx < GGML_MAX_SRC); + + switch (a->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + if (idx != 1) { + return GGML_PREC_UNDEFINED; + } + return (enum ggml_prec) ggml_get_op_params_i32(a, 2 + idx); + default: + return GGML_PREC_UNDEFINED; + }; +} + // ggml_mul_mat static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {