vulkan: ggml src1 precision support (#30656)
* vulkan: update accumulator precision selection * handle fp32 overflows through scaling * handle overflows in copy-to-fp16 activations * add missing cases * simplify code * add tests * add ggml prec get helpers, use in Vulkan backend * clean up comments * revert comment edit * trim tests * also handle fp32 src0, fix test scaling src0 values too * don't scale if disabled * remove fp16 scaling * add check to mul_mat * fix inconsistency * rename to acc_prec
This commit is contained in:
1 parent
1ddffe9f70
commit
8e5a58df87
3 files changed
+88
-19
No files matched your search
@@ -1477,6 +1477,16 @@ extern "C" {
|
||||
enum ggml_prec prec,
|
||||
int idx);
|
||||
|
||||
// [TAG_GGML_PREC]
|
||||
// get the accumulator / src[idx] precision set by the functions above
|
||||
// return GGML_PREC_UNDEFINED when the op does not carry the requested precision
|
||||
GGML_API enum ggml_prec ggml_prec_get_acc(
|
||||
const struct ggml_tensor * a);
|
||||
|
||||
GGML_API enum ggml_prec ggml_prec_get_src(
|
||||
const struct ggml_tensor * a,
|
||||
int idx);
|
||||
|
||||
// A: k columns, n rows => [ne03, ne02, n, k]
|
||||
// B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
|
||||
// result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
|
||||
|
||||
@@ -6005,20 +6005,49 @@ void ggml_vk_matmul(
|
||||
ctx->prealloc_split_k_need_sync = true;
|
||||
}
|
||||
|
||||
static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) {
|
||||
static bool ggml_vk_src1_prec_allows_q8(const ggml_tensor * dst) {
|
||||
switch (ggml_prec_get_src(dst, 1)) {
|
||||
case GGML_PREC_UNDEFINED:
|
||||
case GGML_PREC_Q8:
|
||||
case GGML_PREC_Q4:
|
||||
return true;
|
||||
case GGML_PREC_F32:
|
||||
case GGML_PREC_BF16:
|
||||
case GGML_PREC_F16:
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool ggml_vk_prec_allows_f16acc(ggml_prec prec) {
|
||||
switch (prec) {
|
||||
case GGML_PREC_F32:
|
||||
case GGML_PREC_BF16:
|
||||
return false;
|
||||
case GGML_PREC_UNDEFINED:
|
||||
case GGML_PREC_F16:
|
||||
case GGML_PREC_Q8:
|
||||
case GGML_PREC_Q4:
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec acc_prec) {
|
||||
if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false;
|
||||
if (src1_type == GGML_TYPE_Q8_1) return false;
|
||||
const bool prec_allows_f16 = ggml_vk_prec_allows_f16acc(acc_prec);
|
||||
if (src0_type == GGML_TYPE_F16) {
|
||||
return prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support);
|
||||
return prec_allows_f16 && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support);
|
||||
}
|
||||
// quant types
|
||||
if (ctx->device->coopmat2) {
|
||||
return prec == GGML_PREC_DEFAULT;
|
||||
return prec_allows_f16;
|
||||
}
|
||||
if (ctx->device->coopmat_support) {
|
||||
return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT;
|
||||
return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec_allows_f16;
|
||||
}
|
||||
return ctx->device->fp16 && prec == GGML_PREC_DEFAULT;
|
||||
return ctx->device->fp16 && prec_allows_f16;
|
||||
}
|
||||
|
||||
static const std::vector<vk_matmul_pipeline_pair>* ggml_vk_get_mul_mat_mat_pipeline_map(
|
||||
@@ -6386,26 +6415,26 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
|
||||
const bool is_coopmat = ctx->device->coopmat_support || ctx->device->coopmat2;
|
||||
const bool can_quantize_y = (ctx->device->integer_dot_product || ctx->device->coopmat_int_support) &&
|
||||
src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0;
|
||||
src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst);
|
||||
// coopmat shaders are fast enough that memory bandwidth reduction from f16 activations helps
|
||||
const bool prefer_f16_b = is_coopmat && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32;
|
||||
|
||||
const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type,
|
||||
(ggml_prec)dst->op_params[0], false, can_quantize_y, prefer_f16_b);
|
||||
ggml_prec_get_acc(dst), false, can_quantize_y, prefer_f16_b);
|
||||
|
||||
const bool quantize_y = y_kernel_type == GGML_TYPE_Q8_1;
|
||||
const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32;
|
||||
const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type);
|
||||
|
||||
const std::vector<vk_matmul_pipeline_pair>* mmp_map =
|
||||
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, (ggml_prec)dst->op_params[0]);
|
||||
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst));
|
||||
|
||||
const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;
|
||||
const bool qy_needs_dequant = y_needs_reformat;
|
||||
|
||||
if (qx_needs_dequant) {
|
||||
// dequant src0 to f16
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, (ggml_prec)dst->op_params[0]);
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst));
|
||||
}
|
||||
|
||||
GGML_ASSERT(mmp_map != nullptr);
|
||||
@@ -6736,7 +6765,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
|
||||
const ggml_type y_kernel_type = widen_bf16 ? GGML_TYPE_F32 : src1->type;
|
||||
const bool f16_f32_kernel = y_kernel_type == GGML_TYPE_F32;
|
||||
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
|
||||
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
|
||||
|
||||
vk_pipeline to_fp16_vk_0 = nullptr;
|
||||
vk_pipeline to_fp16_vk_1 = nullptr;
|
||||
@@ -7435,26 +7464,26 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
const bool y_decode_vector_staging = false;
|
||||
#endif
|
||||
const bool can_quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 &&
|
||||
ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0;
|
||||
ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst);
|
||||
// Intel coopmat1 matrix cores prefer f16 activations for quantized weights
|
||||
const bool prefer_f16_b = ctx->device->coopmat_support && !ctx->device->coopmat2 &&
|
||||
ctx->device->vendor_id == VK_VENDOR_ID_INTEL &&
|
||||
ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32;
|
||||
|
||||
const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type,
|
||||
(ggml_prec)dst->op_params[0], true, can_quantize_y, prefer_f16_b);
|
||||
ggml_prec_get_acc(dst), true, can_quantize_y, prefer_f16_b);
|
||||
|
||||
const bool quantize_y = y_kernel_type == GGML_TYPE_Q8_1;
|
||||
const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32;
|
||||
|
||||
const std::vector<vk_matmul_pipeline_pair>* mmp_map =
|
||||
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, (ggml_prec)dst->op_params[0], true);
|
||||
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst), true);
|
||||
|
||||
const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;
|
||||
|
||||
if (qx_needs_dequant) {
|
||||
// dequant src0 to f16
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, (ggml_prec)dst->op_params[0], true);
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst), true);
|
||||
}
|
||||
|
||||
GGML_ASSERT(mmp_map != nullptr);
|
||||
@@ -7749,7 +7778,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte
|
||||
const bool y_non_contig = !ggml_vk_dim01_contiguous(src1);
|
||||
|
||||
const bool f16_f32_kernel = src1->type == GGML_TYPE_F32;
|
||||
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
|
||||
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
|
||||
|
||||
vk_pipeline to_fp16_vk_0 = nullptr;
|
||||
vk_pipeline to_fp16_vk_1 = nullptr;
|
||||
@@ -8128,7 +8157,7 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
uint32_t workgroups_y = (uint32_t)neq2;
|
||||
uint32_t workgroups_z = (uint32_t)neq3;
|
||||
|
||||
const bool f32acc = !ctx->device->fp16 || dst->op_params[3] == GGML_PREC_F32 || k->type == GGML_TYPE_BF16;
|
||||
const bool f32acc = !ctx->device->fp16 || !ggml_vk_prec_allows_f16acc(ggml_prec_get_acc(dst)) || k->type == GGML_TYPE_BF16;
|
||||
|
||||
// dequant K/V once into an f16 scratch, reordered KV layout so FA can read without a stride
|
||||
auto is_dense_kv_cache = [](const ggml_tensor * t) {
|
||||
@@ -15516,9 +15545,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
// If there's not enough shared memory for row_ids and the result tile, fallback to CPU
|
||||
return false;
|
||||
}
|
||||
if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (ggml_prec_get_src(op, 1) == GGML_PREC_F32) {
|
||||
return false;
|
||||
}
|
||||
switch (src0_type) {
|
||||
case GGML_TYPE_F32:
|
||||
|
||||
@@ -3348,6 +3348,36 @@ bool ggml_prec_set_src(
|
||||
return true;
|
||||
}
|
||||
|
||||
enum ggml_prec ggml_prec_get_acc(
|
||||
const struct ggml_tensor * a) {
|
||||
switch (a->op) {
|
||||
case GGML_OP_MUL_MAT:
|
||||
case GGML_OP_MUL_MAT_ID:
|
||||
return (enum ggml_prec) ggml_get_op_params_i32(a, 0);
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
return (enum ggml_prec) ggml_get_op_params_i32(a, 3);
|
||||
default:
|
||||
return GGML_PREC_UNDEFINED;
|
||||
};
|
||||
}
|
||||
|
||||
enum ggml_prec ggml_prec_get_src(
|
||||
const struct ggml_tensor * a,
|
||||
int idx) {
|
||||
GGML_ASSERT(idx >= 0 && idx < GGML_MAX_SRC);
|
||||
|
||||
switch (a->op) {
|
||||
case GGML_OP_MUL_MAT:
|
||||
case GGML_OP_MUL_MAT_ID:
|
||||
if (idx != 1) {
|
||||
return GGML_PREC_UNDEFINED;
|
||||
}
|
||||
return (enum ggml_prec) ggml_get_op_params_i32(a, 2 + idx);
|
||||
default:
|
||||
return GGML_PREC_UNDEFINED;
|
||||
};
|
||||
}
|
||||
|
||||
// ggml_mul_mat
|
||||
|
||||
static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
|
||||
|
||||
Reference in new issue
Block a user