From 7fce5439fe451cd48340b112ec37d2befe02c776 Mon Sep 17 00:00:00 2001 From: anujj Date: Thu, 19 Nov 2026 23:43:55 +0530 Subject: [PATCH] ggml : add BF16 support to ROPE and SET_ROWS (CPU, CUDA) (#32387) * ggml : add BF16 support to ROPE and SET_ROWS (CPU, CUDA) * cuda : SET_ROWS from BF16 into quantized types and F8_E4M3 * cuda : fuse BF16 rope into a BF16 cache * tests : trim the BF16 rope cases to one per code path --- ggml/src/ggml-cann/ggml-cann.cpp | 6 +++ ggml/src/ggml-cpu/ggml-cpu.c | 3 +- ggml/src/ggml-cpu/ops.cpp | 38 ++++++++++++-- ggml/src/ggml-cuda/ggml-cuda.cu | 13 +++-- ggml/src/ggml-cuda/rope.cu | 63 +++++++++++++++++++----- ggml/src/ggml-cuda/set-rows.cu | 53 +++++++++++++------- ggml/src/ggml-metal/ggml-metal-device.m | 2 +- ggml/src/ggml-opencl/ggml-opencl.cpp | 2 +- ggml/src/ggml-openvino/ggml-openvino.cpp | 6 +++ ggml/src/ggml-sycl/ggml-sycl.cpp | 4 +- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 3 +- ggml/src/ggml.c | 2 +- tests/test-backend-ops.cpp | 60 +++++++++++++++++++--- 13 files changed, 204 insertions(+), 51 deletions(-) diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index 7e692f1b5..0adc8680f 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -2501,6 +2501,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten break; case GGML_OP_SET_ROWS: { + if (op->src[0]->type == GGML_TYPE_BF16) { + return false; + } switch (op->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: @@ -2553,6 +2556,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten if (((const int32_t *) op->op_params)[15] != 0) { return false; // FIXME: support ggml_rope_set_offset } + if (op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) { + return false; + } if (op->src[0]->ne[0] > 896) { return false; } diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 8b4c7a4ef..4d9760bad 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2933,7 +2933,8 @@ struct ggml_cplan ggml_graph_plan( } break; case GGML_OP_SET_ROWS: { - if (node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) { + if ((node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) || + (node->src[0]->type == GGML_TYPE_BF16 && node->type != GGML_TYPE_BF16)) { cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; } } break; diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 0b540c5a7..cd8134740 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -5531,7 +5531,7 @@ static void ggml_compute_forward_set_rows_impl( assert(ne0 == nc); assert(ne2 == ne02); assert(ne3 == ne03); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); assert(ne02 % ne11 == 0); assert(ne03 % ne12 == 0); @@ -5578,6 +5578,20 @@ static void ggml_compute_forward_set_rows_impl( from_float(wdata, ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); } + } else if constexpr (std::is_same_v) { + if (dst->type == GGML_TYPE_BF16) { + memcpy( + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), + ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + rs); + } else { + float * wdata = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith; + ggml_bf16_to_fp32_row( + (const ggml_bf16_t *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + wdata, nc); + from_float(wdata, + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); + } } else { GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type)); } @@ -5614,6 +5628,16 @@ void ggml_compute_forward_set_rows( GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); } } break; + case GGML_TYPE_BF16: + { + if (src1->type == GGML_TYPE_I64) { + ggml_compute_forward_set_rows_impl(params, dst); + } else if (src1->type == GGML_TYPE_I32) { + ggml_compute_forward_set_rows_impl(params, dst); + } else { + GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); + } + } break; default: { GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type)); @@ -6302,7 +6326,7 @@ static void rotate_pairs(const int64_t n, const int64_t n_offset, const float * } } -template //float or ggml_fp16_t +template //float, ggml_fp16_t or ggml_bf16_t static void ggml_compute_forward_rope_flt( const ggml_compute_params * params, ggml_tensor * dst, @@ -6312,7 +6336,7 @@ static void ggml_compute_forward_rope_flt( const ggml_tensor * src1 = dst->src[1]; const ggml_tensor * src2 = dst->src[2]; - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); GGML_ASSERT(src1->type == GGML_TYPE_I32); float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow; @@ -6472,6 +6496,10 @@ void ggml_compute_forward_rope( { ggml_compute_forward_rope_flt(params, dst, true); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_rope_flt(params, dst, true); + } break; case GGML_TYPE_F32: { ggml_compute_forward_rope_flt(params, dst, true); @@ -6496,6 +6524,10 @@ void ggml_compute_forward_rope_back( { ggml_compute_forward_rope_flt(params, dst, false); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_rope_flt(params, dst, false); + } break; case GGML_TYPE_F32: { ggml_compute_forward_rope_flt(params, dst, false); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 2ce3d0b80..a58f020c2 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3504,7 +3504,12 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope, return false; } - if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16) { + if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16 && set_rows->type != GGML_TYPE_BF16) { + return false; + } + + // BF16 rope fuses into an F16 or BF16 cache, other ropes not into a BF16 cache + if (rope->type == GGML_TYPE_BF16 ? set_rows->type == GGML_TYPE_F32 : set_rows->type == GGML_TYPE_BF16) { return false; } @@ -6510,7 +6515,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_F8_E4M3 || op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && - op->src[0]->type == GGML_TYPE_F32 + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) ) || ( op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F16 ) @@ -6731,7 +6736,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return false; case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: { - return op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]); + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) && + op->type == op->src[0]->type && + op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]); } case GGML_OP_IM2COL: case GGML_OP_IM2COL_3D: diff --git a/ggml/src/ggml-cuda/rope.cu b/ggml/src/ggml-cuda/rope.cu index 2604d014f..657b58a4d 100644 --- a/ggml/src/ggml-cuda/rope.cu +++ b/ggml/src/ggml-cuda/rope.cu @@ -92,13 +92,16 @@ static __global__ void rope_norm(const T * x, } else if constexpr (std::is_same_v) { half2 v = make_half2(x0, x1); ggml_cuda_memcpy_1<4>(dst + idst, &v); + } else if constexpr (std::is_same_v) { + const __align__(4) nv_bfloat16 v[2] = { ggml_cuda_cast(x0), ggml_cuda_cast(x1) }; + ggml_cuda_memcpy_1(dst + idst, v); } }; if (i0 < n_offs || i0 >= n_offs + n_dims) { if (inplace) { return; } - store_coaelsced(x[ix + 0], x[ix + 1]); + store_coaelsced(ggml_cuda_cast(x[ix + 0]), ggml_cuda_cast(x[ix + 1])); return; } @@ -113,8 +116,8 @@ static __global__ void rope_norm(const T * x, rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); - const float x0 = x[ix + 0]; - const float x1 = x[ix + 1]; + const float x0 = ggml_cuda_cast(x[ix + 0]); + const float x1 = ggml_cuda_cast(x[ix + 1]); store_coaelsced(x0 * cos_theta - x1 * sin_theta, x0 * sin_theta + x1 * cos_theta); } @@ -189,8 +192,8 @@ static __global__ void rope_neox(const T * x, rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2 - const float x0 = x[ix + n_offs/2 + 0]; - const float x1 = x[ix + n_offs/2 + n_dims/2]; + const float x0 = ggml_cuda_cast(x[ix + n_offs/2 + 0]); + const float x1 = ggml_cuda_cast(x[ix + n_offs/2 + n_dims/2]); dst[idst + n_offs/2 + 0] = ggml_cuda_cast(x0 * cos_theta - x1 * sin_theta); dst[idst + n_offs/2 + n_dims / 2] = ggml_cuda_cast(x0 * sin_theta + x1 * cos_theta); @@ -293,8 +296,8 @@ static __global__ void rope_multi(const T * x, rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2 - const float x0 = x[ix + n_offs/2 + 0]; - const float x1 = x[ix + n_offs/2 + n_dims/2]; + const float x0 = ggml_cuda_cast(x[ix + n_offs/2 + 0]); + const float x1 = ggml_cuda_cast(x[ix + n_offs/2 + n_dims/2]); dst[idst + n_offs/2 + 0] = ggml_cuda_cast(x0*cos_theta - x1*sin_theta); dst[idst + n_offs/2 + n_dims/2] = ggml_cuda_cast(x0*sin_theta + x1*cos_theta); @@ -357,11 +360,11 @@ static __global__ void rope_vision(const T * x, rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta); - const float x0 = x[ix + 0]; - const float x1 = x[ix + n_dims]; + const float x0 = ggml_cuda_cast(x[ix + 0]); + const float x1 = ggml_cuda_cast(x[ix + n_dims]); - dst[idst + 0] = x0*cos_theta - x1*sin_theta; - dst[idst + n_dims] = x0*sin_theta + x1*cos_theta; + dst[idst + 0] = ggml_cuda_cast(x0*cos_theta - x1*sin_theta); + dst[idst + n_dims] = ggml_cuda_cast(x0*sin_theta + x1*cos_theta); } template @@ -569,8 +572,8 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, } cudaStream_t stream = ctx.stream(); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); + GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16); // When not fused, src0 and dst types must match // When fused (ROPE+VIEW+SET_ROWS), src0 may be F32 and dst may be F16 GGML_ASSERT(src0->type == dst->type || (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16)); @@ -655,6 +658,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) { + rope_neox_cuda((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, row_indices, + set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) { + rope_neox_cuda((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, row_indices, + set_rows_stride, inplace, stream); } else { GGML_ABORT("fatal error"); } @@ -674,6 +687,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, row_indices, set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) { + rope_multi_cuda((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, + row_indices, set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) { + rope_multi_cuda((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, + row_indices, set_rows_stride, inplace, stream); } else { GGML_ABORT("fatal error"); } @@ -686,6 +709,10 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, rope_vision_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, sections, stream); + } else if (src0->type == GGML_TYPE_BF16) { + rope_vision_cuda((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1, + s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, + corr_dims, freq_factors, sections, stream); } else { GGML_ABORT("fatal error"); } @@ -705,6 +732,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) { + rope_norm_cuda((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, row_indices, + set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) { + rope_norm_cuda((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, row_indices, + set_rows_stride, inplace, stream); } else { GGML_ABORT("fatal error"); } diff --git a/ggml/src/ggml-cuda/set-rows.cu b/ggml/src/ggml-cuda/set-rows.cu index 4be3e649b..e4fb51619 100644 --- a/ggml/src/ggml-cuda/set-rows.cu +++ b/ggml/src/ggml-cuda/set-rows.cu @@ -4,8 +4,8 @@ typedef void (*set_rows_kernel_t)(const char * src, char * dst); // Generic quantized set_rows kernel template -template -static __global__ void k_set_rows_quant(const float * __restrict__ src0, +template +static __global__ void k_set_rows_quant(const src_t * __restrict__ src0, const idx_t * __restrict__ src1, block_type * __restrict__ dst, const int64_t ne_total, @@ -56,13 +56,23 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0, ggml_cuda_pdl_sync(); const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12); - const float * src0_row = src0 + i01*s01 + i02*s02 + i03*s03; + const src_t * src0_row = src0 + i01*s01 + i02*s02 + i03*s03; block_type * dst_row_ptr = dst + (dst_row*s1 + i02*s2 + i03*s3) / sizeof(block_type); - const float * src_block = src0_row + i00; + const src_t * src_block = src0_row + i00; block_type * dst_block = dst_row_ptr + i00 / qk; - quantize_func(src_block, dst_block); + if constexpr (std::is_same_v) { + quantize_func(src_block, dst_block); + } else { + // the quantize functions take F32 input: widen the block first + float src_block_f32[qk]; +#pragma unroll + for (int j = 0; j < qk; ++j) { + src_block_f32[j] = ggml_cuda_cast(src_block[j]); + } + quantize_func(src_block_f32, dst_block); + } GGML_UNUSED(ne10); GGML_UNUSED(ne11); @@ -71,9 +81,9 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0, } // Template dispatch function for quantized set_rows -template +template static void set_rows_cuda_quant( - const float * src0_d, const idx_t * src1_d, block_type * dst_d, + const src_t * src0_d, const idx_t * src1_d, block_type * dst_d, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, const size_t nb01, const size_t nb02, const size_t nb03, @@ -87,9 +97,9 @@ static void set_rows_cuda_quant( const dim3 block_size(CUDA_SET_ROWS_BLOCK_SIZE); const dim3 grid_size(num_blocks); - const int64_t s01 = nb01/sizeof(float); - const int64_t s02 = nb02/sizeof(float); - const int64_t s03 = nb03/sizeof(float); + const int64_t s01 = nb01/sizeof(src_t); + const int64_t s02 = nb02/sizeof(src_t); + const int64_t s03 = nb03/sizeof(src_t); const int64_t s10 = nb10/sizeof(idx_t); const int64_t s11 = nb11/sizeof(idx_t); const int64_t s12 = nb12/sizeof(idx_t); @@ -104,7 +114,7 @@ static void set_rows_cuda_quant( const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11); const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12); - k_set_rows_quant<<>>( + k_set_rows_quant<<>>( src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01, s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd, ne01_fd, ne02_fd, ne11_fd, ne12_fd); } @@ -268,7 +278,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s stream ); } else if (dst->type == GGML_TYPE_Q4_0) { - set_rows_cuda_quant( + set_rows_cuda_quant( src0_d, src1_d, (block_q4_0*)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, @@ -278,7 +288,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s stream ); } else if (dst->type == GGML_TYPE_Q4_1) { - set_rows_cuda_quant( + set_rows_cuda_quant( src0_d, src1_d, (block_q4_1*)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, @@ -288,7 +298,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s stream ); } else if (dst->type == GGML_TYPE_Q5_0) { - set_rows_cuda_quant( + set_rows_cuda_quant( src0_d, src1_d, (block_q5_0*)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, @@ -298,7 +308,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s stream ); } else if (dst->type == GGML_TYPE_Q5_1) { - set_rows_cuda_quant( + set_rows_cuda_quant( src0_d, src1_d, (block_q5_1*)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, @@ -308,7 +318,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s stream ); } else if (dst->type == GGML_TYPE_Q8_0) { - set_rows_cuda_quant( + set_rows_cuda_quant( src0_d, src1_d, (block_q8_0*)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, @@ -318,7 +328,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s stream ); } else if (dst->type == GGML_TYPE_IQ4_NL) { - set_rows_cuda_quant( + set_rows_cuda_quant( src0_d, src1_d, (block_iq4_nl*)dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, @@ -382,12 +392,11 @@ void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_te } } - void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16)); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16)); GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32); if (src0->type == GGML_TYPE_F32) { @@ -402,6 +411,12 @@ void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { } else { set_rows_cuda(ctx, src0, src1, dst); } + } else if (src0->type == GGML_TYPE_BF16) { + if (src1->type == GGML_TYPE_I64) { + set_rows_cuda(ctx, src0, src1, dst); + } else { + set_rows_cuda(ctx, src0, src1, dst); + } } else { GGML_ABORT("unsupported type %s", ggml_type_name(src0->type)); } diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 26746d0b3..2e085453a 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1713,7 +1713,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32 && (ggml_is_contiguous_rows(op->src[0])); case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: - return ggml_rope_get_truncate(op); + return ggml_rope_get_truncate(op) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); case GGML_OP_IM2COL: return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32); case GGML_OP_CONV_2D: diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 0d9212232..8162ace2e 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -12305,7 +12305,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te } return false; } - return true; + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; } case GGML_OP_SOLVE_TRI: return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 98f22cfd7..bd49560d8 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1766,6 +1766,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { !GgmlOvDecoder::is_conv_state_snapshot_writeback(op)) { return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"}; } + if (op->op == GGML_OP_SET_ROWS && op->src[0]->type == GGML_TYPE_BF16) { + return {false, "SET_ROWS with BF16 src0 is not supported"}; + } if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) { // These are all f16-arithmetic dequant rounding errors that intermittently exceed the @@ -2035,6 +2038,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) { return {false, "ROPE on VIEW / non-contiguous input is not supported"}; } + if (op->type == GGML_TYPE_BF16) { + return {false, "ROPE with BF16 type is not supported"}; + } break; } case GGML_OP_TRANSPOSE: { diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 4fc90e543..9da436ced 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -7434,8 +7434,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons if (op->type == GGML_TYPE_TQ2_0 || op->type == GGML_TYPE_TQ1_0) { return false; } - auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || - op->src[0]->type == GGML_TYPE_BF16) && + auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); return res; } @@ -7624,6 +7623,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons } case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; case GGML_OP_IM2COL: case GGML_OP_IM2COL_3D: case GGML_OP_UPSCALE: diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 6955bb406..ecd46d828 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -16729,8 +16729,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_REPEAT_BACK: return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_ROPE: - return ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]); + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_ROPE_BACK: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; case GGML_OP_NONE: case GGML_OP_RESHAPE: case GGML_OP_VIEW: diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 68e3a011a..ecd667753 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -4019,7 +4019,7 @@ struct ggml_tensor * ggml_set_rows( GGML_ASSERT(b->ne[2] % c->ne[1] == 0); GGML_ASSERT(b->ne[3] % c->ne[2] == 0); GGML_ASSERT(c->ne[3] == 1); - GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16); + GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16 || b->type == GGML_TYPE_BF16); GGML_ASSERT(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32); GGML_ASSERT(ggml_is_contiguous_rows(a)); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 01ea4c115..d6b849944 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2830,6 +2830,12 @@ struct test_set_rows : public test_case { if (type_src == GGML_TYPE_F16) { err_estimate *= 16.0f; } + if (type_src == GGML_TYPE_BF16) { + // BF16 values have 8 significant bits, so many of them land exactly on a rounding tie, + // and backends break ties differently (e.g. the CPU q8_0 rounds half to even, CUDA half away from zero). + // Allow 1 in 32 values to be off by one instead of one value in total. + return err_estimate / (0.25f*32.0f); + } err_estimate /= 0.25f*float(ne[0] * r * ne[2]*nr23[0] * ne[3]*nr23[1]); return err_estimate; } @@ -2860,10 +2866,15 @@ struct test_rope_set_rows : public test_case { const int n_dims; const bool yarn; const bool truncate; + const ggml_type type_a; // type of the rope input const int n_ctx{512}; std::string vars() override { - return VARS_TO_STR7(type, type_idx, ne_a, mode, n_dims, yarn, truncate); + std::string s = VARS_TO_STR7(type, type_idx, ne_a, mode, n_dims, yarn, truncate); + if (type_a != GGML_TYPE_F32) { + s += "," + VAR_TO_STR(type_a); + } + return s; } std::string op_desc(ggml_tensor * t) override { @@ -2871,17 +2882,27 @@ struct test_rope_set_rows : public test_case { return "ROPE_SET_ROWS"; } + double max_nmse_err() override { + if (type_a != GGML_TYPE_BF16) { + return 1e-7; + } + // into F16 the reference rounds the BF16 rope result to BF16 first, the fused rope + set_rows does not + // otherwise the limit of a BF16 test_rope: the angle error n_ctx*FLT_EPSILON can flip a BF16 rounding + return type == GGML_TYPE_F16 ? 1e-5 : n_ctx * FLT_EPSILON / 128.0; + } + bool run_whole_graph() override { return true; } test_rope_set_rows(ggml_type type, ggml_type type_idx, std::array ne_a, int mode, - int n_dims = 128, bool yarn = false, bool truncate = true) - : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims), yarn(yarn), truncate(truncate) {} + int n_dims = 128, bool yarn = false, bool truncate = true, + ggml_type type_a = GGML_TYPE_F32) + : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims), yarn(yarn), truncate(truncate), type_a(type_a) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne_a[0], ne_a[1], ne_a[2], 1); + ggml_tensor * a = ggml_new_tensor_4d(ctx, type_a, ne_a[0], ne_a[1], ne_a[2], 1); ggml_set_name(a, "a"); const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; @@ -6793,6 +6814,10 @@ struct test_rope : public test_case { // the f32 rotation angle of a position below n_ctx carries a rounding error of up to n_ctx*FLT_EPSILON radians // backends compute the angle with different rounding sequences, so their agreement is bounded by that error const double eps_theta = n_ctx * FLT_EPSILON; + if (type == GGML_TYPE_BF16) { + // with BF16 output that error can also change the rounding: a step of 2^-7 with a chance of about eps_theta/2^-7 + return std::max(test_case::max_nmse_err(), eps_theta / 128.0); + } return std::max(test_case::max_nmse_err(), eps_theta * eps_theta); } @@ -10895,6 +10920,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, true)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, true)); + // BF16 source: one case per type covers batch dims, broadcast indices and gaps between rows + for (ggml_type type : all_types) { + test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, type, GGML_TYPE_I64, { 256, 11, 1, 1 }, { 2, 3, }, 7, true)); + } + test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false)); + test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION, GGML_ROPE_TYPE_IMROPE }) { for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { @@ -10905,11 +10936,19 @@ static std::vector> make_test_cases_eval() { } } test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 128, 32, 8, 1 }, GGML_ROPE_TYPE_IMROPE)); + for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION, GGML_ROPE_TYPE_IMROPE }) { + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_BF16}) { + for (int ne2 : {1, 8, 512}) { + test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, ne2, 1 }, mode, 128, false, true, GGML_TYPE_BF16)); + } + } + } // partial rotation for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE }) { for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 256, 4, 8, 1 }, mode, 64)); } + test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F16, GGML_TYPE_I64, { 256, 4, 8, 1 }, mode, 64, false, true, GGML_TYPE_BF16)); } for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { @@ -12716,8 +12755,17 @@ static std::vector> make_test_cases_eval() { } } + // BF16: each mode forward on a strided view and backward on an offset view, with and without freq factors, and one in place + for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION}) { + for (bool ff : {false, true}) { + test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 1, true)); + test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 2, false)); + } + test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true)); + } + // rotated dims window at an offset (ggml_rope_set_offset), not supported for vision mode - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { for (bool fw : {true, false}) { // fw == forward for (bool ff : {false, true}) { test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, 1.4245f, 0.7465f, 1.4245f, ff, 0, fw, false, 32)); @@ -14108,7 +14156,7 @@ static std::vector> make_test_cases_perf() { } for (bool fw : {true, false}) { // fw == forward - for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) { for (bool ff : {false, true}) { // freq_factors for (float v : { 0, 1 }) { test_cases.emplace_back(new test_rope(type, {128, 32, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // llama 7B