diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 2745a7dbb..d9347e3c2 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2863,6 +2863,12 @@ struct ggml_cplan ggml_graph_plan( cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; } } break; + case GGML_OP_SET_ROWS: + { + if (node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) { + cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; + } + } break; case GGML_OP_SOFT_MAX: case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 7c54cb6f4..c49719374 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -5041,7 +5041,7 @@ static void ggml_compute_forward_set_rows_impl( assert(ne0 == nc); assert(ne2 == ne02); assert(ne3 == ne03); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16)); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); assert(ne02 % ne11 == 0); assert(ne03 % ne12 == 0); @@ -5075,10 +5075,19 @@ static void ggml_compute_forward_set_rows_impl( (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); } else if constexpr (std::is_same_v) { - memcpy( + if (dst->type == GGML_TYPE_F16) { + memcpy( ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), rs); + } else { + float * wdata = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith; + ggml_fp16_to_fp32_row( + (const ggml_fp16_t *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + wdata, nc); + from_float(wdata, + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); + } } else { GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type)); } @@ -5107,16 +5116,12 @@ void ggml_compute_forward_set_rows( } break; case GGML_TYPE_F16: { - if (dst->type == GGML_TYPE_F16) { - if (src1->type == GGML_TYPE_I64) { - ggml_compute_forward_set_rows_impl(params, dst); - } else if (src1->type == GGML_TYPE_I32) { - ggml_compute_forward_set_rows_impl(params, dst); - } else { - GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); - } + if (src1->type == GGML_TYPE_I64) { + ggml_compute_forward_set_rows_impl(params, dst); + } else if (src1->type == GGML_TYPE_I32) { + ggml_compute_forward_set_rows_impl(params, dst); } else { - GGML_ABORT("dst->type = %d (%s) not supported with src0->type = %d (%s)", dst->type, ggml_type_name(dst->type), src0->type, ggml_type_name(src0->type)); + GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); } } break; default: diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 1dfe0bdd5..80e47f2c2 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1340,7 +1340,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return op->src[0]->type != GGML_TYPE_NVFP4; case GGML_OP_SET_ROWS: { - if (op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) { + if (op->src[0]->type == GGML_TYPE_F16) { + return op->type == GGML_TYPE_F16; + } + + if (op->src[0]->type != GGML_TYPE_F32) { return false; } diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 932c561e0..1704da07e 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -723,6 +723,7 @@ struct vk_device_struct { bool uma; bool prefer_host_memory; bool float_controls_rte_fp16; + bool float_controls_denorm_preserve_fp16; bool subgroup_basic; bool subgroup_arithmetic; bool subgroup_shuffle; @@ -868,8 +869,9 @@ struct vk_device_struct { vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT]; vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT]; vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32; - vk_pipeline pipeline_set_rows_i32[GGML_TYPE_COUNT]; - vk_pipeline pipeline_set_rows_i64[GGML_TYPE_COUNT]; + // [src0 0=fp32,1=fp16][dst] + vk_pipeline pipeline_set_rows_i32[2][GGML_TYPE_COUNT]; + vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT]; vk_pipeline pipeline_norm_f32; vk_pipeline pipeline_group_norm_f32; vk_pipeline pipeline_rms_norm_f32; @@ -2595,10 +2597,10 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin vk::ShaderModuleCreateInfo shader_module_create_info({}, spv_size, reinterpret_cast(spv_data)); - // Patch SPIR-V to enable RTE rounding for FP16, avoiding the need for - // separate shader variants compiled with -DRTE16. + // Patch SPIR-V to enable supported FP16 float controls, avoiding the need + // for separate shader variants. std::vector spirv; - if (device->float_controls_rte_fp16) { + if (device->float_controls_rte_fp16 || device->float_controls_denorm_preserve_fp16) { const uint32_t* spv_words = reinterpret_cast(spv_data); size_t word_count = spv_size / sizeof(uint32_t); spirv.assign(spv_words, spv_words + word_count); @@ -2635,9 +2637,17 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin // Insert from latest position first so earlier indices stay valid. - // OpExecutionMode %entrypoint RoundingModeRTE 16 - uint32_t exec_mode[] = { (4u << spv::WordCountShift) | spv::OpExecutionMode, entry_point_id, spv::ExecutionModeRoundingModeRTE, 16 }; - spirv.insert(spirv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode)); + if (device->float_controls_rte_fp16) { + // OpExecutionMode %entrypoint RoundingModeRTE 16 + uint32_t exec_mode[] = { (4u << spv::WordCountShift) | spv::OpExecutionMode, entry_point_id, spv::ExecutionModeRoundingModeRTE, 16 }; + spirv.insert(spirv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode)); + } + + if (device->float_controls_denorm_preserve_fp16) { + // OpExecutionMode %entrypoint DenormPreserve 16 + uint32_t exec_mode[] = { (4u << spv::WordCountShift) | spv::OpExecutionMode, entry_point_id, spv::ExecutionModeDenormPreserve, 16 }; + spirv.insert(spirv.begin() + exec_insert_pos, std::begin(exec_mode), std::end(exec_mode)); + } // OpExtension "SPV_KHR_float_controls" const char ext_str[] = "SPV_KHR_float_controls"; @@ -2647,9 +2657,17 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin memcpy(&extension[1], ext_str, sizeof(ext_str)); spirv.insert(spirv.begin() + ext_insert_pos, extension.begin(), extension.end()); - // OpCapability RoundingModeRTE - uint32_t capability[] = { (2u << spv::WordCountShift) | spv::OpCapability, spv::CapabilityRoundingModeRTE }; - spirv.insert(spirv.begin() + cap_insert_pos, std::begin(capability), std::end(capability)); + if (device->float_controls_rte_fp16) { + // OpCapability RoundingModeRTE + uint32_t capability[] = { (2u << spv::WordCountShift) | spv::OpCapability, spv::CapabilityRoundingModeRTE }; + spirv.insert(spirv.begin() + cap_insert_pos, std::begin(capability), std::end(capability)); + } + + if (device->float_controls_denorm_preserve_fp16) { + // OpCapability DenormPreserve + uint32_t capability[] = { (2u << spv::WordCountShift) | spv::OpCapability, spv::CapabilityDenormPreserve }; + spirv.insert(spirv.begin() + cap_insert_pos, std::begin(capability), std::end(capability)); + } shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data()); } @@ -5187,20 +5205,22 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_Q8_0], "cpy_f32_q8_0", cpy_f32_q8_0_len, cpy_f32_q8_0_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_quant[GGML_TYPE_IQ4_NL], "cpy_f32_iq4_nl", cpy_f32_iq4_nl_len, cpy_f32_iq4_nl_data, "main", 2, sizeof(vk_op_unary_push_constants), {32, 1, 1}, {}, 1); -#define SET_ROWS(itype) \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_F32], "set_rows_f32" #itype, set_rows_f32 ## itype ## _len, set_rows_f32 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_F16], "set_rows_f16" #itype, set_rows_f16 ## itype ## _len, set_rows_f16 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_BF16], "set_rows_bf16" #itype, set_rows_bf16 ## itype ## _len, set_rows_bf16 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q1_0], "set_rows_q1_0" #itype, set_rows_q1_0 ## itype ## _len, set_rows_q1_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q4_0], "set_rows_q4_0" #itype, set_rows_q4_0 ## itype ## _len, set_rows_q4_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q4_1], "set_rows_q4_1" #itype, set_rows_q4_1 ## itype ## _len, set_rows_q4_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q5_0], "set_rows_q5_0" #itype, set_rows_q5_0 ## itype ## _len, set_rows_q5_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q5_1], "set_rows_q5_1" #itype, set_rows_q5_1 ## itype ## _len, set_rows_q5_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_Q8_0], "set_rows_q8_0" #itype, set_rows_q8_0 ## itype ## _len, set_rows_q8_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [GGML_TYPE_IQ4_NL], "set_rows_iq4_nl" #itype, set_rows_iq4_nl ## itype ## _len, set_rows_iq4_nl ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); +#define SET_ROWS(src_idx, src, itype) \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_F32], "set_rows_" #src "_f32" #itype, set_rows_ ## src ## _f32 ## itype ## _len, set_rows_ ## src ## _f32 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_F16], "set_rows_" #src "_f16" #itype, set_rows_ ## src ## _f16 ## itype ## _len, set_rows_ ## src ## _f16 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_BF16], "set_rows_" #src "_bf16" #itype, set_rows_ ## src ## _bf16 ## itype ## _len, set_rows_ ## src ## _bf16 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q1_0], "set_rows_" #src "_q1_0" #itype, set_rows_ ## src ## _q1_0 ## itype ## _len, set_rows_ ## src ## _q1_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q4_0], "set_rows_" #src "_q4_0" #itype, set_rows_ ## src ## _q4_0 ## itype ## _len, set_rows_ ## src ## _q4_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q4_1], "set_rows_" #src "_q4_1" #itype, set_rows_ ## src ## _q4_1 ## itype ## _len, set_rows_ ## src ## _q4_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q5_0], "set_rows_" #src "_q5_0" #itype, set_rows_ ## src ## _q5_0 ## itype ## _len, set_rows_ ## src ## _q5_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q5_1], "set_rows_" #src "_q5_1" #itype, set_rows_ ## src ## _q5_1 ## itype ## _len, set_rows_ ## src ## _q5_1 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_Q8_0], "set_rows_" #src "_q8_0" #itype, set_rows_ ## src ## _q8_0 ## itype ## _len, set_rows_ ## src ## _q8_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_IQ4_NL], "set_rows_" #src "_iq4_nl" #itype, set_rows_ ## src ## _iq4_nl ## itype ## _len, set_rows_ ## src ## _iq4_nl ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); - SET_ROWS(_i32) - SET_ROWS(_i64) + SET_ROWS(0, f32, _i32) + SET_ROWS(0, f32, _i64) + SET_ROWS(1, f16, _i32) + SET_ROWS(1, f16, _i64) #undef SET_ROWS @@ -6031,6 +6051,7 @@ static vk_device ggml_vk_get_device(size_t idx) { device->shader_core_count = 0; } device->float_controls_rte_fp16 = vk12_props.shaderRoundingModeRTEFloat16; + device->float_controls_denorm_preserve_fp16 = vk12_props.shaderDenormPreserveFloat16; device->subgroup_basic = (vk11_props.subgroupSupportedStages & vk::ShaderStageFlagBits::eCompute) && (vk11_props.subgroupSupportedOperations & vk::SubgroupFeatureFlagBits::eBasic); @@ -10843,10 +10864,17 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const case GGML_OP_DUP: return ggml_vk_get_cpy_pipeline(ctx, src0, dst, dst->type); case GGML_OP_SET_ROWS: - if (src1->type == GGML_TYPE_I64) { - return ctx->device->pipeline_set_rows_i64[dst->type]; - } else { - return ctx->device->pipeline_set_rows_i32[dst->type]; + { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) { + return nullptr; + } + const int src_idx = src0->type == GGML_TYPE_F16; + if (src1->type == GGML_TYPE_I64) { + return ctx->device->pipeline_set_rows_i64[src_idx][dst->type]; + } else if (src1->type == GGML_TYPE_I32) { + return ctx->device->pipeline_set_rows_i32[src_idx][dst->type]; + } + return nullptr; } case GGML_OP_SILU_BACK: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { @@ -17500,24 +17528,25 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_SET_ROWS: { - if (op->src[0]->type == GGML_TYPE_F32) { - switch (op->type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - case GGML_TYPE_BF16: - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_IQ4_NL: - return true; - default: - return false; - } + if ((op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) || + (op->src[1]->type != GGML_TYPE_I32 && op->src[1]->type != GGML_TYPE_I64)) { + return false; + } + switch (op->type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + return true; + default: + return false; } - return false; } case GGML_OP_CONT: case GGML_OP_CPY: diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp b/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp index 710c15296..c92798f8e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp @@ -10,7 +10,7 @@ layout(local_size_x = 32, local_size_y = 1, local_size_z = 1) in; const uint BLOCK_SIZE = 32; #endif -layout (binding = 0) readonly buffer S {float data_s[];}; +layout (binding = 0) readonly buffer S {S_TYPE data_s[];}; #if defined(SET_ROWS) #include "generic_binary_head.glsl" @@ -35,7 +35,7 @@ void quantize(uint dst_idx, uint src_idx) float vmax = 0.0; [[unroll]] for (int j = 0; j < QUANT_K_Q4_0; ++j) { - const float v = data_s[src_idx + j]; + const float v = float(data_s[src_idx + j]); if (amax < abs(v)) { amax = abs(v); vmax = v; @@ -48,8 +48,8 @@ void quantize(uint dst_idx, uint src_idx) data_q[dst_idx].d = float16_t(d); [[unroll]] for (int j = 0; j < QUANT_K_Q4_0/2; ++j) { - const float x0 = data_s[src_idx + 0 + j]*id; - const float x1 = data_s[src_idx + QUANT_K_Q4_0/2 + j]*id; + const float x0 = float(data_s[src_idx + 0 + j])*id; + const float x1 = float(data_s[src_idx + QUANT_K_Q4_0/2 + j])*id; const uint xi0 = min(15, int(x0 + 8.5)); const uint xi1 = min(15, int(x1 + 8.5)); @@ -66,7 +66,7 @@ void quantize(uint dst_idx, uint src_idx) float vmax = -vmin; [[unroll]] for (int j = 0; j < QUANT_K_Q4_1; ++j) { - const float v = data_s[src_idx + j]; + const float v = float(data_s[src_idx + j]); if (v < vmin) vmin = v; if (v > vmax) vmax = v; @@ -79,8 +79,8 @@ void quantize(uint dst_idx, uint src_idx) data_q[dst_idx].m = float16_t(vmin); [[unroll]] for (int j = 0; j < QUANT_K_Q4_1/2; ++j) { - const float x0 = (data_s[src_idx + 0 + j] - vmin)*id; - const float x1 = (data_s[src_idx + QUANT_K_Q4_1/2 + j] - vmin)*id; + const float x0 = (float(data_s[src_idx + 0 + j]) - vmin)*id; + const float x1 = (float(data_s[src_idx + QUANT_K_Q4_1/2 + j]) - vmin)*id; const uint xi0 = min(15, int(x0 + 0.5)); const uint xi1 = min(15, int(x1 + 0.5)); @@ -97,7 +97,7 @@ void quantize(uint dst_idx, uint src_idx) float vmax = 0.0; [[unroll]] for (int j = 0; j < QUANT_K_Q5_0; ++j) { - const float v = data_s[src_idx + j]; + const float v = float(data_s[src_idx + j]); if (amax < abs(v)) { amax = abs(v); vmax = v; @@ -111,8 +111,8 @@ void quantize(uint dst_idx, uint src_idx) uint32_t qh = 0; [[unroll]] for (int j = 0; j < QUANT_K_Q5_0/2; ++j) { - const float x0 = data_s[src_idx + 0 + j]*id; - const float x1 = data_s[src_idx + QUANT_K_Q5_0/2 + j]*id; + const float x0 = float(data_s[src_idx + 0 + j])*id; + const float x1 = float(data_s[src_idx + QUANT_K_Q5_0/2 + j])*id; const uint xi0 = min(31, int(x0 + 16.5)); const uint xi1 = min(31, int(x1 + 16.5)); @@ -129,11 +129,11 @@ void quantize(uint dst_idx, uint src_idx) #if defined(DATA_A_Q5_1) void quantize(uint dst_idx, uint src_idx) { - float min = data_s[src_idx + 0]; + float min = float(data_s[src_idx + 0]); float max = min; [[unroll]] for (int j = 1; j < QUANT_K_Q5_1; ++j) { - const float v = data_s[src_idx + j]; + const float v = float(data_s[src_idx + j]); min = v < min ? v : min; max = v > max ? v : max; } @@ -146,8 +146,8 @@ void quantize(uint dst_idx, uint src_idx) uint32_t qh = 0; [[unroll]] for (int j = 0; j < QUANT_K_Q5_1/2; ++j) { - const float x0 = (data_s[src_idx + 0 + j] - min)*id; - const float x1 = (data_s[src_idx + QUANT_K_Q5_1/2 + j] - min)*id; + const float x0 = (float(data_s[src_idx + 0 + j]) - min)*id; + const float x1 = (float(data_s[src_idx + QUANT_K_Q5_1/2 + j]) - min)*id; const uint xi0 = uint(x0 + 0.5); const uint xi1 = uint(x1 + 0.5); @@ -166,7 +166,7 @@ void quantize(uint dst_idx, uint src_idx) float amax = 0.0; // absolute max [[unroll]] for (int j = 0; j < QUANT_K_Q8_0; j++) { - const float v = data_s[src_idx + j]; + const float v = float(data_s[src_idx + j]); amax = max(amax, abs(v)); } @@ -176,7 +176,7 @@ void quantize(uint dst_idx, uint src_idx) data_q[dst_idx].d = float16_t(d); [[unroll]] for (int j = 0; j < QUANT_K_Q8_0; ++j) { - const float x0 = data_s[src_idx + j]*id; + const float x0 = float(data_s[src_idx + j])*id; data_q[dst_idx].qs[j] = int8_t(round(x0)); } @@ -189,7 +189,7 @@ void quantize(uint dst_idx, uint src_idx) float sum_abs = 0.0; [[unroll]] for (int j = 0; j < QUANT_K_Q1_0; j++) { - sum_abs += abs(data_s[src_idx + j]); + sum_abs += abs(float(data_s[src_idx + j])); } const float d = sum_abs / QUANT_K_Q1_0; @@ -201,7 +201,7 @@ void quantize(uint dst_idx, uint src_idx) } [[unroll]] for (int j = 0; j < QUANT_K_Q1_0; ++j) { - if (data_s[src_idx + j] >= 0.0) { + if (float(data_s[src_idx + j]) >= 0.0) { data_q[dst_idx].qs[j / 8] |= uint8_t(1 << (j % 8)); } } @@ -226,7 +226,7 @@ void quantize(uint dst_idx, uint src_idx) float vmax = 0.0; [[unroll]] for (int j = 0; j < QUANT_K_IQ4_NL; ++j) { - const float v = data_s[src_idx + j]; + const float v = float(data_s[src_idx + j]); if (amax < abs(v)) { amax = abs(v); vmax = v; @@ -238,16 +238,16 @@ void quantize(uint dst_idx, uint src_idx) float sumqx = 0, sumq2 = 0; [[unroll]] for (int j = 0; j < QUANT_K_IQ4_NL/2; ++j) { - const float x0 = data_s[src_idx + 0 + j]*id; - const float x1 = data_s[src_idx + QUANT_K_IQ4_NL/2 + j]*id; + const float x0 = float(data_s[src_idx + 0 + j])*id; + const float x1 = float(data_s[src_idx + QUANT_K_IQ4_NL/2 + j])*id; const uint xi0 = best_index(x0); const uint xi1 = best_index(x1); data_q[dst_idx].qs[j] = uint8_t(xi0 | (xi1 << 4)); const float v0 = kvalues_iq4nl[xi0]; const float v1 = kvalues_iq4nl[xi1]; - const float w0 = data_s[src_idx + 0 + j]*data_s[src_idx + 0 + j]; - const float w1 = data_s[src_idx + QUANT_K_IQ4_NL/2 + j]*data_s[src_idx + QUANT_K_IQ4_NL/2 + j]; - sumqx += w0*v0*data_s[src_idx + j] + w1*v1*data_s[src_idx + QUANT_K_IQ4_NL/2 + j]; + const float w0 = float(data_s[src_idx + 0 + j])*float(data_s[src_idx + 0 + j]); + const float w1 = float(data_s[src_idx + QUANT_K_IQ4_NL/2 + j])*float(data_s[src_idx + QUANT_K_IQ4_NL/2 + j]); + sumqx += w0*v0*float(data_s[src_idx + j]) + w1*v1*float(data_s[src_idx + QUANT_K_IQ4_NL/2 + j]); sumq2 += w0*v0*v0 + w1*v1*v1; } @@ -259,14 +259,14 @@ void quantize(uint dst_idx, uint src_idx) #if defined(DATA_A_F32) || defined(DATA_A_F16) void quantize(uint dst_idx, uint src_idx) { - data_q[dst_idx] = A_TYPE(data_s[src_idx]); + data_q[dst_idx] = A_TYPE(float(data_s[src_idx])); } #endif #if defined(DATA_A_BF16) void quantize(uint dst_idx, uint src_idx) { - data_q[dst_idx] = A_TYPE(fp32_to_bf16(data_s[src_idx])); + data_q[dst_idx] = A_TYPE(fp32_to_bf16(float(data_s[src_idx]))); } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 7d09ac4a5..240e1d1b3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -824,13 +824,15 @@ void process_shaders() { string_to_spv("cpy_transpose_32", "copy_transpose.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}}); for (std::string t : {"q1_0", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "iq4_nl"}) { - string_to_spv("cpy_f32_" + t, "copy_to_quant.comp", {{"DATA_A_" + to_uppercase(t), "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); + string_to_spv("cpy_f32_" + t, "copy_to_quant.comp", {{"DATA_A_" + to_uppercase(t), "1"}, {"S_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); string_to_spv("cpy_" + t + "_f32", "copy_from_quant.comp", {{"DATA_A_" + to_uppercase(t), "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); } - for (std::string t : {"f32", "f16", "bf16", "q1_0", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "iq4_nl"}) { - string_to_spv("set_rows_" + t + "_i32", "copy_to_quant.comp", {{"SET_ROWS", "1"}, {"DATA_A_" + to_uppercase(t), "1"}, {"B_TYPE", "uint"}, {"B_SIZE", "32"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); - string_to_spv("set_rows_" + t + "_i64", "copy_to_quant.comp", {{"SET_ROWS", "1"}, {"DATA_A_" + to_uppercase(t), "1"}, {"B_TYPE", "uvec2"}, {"B_SIZE", "64"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); + for (auto src : {std::pair{"f32", "float"}, std::pair{"f16", "float16_t"}}) { + for (std::string dst : {"f32", "f16", "bf16", "q1_0", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "iq4_nl"}) { + string_to_spv("set_rows_" + std::string(src.first) + "_" + dst + "_i32", "copy_to_quant.comp", {{"SET_ROWS", "1"}, {"DATA_A_" + to_uppercase(dst), "1"}, {"B_TYPE", "uint"}, {"B_SIZE", "32"}, {"S_TYPE", src.second}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); + string_to_spv("set_rows_" + std::string(src.first) + "_" + dst + "_i64", "copy_to_quant.comp", {{"SET_ROWS", "1"}, {"DATA_A_" + to_uppercase(dst), "1"}, {"B_TYPE", "uvec2"}, {"B_SIZE", "64"}, {"S_TYPE", src.second}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); + } } auto get_type_str = [](bool f16) { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index a83cb302b..ae49d2d0d 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2423,11 +2423,10 @@ struct test_set_rows : public test_case { void initialize_tensors(ggml_context * ctx) override { for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_is_view_op(t->op)) { + continue; + } if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { - continue; - } - init_set_rows_row_ids(t, ne[1]); } else { init_tensor_uniform(t); @@ -2450,6 +2449,9 @@ struct test_set_rows : public test_case { err_estimate /= 8.0f; } err_estimate *= err_estimate; + if (type_src == GGML_TYPE_F16) { + err_estimate *= 16.0f; + } err_estimate /= 0.25f*float(ne[0] * r * ne[2]*nr23[0] * ne[3]*nr23[1]); return err_estimate; } @@ -7928,17 +7930,19 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false)); test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false)); - for (ggml_type type : all_types) { - for (int b : {1, 7}) { - for (bool v : {false, true}) { - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 5, b, 3 }, { 1, 1, }, 1, v)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v)); + for (ggml_type src_type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + for (ggml_type type : all_types) { + for (int b : {1, 7}) { + for (bool v : {false, true}) { + test_cases.emplace_back(new test_set_rows(src_type, type, GGML_TYPE_I64, { 256, 5, b, 3 }, { 1, 1, }, 1, v)); + test_cases.emplace_back(new test_set_rows(src_type, type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v)); + test_cases.emplace_back(new test_set_rows(src_type, type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v)); - if (ggml_blck_size(type) == 1) { - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v)); - test_cases.emplace_back(new test_set_rows(GGML_TYPE_F32, type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v)); + if (ggml_blck_size(type) == 1) { + test_cases.emplace_back(new test_set_rows(src_type, type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v)); + test_cases.emplace_back(new test_set_rows(src_type, type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v)); + } } } }