vulkan: add bfloat16 support to various ops (#32202)
* vulkan: expand bf16 support for unary and binary ops and rms_norm * cpu: add bf16 support for scale, clamp, leaky_relu, swiglu_oai, swiglu_clamp, tri and diag * fix sycl fill supports_op condition * restore CUDA op order
This commit is contained in:
1 parent
65758d01f5
commit
43bd7a2a42
24 files changed
+410
-461
No files matched your search
+87
-49
@@ -2240,7 +2240,8 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor *
|
||||
|
||||
// ggml_compute_tri
|
||||
|
||||
static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_tri_t(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
|
||||
const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0);
|
||||
@@ -2266,11 +2267,11 @@ static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggm
|
||||
const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
|
||||
const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
|
||||
|
||||
const float * src_ptr = (const float *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
|
||||
float * dst_ptr = ( float *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
|
||||
const T * src_ptr = (const T *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
|
||||
T * dst_ptr = ( T *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
|
||||
|
||||
for (int i0 = 0; i0 < ne0; ++i0) {
|
||||
dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : 0.0f;
|
||||
dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : T(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2281,7 +2282,12 @@ void ggml_compute_forward_tri(const ggml_compute_params * params, ggml_tensor *
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_tri_f32(params, dst);
|
||||
ggml_compute_forward_tri_t<uint32_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_tri_t<uint16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -2708,7 +2714,8 @@ static void ggml_compute_forward_leaky_relu_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_leaky_relu_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_leaky_relu_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -2728,13 +2735,16 @@ static void ggml_compute_forward_leaky_relu_f16(
|
||||
float negative_slope;
|
||||
memcpy(&negative_slope, dst->op_params, sizeof(float));
|
||||
|
||||
assert(dst->nb[0] == sizeof(ggml_fp16_t));
|
||||
assert(src0->nb[0] == sizeof(ggml_fp16_t));
|
||||
assert(dst->nb[0] == sizeof(T));
|
||||
assert(src0->nb[0] == sizeof(T));
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
ggml_vec_leaky_relu_f16(nc,
|
||||
(ggml_fp16_t *) ((char *) dst->data + i*( dst->nb[1])),
|
||||
(ggml_fp16_t *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);
|
||||
T * y = (T *) ((char *) dst->data + i*( dst->nb[1]));
|
||||
const T * x = (const T *) ((const char *) src0->data + i*(src0->nb[1]));
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float v = type_conversion_table<T>::to_f32(x[k]);
|
||||
y[k] = type_conversion_table<T>::from_f32(((v > 0.f) ? v : 0.f) + negative_slope * ((v < 0.0f) ? v : 0.f));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2751,7 +2761,11 @@ void ggml_compute_forward_leaky_relu(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_leaky_relu_f16(params, dst);
|
||||
ggml_compute_forward_leaky_relu_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_leaky_relu_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -3452,7 +3466,8 @@ static void ggml_compute_forward_swiglu_oai_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_swiglu_oai_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_swiglu_oai_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -3492,9 +3507,9 @@ static void ggml_compute_forward_swiglu_oai_f16(
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int i1 = ir0; i1 < ir1; i1++) {
|
||||
ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
|
||||
ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
|
||||
ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1]));
|
||||
T * src0_p = (T *) (src0_d + i1*src0_o);
|
||||
T * src1_p = (T *) (src1_d + i1*src1_o);
|
||||
T * dst_p = (T *) ((char *) dst->data + i1*(dst->nb[1]));
|
||||
|
||||
if (!src1) {
|
||||
src0_p += swapped ? nc : 0;
|
||||
@@ -3502,15 +3517,15 @@ static void ggml_compute_forward_swiglu_oai_f16(
|
||||
}
|
||||
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit);
|
||||
const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit);
|
||||
const float x = std::min(type_conversion_table<T>::to_f32(src0_p[k]), limit);
|
||||
const float y = std::clamp(type_conversion_table<T>::to_f32(src1_p[k]), -limit, limit);
|
||||
const float out_glu = x / (1.f + expf(alpha * (-x)));
|
||||
dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f));
|
||||
dst_p[k] = type_conversion_table<T>::from_f32(out_glu * (y + 1.f));
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]);
|
||||
const float x = type_conversion_table<T>::to_f32(dst_p[k]);
|
||||
GGML_UNUSED(x);
|
||||
assert(!isnan(x));
|
||||
assert(!isinf(x));
|
||||
@@ -3532,7 +3547,11 @@ static void ggml_compute_forward_swiglu_oai(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_swiglu_oai_f16(params, dst);
|
||||
ggml_compute_forward_swiglu_oai_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_swiglu_oai_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -3602,7 +3621,8 @@ static void ggml_compute_forward_swiglu_clamp_f32(const ggml_compute_params * pa
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_swiglu_clamp_half(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
char * src0_d = (char *) src0->data;
|
||||
@@ -3635,9 +3655,9 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int i1 = ir0; i1 < ir1; i1++) {
|
||||
ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1 * src0_o);
|
||||
ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1 * src1_o);
|
||||
ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1 * (dst->nb[1]));
|
||||
T * src0_p = (T *) (src0_d + i1 * src0_o);
|
||||
T * src1_p = (T *) (src1_d + i1 * src1_o);
|
||||
T * dst_p = (T *) ((char *) dst->data + i1 * (dst->nb[1]));
|
||||
|
||||
if (!src1) {
|
||||
src0_p += swapped ? nc : 0;
|
||||
@@ -3645,14 +3665,14 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa
|
||||
}
|
||||
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float gate = std::min(GGML_FP16_TO_FP32(src0_p[k]), limit);
|
||||
const float up = std::clamp(GGML_FP16_TO_FP32(src1_p[k]), -limit, limit);
|
||||
dst_p[k] = GGML_FP32_TO_FP16(gate / (1.f + expf(-gate)) * up);
|
||||
const float gate = std::min(type_conversion_table<T>::to_f32(src0_p[k]), limit);
|
||||
const float up = std::clamp(type_conversion_table<T>::to_f32(src1_p[k]), -limit, limit);
|
||||
dst_p[k] = type_conversion_table<T>::from_f32(gate / (1.f + expf(-gate)) * up);
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = GGML_FP16_TO_FP32(dst_p[k]);
|
||||
const float x = type_conversion_table<T>::to_f32(dst_p[k]);
|
||||
GGML_UNUSED(x);
|
||||
assert(!isnan(x));
|
||||
assert(!isinf(x));
|
||||
@@ -3667,7 +3687,10 @@ static void ggml_compute_forward_swiglu_clamp(const ggml_compute_params * params
|
||||
ggml_compute_forward_swiglu_clamp_f32(params, dst);
|
||||
break;
|
||||
case GGML_TYPE_F16:
|
||||
ggml_compute_forward_swiglu_clamp_f16(params, dst);
|
||||
ggml_compute_forward_swiglu_clamp_half<ggml_fp16_t>(params, dst);
|
||||
break;
|
||||
case GGML_TYPE_BF16:
|
||||
ggml_compute_forward_swiglu_clamp_half<ggml_bf16_t>(params, dst);
|
||||
break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
@@ -4953,7 +4976,8 @@ static void ggml_compute_forward_scale_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_scale_bf16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_scale_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -4989,10 +5013,10 @@ static void ggml_compute_forward_scale_bf16(
|
||||
const int64_t i2 = (i / ne01) % ne02;
|
||||
const int64_t i3 = i / (ne01*ne02);
|
||||
|
||||
const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
const T * x = (const T *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
T * y = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
for (int k = 0; k < nc; k++) {
|
||||
y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b);
|
||||
y[k] = type_conversion_table<T>::from_f32(s*type_conversion_table<T>::to_f32(x[k]) + b);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5008,9 +5032,13 @@ void ggml_compute_forward_scale(
|
||||
{
|
||||
ggml_compute_forward_scale_f32(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_scale_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_scale_bf16(params, dst);
|
||||
ggml_compute_forward_scale_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -5694,7 +5722,8 @@ void ggml_compute_forward_get_rows_back(
|
||||
|
||||
// ggml_compute_forward_diag
|
||||
|
||||
static void ggml_compute_forward_diag_f32(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_diag_t(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -5714,14 +5743,14 @@ static void ggml_compute_forward_diag_f32(
|
||||
GGML_ASSERT(ne02 == ne2);
|
||||
GGML_ASSERT(ne03 == ne3);
|
||||
|
||||
GGML_ASSERT(nb00 == sizeof(float));
|
||||
GGML_ASSERT(nb0 == sizeof(float));
|
||||
GGML_ASSERT(nb00 == sizeof(T));
|
||||
GGML_ASSERT(nb0 == sizeof(T));
|
||||
|
||||
for (int i3 = 0; i3 < ne3; i3++) {
|
||||
for (int i2 = 0; i2 < ne2; i2++) {
|
||||
for (int i1 = 0; i1 < ne1; i1++) {
|
||||
float * d = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
|
||||
float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02);
|
||||
T * d = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
|
||||
T * s = (T *)((char *) src0->data + i3*nb03 + i2*nb02);
|
||||
for (int i0 = 0; i0 < i1; i0++) {
|
||||
d[i0] = 0;
|
||||
}
|
||||
@@ -5743,7 +5772,12 @@ void ggml_compute_forward_diag(
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_diag_f32(params, dst);
|
||||
ggml_compute_forward_diag_t<uint32_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_diag_t<uint16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -6026,7 +6060,8 @@ static void ggml_compute_forward_clamp_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_clamp_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_clamp_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -6045,20 +6080,20 @@ static void ggml_compute_forward_clamp_f16(
|
||||
|
||||
GGML_TENSOR_UNARY_OP_LOCALS
|
||||
|
||||
GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
|
||||
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
|
||||
GGML_ASSERT( nb0 == sizeof(T));
|
||||
GGML_ASSERT(nb00 == sizeof(T));
|
||||
|
||||
for (int j = ith; j < n; j += nth) {
|
||||
const int64_t i1 = j % ne01;
|
||||
const int64_t i2 = (j / ne01) % ne02;
|
||||
const int64_t i3 = j / (ne01*ne02);
|
||||
|
||||
ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
T * dst_ptr = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
T * src0_ptr = (T *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
|
||||
for (int i = 0; i < nc; i++) {
|
||||
float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]);
|
||||
dst_ptr[i] = GGML_CPU_FP32_TO_FP16(MAX(MIN(v, max), min));
|
||||
float v = type_conversion_table<T>::to_f32(src0_ptr[i]);
|
||||
dst_ptr[i] = type_conversion_table<T>::from_f32(MAX(MIN(v, max), min));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6076,9 +6111,12 @@ void ggml_compute_forward_clamp(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_clamp_f16(params, dst);
|
||||
ggml_compute_forward_clamp_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_clamp_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_Q1_0:
|
||||
case GGML_TYPE_Q2_0:
|
||||
case GGML_TYPE_Q4_0:
|
||||
|
||||
@@ -962,12 +962,6 @@ inline static void ggml_vec_relu_f16 (const int n, ggml_fp16_t * y, const ggml_f
|
||||
}
|
||||
}
|
||||
inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }
|
||||
inline static void ggml_vec_leaky_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float ns) {
|
||||
for (int i = 0; i < n; ++i) {
|
||||
float v = GGML_CPU_FP16_TO_FP32(x[i]);
|
||||
y[i] = GGML_CPU_FP32_TO_FP16(((v > 0.f) ? v : 0.f) + ns * ((v < 0.0f) ? v : 0.f));
|
||||
}
|
||||
}
|
||||
inline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }
|
||||
inline static void ggml_vec_sigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
|
||||
for (int i = 0; i < n; ++i) {
|
||||
|
||||
@@ -6564,7 +6564,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type &&
|
||||
ggml_is_contiguous_rows(op->src[0]);
|
||||
case GGML_OP_CLAMP:
|
||||
return ggml_is_contiguous_rows(op->src[0]);
|
||||
return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
||||
case GGML_OP_ADD:
|
||||
case GGML_OP_SUB:
|
||||
case GGML_OP_MUL:
|
||||
@@ -6664,7 +6664,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
case GGML_OP_ARANGE:
|
||||
case GGML_OP_TIMESTEP_EMBEDDING:
|
||||
return true;
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
case GGML_OP_RWKV_WKV6:
|
||||
case GGML_OP_GATED_LINEAR_ATTN:
|
||||
case GGML_OP_RWKV_WKV7:
|
||||
@@ -6690,7 +6692,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_OP_FILL:
|
||||
case GGML_OP_CUMSUM:
|
||||
case GGML_OP_TRI:
|
||||
return true;
|
||||
case GGML_OP_DIAG:
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
return true;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
|
||||
@@ -2019,7 +2019,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
};
|
||||
}
|
||||
case GGML_OP_DIAG:
|
||||
return true;
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_OPT_STEP_ADAMW:
|
||||
case GGML_OP_OPT_STEP_SGD:
|
||||
return has_simdgroup_reduction;
|
||||
|
||||
@@ -12283,7 +12283,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
case GGML_OP_TRANSPOSE:
|
||||
return true;
|
||||
case GGML_OP_DIAG:
|
||||
return true;
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_ROPE: {
|
||||
const int mode = ((const int32_t *) op->op_params)[2];
|
||||
if (!ggml_rope_get_truncate(op)) {
|
||||
|
||||
@@ -7700,8 +7700,10 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
return false;
|
||||
}
|
||||
case GGML_OP_FILL:
|
||||
return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
|
||||
case GGML_OP_CUMSUM:
|
||||
case GGML_OP_DIAG:
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_CROSS_ENTROPY_LOSS:
|
||||
case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
|
||||
return true;
|
||||
|
||||
@@ -838,15 +838,16 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_acc_f32;
|
||||
vk_pipeline pipeline_set_f32;
|
||||
|
||||
// [src0][src1][dst], 0=fp32,1=fp16,2=bf16, null where unsupported
|
||||
vk_pipeline pipeline_add[3][3][3];
|
||||
vk_pipeline pipeline_add_norepeat[3][3][3];
|
||||
vk_pipeline pipeline_sub[3][3][3];
|
||||
vk_pipeline pipeline_sub_norepeat[3][3][3];
|
||||
vk_pipeline pipeline_mul[3][3][3];
|
||||
vk_pipeline pipeline_mul_norepeat[3][3][3];
|
||||
vk_pipeline pipeline_div[3][3][3];
|
||||
vk_pipeline pipeline_div_norepeat[3][3][3];
|
||||
// [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16]
|
||||
vk_pipeline pipeline_add[2][2][2];
|
||||
vk_pipeline pipeline_add_norepeat[2][2][2];
|
||||
vk_pipeline pipeline_sub[2][2][2];
|
||||
vk_pipeline pipeline_sub_norepeat[2][2][2];
|
||||
vk_pipeline pipeline_mul[2][2][2];
|
||||
vk_pipeline pipeline_mul_norepeat[2][2][2];
|
||||
vk_pipeline pipeline_div[2][2][2];
|
||||
vk_pipeline pipeline_div_norepeat[2][2][2];
|
||||
vk_pipeline pipeline_add_rms[2][2][2];
|
||||
vk_pipeline pipeline_add_rms_norepeat[2][2][2];
|
||||
|
||||
@@ -859,11 +860,17 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;
|
||||
vk_pipeline pipeline_concat_transpose_i8, pipeline_concat_transpose_i16, pipeline_concat_transpose_i32;
|
||||
vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;
|
||||
vk_pipeline pipeline_scale_f32;
|
||||
vk_pipeline pipeline_log[2];
|
||||
vk_pipeline pipeline_tri[2];
|
||||
vk_pipeline pipeline_diag[2];
|
||||
vk_pipeline pipeline_clamp[2];
|
||||
// [src/dst 0=fp32,1=fp16,2=bf16]
|
||||
vk_pipeline pipeline_scale[3];
|
||||
vk_pipeline pipeline_log[3];
|
||||
vk_pipeline pipeline_tri[3];
|
||||
vk_pipeline pipeline_diag[3];
|
||||
vk_pipeline pipeline_clamp[3];
|
||||
vk_pipeline pipeline_sqr[3];
|
||||
vk_pipeline pipeline_sqrt[3];
|
||||
vk_pipeline pipeline_sin[3];
|
||||
vk_pipeline pipeline_cos[3];
|
||||
vk_pipeline pipeline_leaky_relu[3];
|
||||
vk_pipeline pipeline_pad_f32;
|
||||
vk_pipeline pipeline_pad_reflect_1d_f32;
|
||||
vk_pipeline pipeline_roll_f32;
|
||||
@@ -880,10 +887,10 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT];
|
||||
vk_pipeline pipeline_norm_f32;
|
||||
vk_pipeline pipeline_group_norm_f32;
|
||||
std::array<vk_pipeline, 3> pipeline_rms_norm_f32;
|
||||
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_f32;
|
||||
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_f32;
|
||||
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_mul_f32;
|
||||
// [src/dst 0=fp32,1=bf16][mul]
|
||||
std::array<vk_pipeline, 3> pipeline_rms_norm[2][2];
|
||||
// [src/dst 0=fp32,1=bf16][residual 0=fp32,1=bf16][post mul]
|
||||
std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add[2][2][2];
|
||||
vk_pipeline pipeline_rms_norm_mul_add_partials_f32;
|
||||
vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32;
|
||||
vk_pipeline pipeline_rms_norm_set_rows_f32_f32;
|
||||
@@ -895,36 +902,11 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_rms_norm_back_f32;
|
||||
vk_pipeline pipeline_l2_norm_f32;
|
||||
|
||||
// [src/dst 0=fp32,1=fp16]
|
||||
vk_pipeline pipeline_exp[2];
|
||||
vk_pipeline pipeline_expm1[2];
|
||||
vk_pipeline pipeline_elu[2];
|
||||
vk_pipeline pipeline_gelu[2];
|
||||
vk_pipeline pipeline_gelu_erf[2];
|
||||
vk_pipeline pipeline_gelu_quick[2];
|
||||
vk_pipeline pipeline_silu[2];
|
||||
vk_pipeline pipeline_relu[2];
|
||||
vk_pipeline pipeline_sqr[2];
|
||||
vk_pipeline pipeline_sqrt[2];
|
||||
vk_pipeline pipeline_sin[2];
|
||||
vk_pipeline pipeline_cos[2];
|
||||
vk_pipeline pipeline_xielu[2];
|
||||
vk_pipeline pipeline_neg[2];
|
||||
vk_pipeline pipeline_tanh[2];
|
||||
vk_pipeline pipeline_sigmoid[2];
|
||||
vk_pipeline pipeline_hardsigmoid[2];
|
||||
vk_pipeline pipeline_hardswish[2];
|
||||
vk_pipeline pipeline_abs[2];
|
||||
vk_pipeline pipeline_softplus[2];
|
||||
vk_pipeline pipeline_step[2];
|
||||
vk_pipeline pipeline_round[2];
|
||||
vk_pipeline pipeline_ceil[2];
|
||||
vk_pipeline pipeline_floor[2];
|
||||
vk_pipeline pipeline_trunc[2];
|
||||
vk_pipeline pipeline_sgn[2];
|
||||
// [src/dst 0=fp32,1=fp16,2=bf16]
|
||||
vk_pipeline pipeline_unary[GGML_UNARY_OP_COUNT][3];
|
||||
|
||||
// fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b]
|
||||
vk_pipeline pipeline_unary_mul[4][2][2][2];
|
||||
// fused UNARY+MUL pipelines: [op][0=fp32,1=fp16,2=bf16][norepeat][op_on_b]
|
||||
vk_pipeline pipeline_unary_mul[4][3][2][2];
|
||||
|
||||
vk_pipeline pipeline_add1_f16_f16;
|
||||
vk_pipeline pipeline_add1_f16_f32;
|
||||
@@ -935,15 +917,9 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_fill_f32;
|
||||
vk_pipeline pipeline_fill_f16;
|
||||
|
||||
vk_pipeline pipeline_geglu[2];
|
||||
vk_pipeline pipeline_reglu[2];
|
||||
vk_pipeline pipeline_swiglu[2];
|
||||
vk_pipeline pipeline_swiglu_oai[2];
|
||||
vk_pipeline pipeline_swiglu_clamp[2];
|
||||
vk_pipeline pipeline_geglu_erf[2];
|
||||
vk_pipeline pipeline_geglu_quick[2];
|
||||
// [src/dst 0=fp32,1=fp16,2=bf16]
|
||||
vk_pipeline pipeline_glu[GGML_GLU_OP_COUNT][3];
|
||||
|
||||
vk_pipeline pipeline_leaky_relu[2];
|
||||
vk_pipeline pipeline_silu_back_f32;
|
||||
vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;
|
||||
vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512;
|
||||
|
||||
@@ -3600,10 +3600,16 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
for (uint32_t i = 0; i < 3; ++i) {
|
||||
const uint32_t block_size = 128u << i;
|
||||
const std::string suffix = "_" + std::to_string(block_size);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_f32[i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_f32[i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_f32[i], "rms_norm_mul_add_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_mul_f32[i], "rms_norm_mul_add_mul_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[0][0][i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[0][1][i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[1][0][i], "rms_norm_bf16" + suffix, RMS_SHADER(rms_norm_bf16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[1][1][i], "rms_norm_mul_bf16" + suffix, RMS_SHADER(rms_norm_bf16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
for (uint32_t pm : {0, 1}) {
|
||||
const std::string name = pm ? "rms_norm_mul_add_mul" : "rms_norm_mul_add";
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[0][0][pm][i], name + "_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[1][0][pm][i], name + "_bf16_f32" + suffix, RMS_SHADER(rms_norm_mul_add_bf16_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[1][1][pm][i], name + "_bf16_bf16" + suffix, RMS_SHADER(rms_norm_mul_add_bf16_bf16), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups);
|
||||
}
|
||||
}
|
||||
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_partials_f32, "rms_norm_mul_add_mul_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true);
|
||||
@@ -3714,6 +3720,22 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
CREATE_BINARY(add_rms, _norepeat, {1}, 4)
|
||||
#undef CREATE_BINARY
|
||||
|
||||
#define CREATE_BINARY_BF16(name, namemod, spec, bindings) \
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_ ## name ## namemod[2][0][2], #name "_bf16_f32_bf16" #namemod, name ## _bf16_f32_bf16_len, name ## _bf16_f32_bf16_data, \
|
||||
"main", (bindings), sizeof(vk_op_binary_push_constants), {512, 1, 1}, spec, 1); \
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_ ## name ## namemod[2][2][2], #name "_bf16_bf16_bf16" #namemod, name ## _bf16_bf16_bf16_len, name ## _bf16_bf16_bf16_data, \
|
||||
"main", (bindings), sizeof(vk_op_binary_push_constants), {512, 1, 1}, spec, 1);
|
||||
|
||||
CREATE_BINARY_BF16(add, , {0}, 4)
|
||||
CREATE_BINARY_BF16(add, _norepeat, {1}, 4)
|
||||
CREATE_BINARY_BF16(sub, , {0}, 3)
|
||||
CREATE_BINARY_BF16(sub, _norepeat, {1}, 3)
|
||||
CREATE_BINARY_BF16(mul, , {0}, 3)
|
||||
CREATE_BINARY_BF16(mul, _norepeat, {1}, 3)
|
||||
CREATE_BINARY_BF16(div, , {0}, 3)
|
||||
CREATE_BINARY_BF16(div, _norepeat, {1}, 3)
|
||||
#undef CREATE_BINARY_BF16
|
||||
|
||||
if (device->multi_add) {
|
||||
for (uint32_t i = 0; i < MAX_FUSED_ADDS; ++i) {
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_multi_add[i], "multi_add_f32_" + std::to_string(i+1), multi_add_f32_len, multi_add_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1);
|
||||
@@ -3739,17 +3761,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_upscale_bicubic_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BICUBIC}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_antialias_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS}, 1);
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_log[0], "log_f32", log_f32_len, log_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_log[1], "log_f16", log_f16_len, log_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_tri[0], "tri_f32", tri_f32_len, tri_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_tri[1], "tri_f16", tri_f16_len, tri_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_diag[0], "diag_f32", diag_f32_len, diag_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_diag[1], "diag_f16", diag_f16_len, diag_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_pad_f32, "pad_f32", pad_f32_len, pad_f32_data, "main", 2, sizeof(vk_op_pad_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_pad_reflect_1d_f32, "pad_reflect_1d_f32", pad_reflect_1d_f32_len, pad_reflect_1d_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
@@ -3762,44 +3773,63 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
|
||||
#define CREATE_UNARY(name) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_ ## name [2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
CREATE_UNARY(elu)
|
||||
CREATE_UNARY(gelu)
|
||||
CREATE_UNARY(gelu_erf)
|
||||
CREATE_UNARY(gelu_quick)
|
||||
CREATE_UNARY(silu)
|
||||
CREATE_UNARY(relu)
|
||||
CREATE_UNARY(scale)
|
||||
CREATE_UNARY(log)
|
||||
CREATE_UNARY(sqr)
|
||||
CREATE_UNARY(sqrt)
|
||||
CREATE_UNARY(sin)
|
||||
CREATE_UNARY(cos)
|
||||
CREATE_UNARY(clamp)
|
||||
CREATE_UNARY(leaky_relu)
|
||||
CREATE_UNARY(xielu)
|
||||
CREATE_UNARY(neg)
|
||||
CREATE_UNARY(tanh)
|
||||
CREATE_UNARY(sigmoid)
|
||||
CREATE_UNARY(hardsigmoid)
|
||||
CREATE_UNARY(hardswish)
|
||||
CREATE_UNARY(abs)
|
||||
CREATE_UNARY(softplus)
|
||||
CREATE_UNARY(step)
|
||||
CREATE_UNARY(round)
|
||||
CREATE_UNARY(ceil)
|
||||
CREATE_UNARY(floor)
|
||||
CREATE_UNARY(trunc)
|
||||
CREATE_UNARY(sgn)
|
||||
CREATE_UNARY(exp)
|
||||
CREATE_UNARY(expm1)
|
||||
#undef CREATE_UNARY
|
||||
|
||||
#define CREATE_COPY_OP(name) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_32", name ## _32_len, name ## _32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_16", name ## _16_len, name ## _16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
|
||||
device->pipeline_ ## name [2] = device->pipeline_ ## name [1];
|
||||
|
||||
CREATE_COPY_OP(tri)
|
||||
CREATE_COPY_OP(diag)
|
||||
#undef CREATE_COPY_OP
|
||||
|
||||
#define CREATE_UNARY_OP(name, op) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
CREATE_UNARY_OP(elu, ELU)
|
||||
CREATE_UNARY_OP(gelu, GELU)
|
||||
CREATE_UNARY_OP(gelu_erf, GELU_ERF)
|
||||
CREATE_UNARY_OP(gelu_quick, GELU_QUICK)
|
||||
CREATE_UNARY_OP(silu, SILU)
|
||||
CREATE_UNARY_OP(relu, RELU)
|
||||
CREATE_UNARY_OP(xielu, XIELU)
|
||||
CREATE_UNARY_OP(neg, NEG)
|
||||
CREATE_UNARY_OP(tanh, TANH)
|
||||
CREATE_UNARY_OP(sigmoid, SIGMOID)
|
||||
CREATE_UNARY_OP(hardsigmoid, HARDSIGMOID)
|
||||
CREATE_UNARY_OP(hardswish, HARDSWISH)
|
||||
CREATE_UNARY_OP(abs, ABS)
|
||||
CREATE_UNARY_OP(softplus, SOFTPLUS)
|
||||
CREATE_UNARY_OP(step, STEP)
|
||||
CREATE_UNARY_OP(round, ROUND)
|
||||
CREATE_UNARY_OP(ceil, CEIL)
|
||||
CREATE_UNARY_OP(floor, FLOOR)
|
||||
CREATE_UNARY_OP(trunc, TRUNC)
|
||||
CREATE_UNARY_OP(sgn, SGN)
|
||||
CREATE_UNARY_OP(exp, EXP)
|
||||
CREATE_UNARY_OP(expm1, EXPM1)
|
||||
#undef CREATE_UNARY_OP
|
||||
|
||||
// spec constants: {norepeat, op_on_b}
|
||||
#define CREATE_UNARY_MUL(name, idx) \
|
||||
for (int dt = 0; dt < 2; ++dt) { \
|
||||
const size_t len_ = dt ? name ## _mul_f16_len : name ## _mul_f32_len; \
|
||||
const unsigned char * data_ = dt ? name ## _mul_f16_data : name ## _mul_f32_data; \
|
||||
const std::string dts_ = dt ? "f16" : "f32"; \
|
||||
for (int dt = 0; dt < 3; ++dt) { \
|
||||
const size_t len_ = dt == 2 ? name ## _mul_bf16_len : dt ? name ## _mul_f16_len : name ## _mul_f32_len; \
|
||||
const unsigned char * data_ = dt == 2 ? name ## _mul_bf16_data : dt ? name ## _mul_f16_data : name ## _mul_f32_data; \
|
||||
const std::string dts_ = dt == 2 ? "bf16" : dt ? "f16" : "f32"; \
|
||||
for (int ob = 0; ob < 2; ++ob) \
|
||||
for (int nr = 0; nr < 2; ++nr) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_unary_mul[(idx)][dt][nr][ob], \
|
||||
@@ -3819,17 +3849,18 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
#define CREATE_GLU(name) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true);
|
||||
#define CREATE_GLU(name, op) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true);
|
||||
|
||||
CREATE_GLU(geglu)
|
||||
CREATE_GLU(reglu)
|
||||
CREATE_GLU(swiglu)
|
||||
CREATE_GLU(swiglu_oai)
|
||||
CREATE_GLU(swiglu_clamp)
|
||||
CREATE_GLU(geglu_erf)
|
||||
CREATE_GLU(geglu_quick)
|
||||
CREATE_GLU(geglu, GEGLU)
|
||||
CREATE_GLU(reglu, REGLU)
|
||||
CREATE_GLU(swiglu, SWIGLU)
|
||||
CREATE_GLU(swiglu_oai, SWIGLU_OAI)
|
||||
CREATE_GLU(swiglu_clamp, SWIGLU_CLAMP)
|
||||
CREATE_GLU(geglu_erf, GEGLU_ERF)
|
||||
CREATE_GLU(geglu_quick, GEGLU_QUICK)
|
||||
#undef CREATE_GLU
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
@@ -9374,6 +9405,33 @@ static bool ggml_vk_concat_is_transposed(const vk_device & device, const ggml_te
|
||||
return src0->ne[1] == src1->ne[1] && dst->ne[1] == src1->ne[1];
|
||||
}
|
||||
|
||||
static bool ggml_vk_is_float_type(ggml_type type) {
|
||||
return type == GGML_TYPE_F32 || type == GGML_TYPE_F16 || type == GGML_TYPE_BF16;
|
||||
}
|
||||
|
||||
static uint32_t ggml_vk_float_type_idx(ggml_type type) {
|
||||
return type == GGML_TYPE_BF16 ? 2 : type == GGML_TYPE_F16 ? 1 : 0;
|
||||
}
|
||||
|
||||
static bool ggml_vk_is_float_elementwise(const ggml_tensor * op) {
|
||||
return ggml_vk_is_float_type(op->type) && op->src[0]->type == op->type;
|
||||
}
|
||||
|
||||
// bf16: bf16 src0/dst, f32 or bf16 src1
|
||||
static bool ggml_vk_binary_types_supported(ggml_type src0, ggml_type src1, ggml_type dst) {
|
||||
if (src0 == GGML_TYPE_BF16 || src1 == GGML_TYPE_BF16 || dst == GGML_TYPE_BF16) {
|
||||
return src0 == GGML_TYPE_BF16 && dst == GGML_TYPE_BF16 && (src1 == GGML_TYPE_BF16 || src1 == GGML_TYPE_F32);
|
||||
}
|
||||
return ggml_vk_is_float_type(src0) && ggml_vk_is_float_type(src1) && ggml_vk_is_float_type(dst);
|
||||
}
|
||||
|
||||
static vk_pipeline ggml_vk_float_pipeline(const vk_pipeline (&pipelines)[3], const ggml_tensor * src0, const ggml_tensor * dst) {
|
||||
if (src0->type != dst->type || !ggml_vk_is_float_type(dst->type)) {
|
||||
return nullptr;
|
||||
}
|
||||
return pipelines[ggml_vk_float_type_idx(dst->type)];
|
||||
}
|
||||
|
||||
static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) {
|
||||
switch (op) {
|
||||
case GGML_OP_GET_ROWS:
|
||||
@@ -9410,14 +9468,19 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
case GGML_OP_SUB:
|
||||
case GGML_OP_MUL:
|
||||
case GGML_OP_DIV:
|
||||
if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||
|
||||
(src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) ||
|
||||
(dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16)) {
|
||||
{
|
||||
if (!ggml_vk_binary_types_supported(src0->type, src1->type, dst->type)) {
|
||||
return nullptr;
|
||||
}
|
||||
const uint32_t s0 = ggml_vk_float_type_idx(src0->type);
|
||||
const uint32_t s1 = ggml_vk_float_type_idx(src1->type);
|
||||
const uint32_t d = ggml_vk_float_type_idx(dst->type);
|
||||
switch (op) {
|
||||
case GGML_OP_ADD:
|
||||
{
|
||||
if ((ctx->num_additional_fused_ops > 0 || ctx->do_add_rms_partials) && (s0 == 2 || s1 == 2 || d == 2)) {
|
||||
return nullptr;
|
||||
}
|
||||
if (ctx->num_additional_fused_ops > 0) {
|
||||
if (ctx->do_add_rms_partials) {
|
||||
return ctx->device->pipeline_multi_add_rms[ctx->num_additional_fused_ops];
|
||||
@@ -9430,28 +9493,29 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
|
||||
} else {
|
||||
auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_add_norepeat : ctx->device->pipeline_add;
|
||||
return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
|
||||
return pipelines[s0][s1][d];
|
||||
}
|
||||
}
|
||||
case GGML_OP_SUB:
|
||||
{
|
||||
auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_sub_norepeat : ctx->device->pipeline_sub;
|
||||
return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
|
||||
return pipelines[s0][s1][d];
|
||||
}
|
||||
case GGML_OP_MUL:
|
||||
{
|
||||
auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_mul_norepeat : ctx->device->pipeline_mul;
|
||||
return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
|
||||
return pipelines[s0][s1][d];
|
||||
}
|
||||
case GGML_OP_DIV:
|
||||
{
|
||||
auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_div_norepeat : ctx->device->pipeline_div;
|
||||
return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
|
||||
return pipelines[s0][s1][d];
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
case GGML_OP_ADD_ID:
|
||||
if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) {
|
||||
return ctx->device->pipeline_add_id_f32;
|
||||
@@ -9509,58 +9573,23 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
}
|
||||
return nullptr;
|
||||
case GGML_OP_SCALE:
|
||||
if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
|
||||
return ctx->device->pipeline_scale_f32;
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_scale, src0, dst);
|
||||
case GGML_OP_SQR:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_sqr[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_sqr, src0, dst);
|
||||
case GGML_OP_SQRT:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_sqrt[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_sqrt, src0, dst);
|
||||
case GGML_OP_SIN:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_sin[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_sin, src0, dst);
|
||||
case GGML_OP_COS:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_cos[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_cos, src0, dst);
|
||||
case GGML_OP_LOG:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_log[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_log, src0, dst);
|
||||
case GGML_OP_TRI:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_tri[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_tri, src0, dst);
|
||||
case GGML_OP_DIAG:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_diag[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_diag, src0, dst);
|
||||
case GGML_OP_CLAMP:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_clamp[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_clamp, src0, dst);
|
||||
case GGML_OP_PAD:
|
||||
if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
|
||||
return ctx->device->pipeline_pad_f32;
|
||||
@@ -9626,7 +9655,10 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
if (ctx->do_add_rms_partials) {
|
||||
return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32;
|
||||
}
|
||||
return ggml_vk_rms_norm_pipeline(ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32, src0->ne[0]);
|
||||
return ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm[0][ctx->fused_rms_norm_mode == RMS_NORM_MUL], src0->ne[0]);
|
||||
}
|
||||
if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16 && !ctx->do_add_rms_partials) {
|
||||
return ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm[1][ctx->fused_rms_norm_mode == RMS_NORM_MUL], src0->ne[0]);
|
||||
}
|
||||
return nullptr;
|
||||
case GGML_OP_RMS_NORM_BACK:
|
||||
@@ -9640,87 +9672,9 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
}
|
||||
return nullptr;
|
||||
case GGML_OP_UNARY:
|
||||
if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||
|
||||
(dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) ||
|
||||
(src0->type != dst->type)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
switch (ggml_get_unary_op(dst)) {
|
||||
case GGML_UNARY_OP_EXP:
|
||||
return ctx->device->pipeline_exp[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_EXPM1:
|
||||
return ctx->device->pipeline_expm1[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_ELU:
|
||||
return ctx->device->pipeline_elu[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_SILU:
|
||||
return ctx->device->pipeline_silu[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_GELU:
|
||||
return ctx->device->pipeline_gelu[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_GELU_ERF:
|
||||
return ctx->device->pipeline_gelu_erf[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_GELU_QUICK:
|
||||
return ctx->device->pipeline_gelu_quick[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_RELU:
|
||||
return ctx->device->pipeline_relu[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_XIELU:
|
||||
return ctx->device->pipeline_xielu[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_NEG:
|
||||
return ctx->device->pipeline_neg[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_TANH:
|
||||
return ctx->device->pipeline_tanh[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_SIGMOID:
|
||||
return ctx->device->pipeline_sigmoid[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_HARDSIGMOID:
|
||||
return ctx->device->pipeline_hardsigmoid[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_HARDSWISH:
|
||||
return ctx->device->pipeline_hardswish[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_ABS:
|
||||
return ctx->device->pipeline_abs[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_SOFTPLUS:
|
||||
return ctx->device->pipeline_softplus[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_STEP:
|
||||
return ctx->device->pipeline_step[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_ROUND:
|
||||
return ctx->device->pipeline_round[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_CEIL:
|
||||
return ctx->device->pipeline_ceil[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_FLOOR:
|
||||
return ctx->device->pipeline_floor[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_TRUNC:
|
||||
return ctx->device->pipeline_trunc[dst->type == GGML_TYPE_F16];
|
||||
case GGML_UNARY_OP_SGN:
|
||||
return ctx->device->pipeline_sgn[dst->type == GGML_TYPE_F16];
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_unary[ggml_get_unary_op(dst)], src0, dst);
|
||||
case GGML_OP_GLU:
|
||||
if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||
|
||||
(dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) ||
|
||||
(src0->type != dst->type)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
switch (ggml_get_glu_op(dst)) {
|
||||
case GGML_GLU_OP_GEGLU:
|
||||
return ctx->device->pipeline_geglu[dst->type == GGML_TYPE_F16];
|
||||
case GGML_GLU_OP_REGLU:
|
||||
return ctx->device->pipeline_reglu[dst->type == GGML_TYPE_F16];
|
||||
case GGML_GLU_OP_SWIGLU:
|
||||
return ctx->device->pipeline_swiglu[dst->type == GGML_TYPE_F16];
|
||||
case GGML_GLU_OP_SWIGLU_OAI:
|
||||
return ctx->device->pipeline_swiglu_oai[dst->type == GGML_TYPE_F16];
|
||||
case GGML_GLU_OP_SWIGLU_CLAMP:
|
||||
return ctx->device->pipeline_swiglu_clamp[dst->type == GGML_TYPE_F16];
|
||||
case GGML_GLU_OP_GEGLU_ERF:
|
||||
return ctx->device->pipeline_geglu_erf[dst->type == GGML_TYPE_F16];
|
||||
case GGML_GLU_OP_GEGLU_QUICK:
|
||||
return ctx->device->pipeline_geglu_quick[dst->type == GGML_TYPE_F16];
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_glu[ggml_get_glu_op(dst)], src0, dst);
|
||||
case GGML_OP_SOFT_MAX:
|
||||
GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F32);
|
||||
@@ -9965,11 +9919,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
}
|
||||
return nullptr;
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
if (src0->type == dst->type &&
|
||||
(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
|
||||
return ctx->device->pipeline_leaky_relu[dst->type == GGML_TYPE_F16];
|
||||
}
|
||||
return nullptr;
|
||||
return ggml_vk_float_pipeline(ctx->device->pipeline_leaky_relu, src0, dst);
|
||||
case GGML_OP_CONV_2D:
|
||||
case GGML_OP_CONV_TRANSPOSE_2D:
|
||||
if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
|
||||
@@ -10805,13 +10755,12 @@ void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const
|
||||
const ggml_tensor * src1 = op_on_b ? unary->src[0] :
|
||||
((mul->src[0] == unary) ? mul->src[1] : mul->src[0]);
|
||||
|
||||
const bool f16 = src0->type == GGML_TYPE_F16;
|
||||
const bool norepeat = ggml_are_same_shape(src0, src1);
|
||||
const int oi = ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary));
|
||||
if (oi < 0) {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][f16][norepeat][op_on_b];
|
||||
vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][ggml_vk_float_type_idx(src0->type)][norepeat][op_on_b];
|
||||
|
||||
const uint32_t src0_type_size = ggml_type_size(src0->type);
|
||||
const uint32_t src1_type_size = ggml_type_size(src1->type);
|
||||
@@ -11592,11 +11541,13 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s
|
||||
|
||||
vk_pipeline pipeline;
|
||||
if (ctx->do_add_rms_partials) {
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||
pipeline = do_post_multiply ?
|
||||
ctx->device->pipeline_rms_norm_mul_add_mul_partials_f32 : ctx->device->pipeline_rms_norm_mul_add_partials_f32;
|
||||
} else {
|
||||
pipeline = ggml_vk_rms_norm_pipeline(do_post_multiply ?
|
||||
ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32, src0->ne[0]);
|
||||
const bool src0_bf16 = src0->type == GGML_TYPE_BF16;
|
||||
const bool residual_bf16 = residual->type == GGML_TYPE_BF16;
|
||||
pipeline = ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm_mul_add[src0_bf16][residual_bf16][do_post_multiply], src0->ne[0]);
|
||||
}
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
|
||||
if (ctx->do_add_rms_partials) {
|
||||
@@ -12994,6 +12945,8 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in
|
||||
if (next_node_idx < cgraph->n_nodes &&
|
||||
cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM &&
|
||||
cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] &&
|
||||
cgraph->nodes[next_node_idx]->type == GGML_TYPE_F32 &&
|
||||
node->type == GGML_TYPE_F32 &&
|
||||
ggml_nrows(cgraph->nodes[next_node_idx]) == 1 &&
|
||||
ctx->device->add_rms_fusion && !ctx->device->disable_fusion) {
|
||||
uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]);
|
||||
@@ -14292,7 +14245,7 @@ static bool ggml_vk_can_fuse_unary_mul(const struct ggml_cgraph * cgraph, int un
|
||||
if (ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) {
|
||||
return false;
|
||||
}
|
||||
if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) {
|
||||
if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16 && unary->type != GGML_TYPE_BF16) {
|
||||
return false;
|
||||
}
|
||||
if (unary->type != mul->type) {
|
||||
@@ -14350,12 +14303,24 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr
|
||||
const ggml_tensor *rms_norm = cgraph->nodes[node_idx];
|
||||
const ggml_tensor *mul = cgraph->nodes[node_idx + 1];
|
||||
|
||||
GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(rms_norm->type == GGML_TYPE_F32);
|
||||
// rms_norm only supports f32
|
||||
if (mul->src[0]->type != GGML_TYPE_F32 ||
|
||||
mul->src[1]->type != GGML_TYPE_F32 ||
|
||||
mul->type != GGML_TYPE_F32) {
|
||||
const ggml_type t = rms_norm->type;
|
||||
if (t == GGML_TYPE_F32) {
|
||||
if (mul->src[0]->type != GGML_TYPE_F32 ||
|
||||
mul->src[1]->type != GGML_TYPE_F32 ||
|
||||
mul->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
} else if (t == GGML_TYPE_BF16) {
|
||||
// bf16 activations, f32 weights
|
||||
if (mul->src[0] != rms_norm ||
|
||||
mul->src[1]->type != GGML_TYPE_F32 ||
|
||||
mul->type != GGML_TYPE_BF16) {
|
||||
return false;
|
||||
}
|
||||
if (ops.size() >= 3 && ops.begin()[2] != GGML_OP_ADD) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
// if rms_norm is the B operand, then we don't handle broadcast
|
||||
@@ -14374,7 +14339,7 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr
|
||||
if (add->src[0] != mul && add->src[1] != mul) {
|
||||
return false;
|
||||
}
|
||||
if (residual->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 ||
|
||||
if ((residual->type != GGML_TYPE_F32 && residual->type != t) || add->type != t ||
|
||||
!ggml_are_same_shape(add, residual) || !ggml_is_contiguous(residual) ||
|
||||
!ggml_is_contiguous(add) || get_misalign_bytes(ctx, residual) != 0) {
|
||||
return false;
|
||||
@@ -14392,7 +14357,7 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr
|
||||
return false;
|
||||
}
|
||||
// The shader reads data_e[0], so the final multiply must use a scalar.
|
||||
if (scale->type != GGML_TYPE_F32 || post_mul->type != GGML_TYPE_F32 ||
|
||||
if (scale->type != GGML_TYPE_F32 || post_mul->type != t ||
|
||||
ggml_nelements(scale) != 1 || !ggml_is_contiguous(post_mul) ||
|
||||
get_misalign_bytes(ctx, scale) != 0) {
|
||||
return false;
|
||||
@@ -14933,6 +14898,10 @@ bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const str
|
||||
|
||||
const int mode = ((const int32_t *) rope->op_params)[2];
|
||||
|
||||
if (rms->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// noncontig tensors aren't tested, and don't seem common in practice
|
||||
if (!ggml_is_contiguous(rms) ||
|
||||
!ggml_is_contiguous(mul) ||
|
||||
@@ -16257,9 +16226,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
case GGML_UNARY_OP_FLOOR:
|
||||
case GGML_UNARY_OP_TRUNC:
|
||||
case GGML_UNARY_OP_SGN:
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
|
||||
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&
|
||||
(op->src[0]->type == op->type);
|
||||
return ggml_vk_is_float_elementwise(op);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
@@ -16272,10 +16239,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
case GGML_GLU_OP_GEGLU_ERF:
|
||||
case GGML_GLU_OP_GEGLU_QUICK:
|
||||
case GGML_GLU_OP_SWIGLU_CLAMP:
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
|
||||
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&
|
||||
(op->src[0]->type == op->type) &&
|
||||
(!op->src[1] || op->src[1]->type == op->src[0]->type);
|
||||
return ggml_vk_is_float_elementwise(op) && (!op->src[1] || op->src[1]->type == op->type);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
@@ -16548,16 +16512,15 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
case GGML_OP_GROUP_NORM:
|
||||
return ggml_is_contiguous(op->src[0]);
|
||||
case GGML_OP_NORM:
|
||||
case GGML_OP_RMS_NORM:
|
||||
case GGML_OP_L2_NORM:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_RMS_NORM:
|
||||
return ggml_vk_is_float_elementwise(op) && op->type != GGML_TYPE_F16;
|
||||
case GGML_OP_ADD:
|
||||
case GGML_OP_SUB:
|
||||
case GGML_OP_MUL:
|
||||
case GGML_OP_DIV:
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
|
||||
(op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) &&
|
||||
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);
|
||||
return ggml_vk_binary_types_supported(op->src[0]->type, op->src[1]->type, op->type);
|
||||
case GGML_OP_ADD_ID:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->src[2]->type == GGML_TYPE_I32 &&
|
||||
op->type == GGML_TYPE_F32;
|
||||
@@ -16568,10 +16531,12 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
case GGML_OP_SQRT:
|
||||
case GGML_OP_SIN:
|
||||
case GGML_OP_COS:
|
||||
case GGML_OP_LOG:
|
||||
case GGML_OP_CLAMP:
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
|
||||
op->type == op->src[0]->type;
|
||||
case GGML_OP_TRI:
|
||||
case GGML_OP_DIAG:
|
||||
return ggml_vk_is_float_elementwise(op);
|
||||
case GGML_OP_OPT_STEP_ADAMW:
|
||||
case GGML_OP_OPT_STEP_SGD:
|
||||
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
|
||||
@@ -16579,11 +16544,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32
|
||||
&& ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32
|
||||
&& op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_LOG:
|
||||
case GGML_OP_TRI:
|
||||
case GGML_OP_DIAG:
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
|
||||
op->type == op->src[0]->type;
|
||||
case GGML_OP_ARGSORT:
|
||||
{
|
||||
if (!ggml_is_contiguous(op) || !ggml_is_contiguous(op->src[0])) {
|
||||
@@ -16629,7 +16589,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
case GGML_OP_FILL:
|
||||
return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
|
||||
case GGML_OP_SCALE:
|
||||
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
|
||||
return ggml_is_contiguous(op->src[0]) && ggml_vk_is_float_elementwise(op);
|
||||
case GGML_OP_PAD:
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
case GGML_OP_ROLL:
|
||||
|
||||
@@ -36,10 +36,10 @@ void main() {
|
||||
uint i00, i01, i02, i03;
|
||||
get_indices(idx, i00, i01, i02, i03);
|
||||
|
||||
FLOAT_TYPE sum = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]);
|
||||
FLOAT_TYPE sum = A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]);
|
||||
sum_sq += sum*sum;
|
||||
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(sum);
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(sum);
|
||||
|
||||
idx += num_threads;
|
||||
}
|
||||
|
||||
@@ -20,8 +20,7 @@ void main() {
|
||||
const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
|
||||
|
||||
if (i10 == i11) {
|
||||
const float val = float(data_a[get_aoffset() + i13*p.nb03 + i12*p.nb02 + 0*p.nb01 + i10*p.nb00]);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + i13*p.nb03 + i12*p.nb02 + 0*p.nb01 + i10*p.nb00]);
|
||||
} else {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0);
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ void main() {
|
||||
uint i00, i01, i02, i03;
|
||||
get_indices(idx, i00, i01, i02, i03);
|
||||
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) / FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) / B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
||||
|
||||
idx += num_threads;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#extension GL_EXT_shader_16bit_storage : require
|
||||
|
||||
#include "types.glsl"
|
||||
#include "utils.glsl"
|
||||
|
||||
|
||||
|
||||
@@ -21,15 +21,15 @@ void main() {
|
||||
const uint offset = (p.ne00 / 2) * p.nb00;
|
||||
const uint idx = src_idx_a;
|
||||
|
||||
data_d[dst_idx] = D_TYPE(op(float(data_a[idx]), float(data_a[idx + offset])));
|
||||
data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[idx]), A_TO_FLOAT(data_a[idx + offset])));
|
||||
} else if (p.mode == 1) {
|
||||
// Swapped
|
||||
const uint offset = (p.ne00 / 2) * p.nb00;
|
||||
const uint idx = src_idx_a;
|
||||
|
||||
data_d[dst_idx] = D_TYPE(op(float(data_a[idx + offset]), float(data_a[idx])));
|
||||
data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[idx + offset]), A_TO_FLOAT(data_a[idx])));
|
||||
} else {
|
||||
// Split
|
||||
data_d[dst_idx] = D_TYPE(op(float(data_a[src_idx_a]), float(data_b[src_idx_b])));
|
||||
data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[src_idx_a]), A_TO_FLOAT(data_b[src_idx_b])));
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,6 @@ void main() {
|
||||
return;
|
||||
}
|
||||
|
||||
const float val = float(data_a[get_aoffset() + src0_idx(idx)]);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(log(val));
|
||||
const float val = A_TO_FLOAT(data_a[get_aoffset() + src0_idx(idx)]);
|
||||
data_d[get_doffset() + dst_idx(idx)] = FLOAT_TO_D(log(val));
|
||||
}
|
||||
@@ -20,7 +20,7 @@ void main() {
|
||||
uint i00, i01, i02, i03;
|
||||
get_indices(idx, i00, i01, i02, i03);
|
||||
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) * B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
||||
|
||||
idx += num_threads;
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ layout (binding = 6) readonly buffer R_I {uvec2 rope_data_i[];}; // indices for
|
||||
|
||||
#elif RMS_NORM_ADD_FUSION
|
||||
|
||||
layout (binding = 3) readonly buffer C {float data_c[];};
|
||||
layout (binding = 3) readonly buffer C {C_TYPE data_c[];};
|
||||
layout (binding = 4) readonly buffer E {float data_e[];};
|
||||
|
||||
#elif RMS_NORM_SET_ROWS_FUSION
|
||||
@@ -90,7 +90,7 @@ void rms_norm(uint num_iters) {
|
||||
[[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
|
||||
FLOAT_TYPE xi = FLOAT_TYPE(0);
|
||||
if (col < ncols) {
|
||||
xi = FLOAT_TYPE(data_a[a_offset + col]);
|
||||
xi = FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col]));
|
||||
}
|
||||
sum += xi * xi;
|
||||
}
|
||||
@@ -137,28 +137,28 @@ void rms_norm(uint num_iters) {
|
||||
if (col >= ncols) {
|
||||
continue;
|
||||
}
|
||||
FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]);
|
||||
FLOAT_TYPE value = scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])) * FLOAT_TYPE(B_TO_FLOAT(data_b[b_offset + fastmod(col, p.ne10)]));
|
||||
#if RMS_NORM_ADD_FUSION
|
||||
value += FLOAT_TYPE(data_c[d_offset + col]);
|
||||
value += FLOAT_TYPE(C_TO_FLOAT(data_c[d_offset + col]));
|
||||
if (do_post_multiply) {
|
||||
value *= FLOAT_TYPE(data_e[0]);
|
||||
}
|
||||
#endif
|
||||
data_d[d_offset + col] = D_TYPE(value);
|
||||
data_d[d_offset + col] = FLOAT_TO_D(value);
|
||||
}
|
||||
} else {
|
||||
[[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
|
||||
if (col >= ncols) {
|
||||
continue;
|
||||
}
|
||||
FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]);
|
||||
FLOAT_TYPE value = scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])) * FLOAT_TYPE(B_TO_FLOAT(data_b[b_offset + col]));
|
||||
#if RMS_NORM_ADD_FUSION
|
||||
value += FLOAT_TYPE(data_c[d_offset + col]);
|
||||
value += FLOAT_TYPE(C_TO_FLOAT(data_c[d_offset + col]));
|
||||
if (do_post_multiply) {
|
||||
value *= FLOAT_TYPE(data_e[0]);
|
||||
}
|
||||
#endif
|
||||
data_d[d_offset + col] = D_TYPE(value);
|
||||
data_d[d_offset + col] = FLOAT_TO_D(value);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -166,7 +166,7 @@ void rms_norm(uint num_iters) {
|
||||
if (col >= ncols) {
|
||||
continue;
|
||||
}
|
||||
data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]));
|
||||
data_d[d_offset + col] = FLOAT_TO_D(scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])));
|
||||
}
|
||||
}
|
||||
#if RMS_NORM_ROPE_FUSION
|
||||
|
||||
@@ -18,7 +18,7 @@ void main() {
|
||||
continue;
|
||||
}
|
||||
|
||||
data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2));
|
||||
data_d[get_doffset() + idx] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2));
|
||||
idx += num_threads;
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ void main() {
|
||||
uint i00, i01, i02, i03;
|
||||
get_indices(idx, i00, i01, i02, i03);
|
||||
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) - FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) - B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
||||
|
||||
idx += num_threads;
|
||||
}
|
||||
|
||||
@@ -34,8 +34,7 @@ void main() {
|
||||
}
|
||||
|
||||
if (pass) {
|
||||
const float val = float(data_a[get_aoffset() + src0_idx(idx)]);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
|
||||
} else {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0);
|
||||
}
|
||||
|
||||
@@ -1976,6 +1976,27 @@ vec4 bf16_to_fp32(uvec4 u)
|
||||
return vec4(bf16_to_fp32(u.x), bf16_to_fp32(u.y), bf16_to_fp32(u.z), bf16_to_fp32(u.w));
|
||||
}
|
||||
|
||||
#if defined(DATA_A_BF16)
|
||||
#define A_TO_FLOAT(x) bf16_to_fp32(uint32_t(x))
|
||||
#else
|
||||
#define A_TO_FLOAT(x) float(x)
|
||||
#endif
|
||||
#if defined(DATA_B_BF16)
|
||||
#define B_TO_FLOAT(x) bf16_to_fp32(uint32_t(x))
|
||||
#else
|
||||
#define B_TO_FLOAT(x) float(x)
|
||||
#endif
|
||||
#if defined(DATA_C_BF16)
|
||||
#define C_TO_FLOAT(x) bf16_to_fp32(uint32_t(x))
|
||||
#else
|
||||
#define C_TO_FLOAT(x) float(x)
|
||||
#endif
|
||||
#if defined(DATA_D_BF16)
|
||||
#define FLOAT_TO_D(x) D_TYPE(fp32_to_bf16(x))
|
||||
#else
|
||||
#define FLOAT_TO_D(x) D_TYPE(x)
|
||||
#endif
|
||||
|
||||
float e8m0_to_fp32(uint8_t x) {
|
||||
uint32_t bits;
|
||||
|
||||
|
||||
@@ -184,10 +184,10 @@ void main() {
|
||||
|
||||
if (op_on_b) {
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] =
|
||||
D_TYPE(FLOAT_TYPE(OP(float(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]));
|
||||
FLOAT_TO_D(OP(B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])) * A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]));
|
||||
} else {
|
||||
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] =
|
||||
D_TYPE(FLOAT_TYPE(OP(float(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
||||
FLOAT_TO_D(OP(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])) * B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
||||
}
|
||||
|
||||
idx += num_threads;
|
||||
@@ -200,6 +200,6 @@ void main() {
|
||||
const uint a_idx = get_aoffset() + src0_idx(idx);
|
||||
const uint d_idx = get_doffset() + dst_idx(idx);
|
||||
|
||||
data_d[d_idx] = D_TYPE(OP(float(data_a[a_idx])));
|
||||
data_d[d_idx] = FLOAT_TO_D(OP(A_TO_FLOAT(data_a[a_idx])));
|
||||
#endif
|
||||
}
|
||||
@@ -863,9 +863,13 @@ void process_shaders() {
|
||||
string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
|
||||
for (bool use_subgroups : {false, true}) {
|
||||
const std::string suffix = use_subgroups ? "_subgroup" : "";
|
||||
const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}});
|
||||
const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"C_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}});
|
||||
const auto rms_bf16_dict = merge_maps(base_dict, {{"A_TYPE", "uint16_t"}, {"B_TYPE", "float"}, {"C_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}});
|
||||
string_to_spv("rms_norm_f32" + suffix, "rms_norm.comp", rms_dict);
|
||||
string_to_spv("rms_norm_bf16" + suffix, "rms_norm.comp", rms_bf16_dict);
|
||||
string_to_spv("rms_norm_mul_add_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_ADD_FUSION", "1"}}));
|
||||
string_to_spv("rms_norm_mul_add_bf16_f32" + suffix, "rms_norm.comp", merge_maps(rms_bf16_dict, {{"RMS_NORM_ADD_FUSION", "1"}}));
|
||||
string_to_spv("rms_norm_mul_add_bf16_bf16" + suffix, "rms_norm.comp", merge_maps({{"C_TYPE", "uint16_t"}, {"DATA_C_BF16", "1"}, {"RMS_NORM_ADD_FUSION", "1"}}, rms_bf16_dict));
|
||||
string_to_spv("rms_norm_set_rows_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SET_ROWS_FUSION", "1"}}));
|
||||
string_to_spv("rms_norm_set_rows_f32_f16" + suffix, "rms_norm.comp", merge_maps({{"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}, rms_dict)); // keep rms_dict last so 'merge_maps' overwrites the value for key 'D_TYPE'
|
||||
string_to_spv("rms_norm_mul_rope_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
|
||||
@@ -934,6 +938,10 @@ void process_shaders() {
|
||||
}
|
||||
}
|
||||
}
|
||||
for (std::string op : {"add", "sub", "mul", "div"}) {
|
||||
string_to_spv(op + "_bf16_f32_bf16", op + ".comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"FLOAT_TYPE", "float"}, {"ADD_RMS" , "0"}});
|
||||
string_to_spv(op + "_bf16_bf16_bf16", op + ".comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_B_BF16", "1"}, {"DATA_D_BF16", "1"}, {"FLOAT_TYPE", "float"}, {"ADD_RMS" , "0"}});
|
||||
}
|
||||
|
||||
string_to_spv("sub_f32", "sub.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
|
||||
@@ -969,8 +977,6 @@ void process_shaders() {
|
||||
|
||||
string_to_spv("repeat_i16", "repeat.comp", {{"A_TYPE", "int16_t"}, {"D_TYPE", "int16_t"}});
|
||||
|
||||
string_to_spv("scale_f32", "scale.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
|
||||
string_to_spv("pad_f32", "pad.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("pad_reflect_1d_f32", "pad_reflect_1d.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
|
||||
@@ -984,70 +990,36 @@ void process_shaders() {
|
||||
|
||||
string_to_spv("upscale_f32", "upscale.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
|
||||
string_to_spv("exp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_exp"}});
|
||||
string_to_spv("exp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_exp"}});
|
||||
string_to_spv("expm1_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_expm1"}});
|
||||
string_to_spv("expm1_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_expm1"}});
|
||||
const std::vector<std::pair<std::string, std::map<std::string, std::string>>> float_types = {
|
||||
{"f32", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}},
|
||||
{"f16", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}},
|
||||
{"bf16", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}}},
|
||||
};
|
||||
for (const auto & [t, dict] : float_types) {
|
||||
for (const std::string op : {"exp", "expm1", "gelu", "gelu_erf", "gelu_quick", "silu", "relu", "sqr", "sqrt", "sin",
|
||||
"cos", "clamp", "leaky_relu", "neg", "tanh", "sigmoid", "hardsigmoid", "hardswish", "abs",
|
||||
"elu", "xielu", "sgn", "softplus", "step", "round", "ceil", "floor", "trunc"}) {
|
||||
string_to_spv(op + "_" + t, "unary.comp", merge_maps(dict, {{"OP", "op_" + op}}));
|
||||
}
|
||||
for (const std::string op : {"geglu", "reglu", "swiglu", "swiglu_oai", "swiglu_clamp", "geglu_erf", "geglu_quick"}) {
|
||||
string_to_spv(op + "_" + t, op + ".comp", dict);
|
||||
}
|
||||
string_to_spv("log_" + t, "log.comp", dict);
|
||||
string_to_spv("scale_" + t, "scale.comp", merge_maps(dict, {{"FLOAT_TYPE", "float"}}));
|
||||
|
||||
string_to_spv("log_f16", "log.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("log_f32", "log.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("gelu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu"}});
|
||||
string_to_spv("gelu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu"}});
|
||||
string_to_spv("gelu_erf_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu_erf"}});
|
||||
string_to_spv("gelu_erf_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu_erf"}});
|
||||
string_to_spv("gelu_quick_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu_quick"}});
|
||||
string_to_spv("gelu_quick_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu_quick"}});
|
||||
string_to_spv("silu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_silu"}});
|
||||
string_to_spv("silu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_silu"}});
|
||||
string_to_spv("relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_relu"}});
|
||||
string_to_spv("relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_relu"}});
|
||||
string_to_spv("sqr_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqr"}});
|
||||
string_to_spv("sqr_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqr"}});
|
||||
string_to_spv("sqrt_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqrt"}});
|
||||
string_to_spv("sqrt_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqrt"}});
|
||||
string_to_spv("sin_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sin"}});
|
||||
string_to_spv("sin_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sin"}});
|
||||
string_to_spv("cos_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_cos"}});
|
||||
string_to_spv("cos_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_cos"}});
|
||||
string_to_spv("clamp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_clamp"}});
|
||||
string_to_spv("clamp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_clamp"}});
|
||||
string_to_spv("leaky_relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_leaky_relu"}});
|
||||
string_to_spv("leaky_relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_leaky_relu"}});
|
||||
string_to_spv("neg_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_neg"}});
|
||||
string_to_spv("neg_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_neg"}});
|
||||
string_to_spv("tanh_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_tanh"}});
|
||||
string_to_spv("tanh_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_tanh"}});
|
||||
string_to_spv("sigmoid_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sigmoid"}});
|
||||
string_to_spv("sigmoid_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sigmoid"}});
|
||||
string_to_spv("hardsigmoid_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_hardsigmoid"}});
|
||||
string_to_spv("hardsigmoid_f32","unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_hardsigmoid"}});
|
||||
string_to_spv("hardswish_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_hardswish"}});
|
||||
string_to_spv("hardswish_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_hardswish"}});
|
||||
string_to_spv("abs_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_abs"}});
|
||||
string_to_spv("abs_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_abs"}});
|
||||
string_to_spv("elu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_elu"}});
|
||||
string_to_spv("elu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_elu"}});
|
||||
string_to_spv("xielu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_xielu"}});
|
||||
string_to_spv("xielu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_xielu"}});
|
||||
string_to_spv("sgn_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sgn"}});
|
||||
string_to_spv("sgn_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sgn"}});
|
||||
auto mul_dict = merge_maps(dict, {{"B_TYPE", dict.at("A_TYPE")}, {"FLOAT_TYPE", "float"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
if (t == "bf16") {
|
||||
mul_dict["DATA_B_BF16"] = "1";
|
||||
}
|
||||
for (const std::string op : {"gelu", "sigmoid", "silu", "softplus"}) {
|
||||
string_to_spv(op + "_mul_" + t, "unary.comp", merge_maps(mul_dict, {{"OP", "op_" + op}}));
|
||||
}
|
||||
}
|
||||
|
||||
string_to_spv("tri_f16", "tri.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("tri_f32", "tri.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("diag_f16", "diag.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("diag_f32", "diag.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
|
||||
string_to_spv("softplus_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_softplus"}});
|
||||
string_to_spv("softplus_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_softplus"}});
|
||||
|
||||
string_to_spv("gelu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
string_to_spv("gelu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
string_to_spv("sigmoid_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
string_to_spv("sigmoid_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
string_to_spv("silu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
string_to_spv("silu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
string_to_spv("softplus_mul_f32","unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
string_to_spv("softplus_mul_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}});
|
||||
string_to_spv("tri_16", "tri.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});
|
||||
string_to_spv("tri_32", "tri.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}});
|
||||
string_to_spv("diag_16", "diag.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});
|
||||
string_to_spv("diag_32", "diag.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}});
|
||||
|
||||
string_to_spv("add1_f16_f16", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("add1_f16_f32", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
|
||||
@@ -1055,31 +1027,6 @@ void process_shaders() {
|
||||
string_to_spv("arange_f32", "arange.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("fill_f32", "fill.comp", {{"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("fill_f16", "fill.comp", {{"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("step_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_step"}});
|
||||
string_to_spv("step_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_step"}});
|
||||
string_to_spv("round_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_round"}});
|
||||
string_to_spv("round_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_round"}});
|
||||
string_to_spv("ceil_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_ceil"}});
|
||||
string_to_spv("ceil_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_ceil"}});
|
||||
string_to_spv("floor_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_floor"}});
|
||||
string_to_spv("floor_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_floor"}});
|
||||
string_to_spv("trunc_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_trunc"}});
|
||||
string_to_spv("trunc_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_trunc"}});
|
||||
|
||||
string_to_spv("geglu_f16", "geglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("geglu_f32", "geglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("reglu_f16", "reglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("reglu_f32", "reglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("swiglu_f16", "swiglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("swiglu_f32", "swiglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("swiglu_oai_f16", "swiglu_oai.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("swiglu_oai_f32", "swiglu_oai.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("swiglu_clamp_f16", "swiglu_clamp.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("swiglu_clamp_f32", "swiglu_clamp.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("geglu_erf_f16", "geglu_erf.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("geglu_erf_f32", "geglu_erf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
string_to_spv("geglu_quick_f16","geglu_quick.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
|
||||
string_to_spv("geglu_quick_f32","geglu_quick.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
|
||||
string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
|
||||
|
||||
+1
-1
@@ -5280,7 +5280,7 @@ struct ggml_tensor * ggml_tri(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
enum ggml_tri_type type) {
|
||||
GGML_ASSERT(a->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16);
|
||||
|
||||
GGML_ASSERT(ggml_is_contiguous(a));
|
||||
GGML_ASSERT(a->ne[0] == a->ne[1]);
|
||||
|
||||
+18
-10
@@ -10770,7 +10770,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
|
||||
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
|
||||
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
|
||||
for (int v : {0, 1}) {
|
||||
for (float alpha : {.5f, 1.702f}) {
|
||||
for (float limit : {2.0f, 7.0f}) {
|
||||
@@ -10780,7 +10780,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
|
||||
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
|
||||
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {
|
||||
for (int v : {0, 1}) {
|
||||
for (float limit : {2.0f, 10.0f}) {
|
||||
test_cases.emplace_back(new test_swiglu_clamp(type, { 128, 2, 2, 2 }, v, limit));
|
||||
@@ -11574,6 +11574,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f));
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {100, 10, 10, 10}, 0.5f, 0.0f));
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {10, 10, 10, 10}, 2.0f, 1.0f));
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {100, 10, 10, 10}, 0.5f, 0.0f));
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 0.0f, false, 1)); // view test
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test
|
||||
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace
|
||||
@@ -11684,6 +11687,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 1536, 1, 1, 1 }, 1e-6f, false, false, false, true));
|
||||
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 256, 4, 1, 1 }, 1e-6f, false, false, false, true));
|
||||
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, true, false, false, GGML_TYPE_F32));
|
||||
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, true, false, false, GGML_TYPE_F32, GGML_TYPE_BF16));
|
||||
// the ADD operand is the BF16 input itself, added in place
|
||||
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, false, true, false, GGML_TYPE_F32));
|
||||
|
||||
@@ -12471,7 +12475,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
|
||||
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
|
||||
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) {
|
||||
test_cases.emplace_back(new test_sqr (type));
|
||||
test_cases.emplace_back(new test_sqrt (type));
|
||||
test_cases.emplace_back(new test_log (type));
|
||||
@@ -12946,10 +12950,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_xielu(GGML_TYPE_F16, { 512, 16, 1, 1 }));
|
||||
test_cases.emplace_back(new test_xielu(GGML_TYPE_BF16, { 512, 16, 1, 1 }));
|
||||
|
||||
test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER));
|
||||
test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG));
|
||||
test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER));
|
||||
test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG));
|
||||
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
|
||||
test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER, type));
|
||||
test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG, type));
|
||||
test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER, type));
|
||||
test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG, type));
|
||||
}
|
||||
test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER));
|
||||
test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER_DIAG));
|
||||
test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_UPPER));
|
||||
@@ -12963,9 +12969,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_fill(-152.0f, GGML_TYPE_F16, { 800, 600, 4, 4 }));
|
||||
test_cases.emplace_back(new test_fill(3.5f, GGML_TYPE_F16, { 2048, 512, 2, 2 }));
|
||||
|
||||
test_cases.emplace_back(new test_diag());
|
||||
test_cases.emplace_back(new test_diag(GGML_TYPE_F32, { 79, 1, 19, 13 }));
|
||||
test_cases.emplace_back(new test_diag(GGML_TYPE_F32, { 256, 1, 8, 16 }));
|
||||
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
|
||||
test_cases.emplace_back(new test_diag(type));
|
||||
test_cases.emplace_back(new test_diag(type, { 79, 1, 19, 13 }));
|
||||
test_cases.emplace_back(new test_diag(type, { 256, 1, 8, 16 }));
|
||||
}
|
||||
|
||||
test_cases.emplace_back(new test_solve_tri());
|
||||
test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 11, 11, 1, 1 }, { 5, 11, 1, 1 }));
|
||||
|
||||
Reference in new issue
Block a user