Frosty40
ac79caa7ce
* sycl: support reordered Q4_K and Q5_K MoE MUL_MAT_ID Extend reordered-weight handling to fused MoE MUL_MAT_ID for Q4_K and Q5_K expert tensors and add Q5_K reordered DMMV coverage. Unsupported 3D reorder cases now fall back instead of aborting. * sycl: extend MoE reorder to Q6_K mul_mat_id
61 lines
2.3 KiB
C++
61 lines
2.3 KiB
C++
//
|
|
// MIT license
|
|
// Copyright (C) 2024 Intel Corporation
|
|
// SPDX-License-Identifier: MIT
|
|
//
|
|
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
|
|
#ifndef GGML_SYCL_MMVQ_HPP
|
|
#define GGML_SYCL_MMVQ_HPP
|
|
|
|
#include "common.hpp"
|
|
|
|
|
|
void ggml_sycl_op_mul_mat_vec_q(
|
|
ggml_backend_sycl_context & ctx,
|
|
const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,
|
|
const char *src0_dd_i, const float *src1_ddf_i, const char *src1_ddq_i,
|
|
float *dst_dd_i, const int64_t row_low, const int64_t row_high,
|
|
const int64_t src1_ncols, const int64_t src1_padded_row_size,
|
|
const dpct::queue_ptr &stream);
|
|
|
|
// Requires standard (non-reorder) block layout for src0.
|
|
// Returns false if src0_type isn't handled; caller should fall back.
|
|
bool ggml_sycl_mul_mat_vec_q_id(
|
|
enum ggml_type src0_type,
|
|
const void * vx_base, // start of stacked expert weights
|
|
const void * vy, // pre-quantized src1 (Q8_1)
|
|
const int32_t * ids_dev, // device-side int32, length n_experts_used
|
|
float * dst_base,
|
|
int ncols,
|
|
int nrows,
|
|
int n_experts_used,
|
|
size_t expert_weight_stride, // bytes between experts in vx_base
|
|
size_t dst_row_stride, // bytes between dst rows
|
|
size_t src1_row_stride, // 0 = shared src1, else per-expert stride in bytes
|
|
dpct::queue_ptr stream);
|
|
|
|
// Reorder (SoA) variant of the fused MoE expert GEMV.
|
|
// vx_base: each expert slice (stride expert_weight_stride == src0->nb[2]) is a self-contained reorder/SoA layout.
|
|
// vy: src1 quantized with quantize_and_reorder_q8_1_soa (per-row SoA). Returns false if src0_type isn't handled.
|
|
bool ggml_sycl_mul_mat_vec_q_id_reorder(
|
|
enum ggml_type src0_type,
|
|
const void * vx_base,
|
|
const void * vy,
|
|
const int32_t * ids_dev,
|
|
float * dst_base,
|
|
int ncols,
|
|
int nrows,
|
|
int n_experts_used,
|
|
size_t expert_weight_stride,
|
|
size_t dst_row_stride,
|
|
size_t src1_row_stride,
|
|
dpct::queue_ptr stream);
|
|
|
|
#endif // GGML_SYCL_MMVQ_HPP
|