Aman GuptaandAman Gupta <aman>
c8a2417d7b
* CUDA: experimental native mxfp4 support for blackwell * optimize load_tiles * optimize quantize_mxfp4 * cleanup * first pass review: formatting * use interleaved layout for mma * mmq: add assert for size * use __nv_fp4x4_e2m1 * use iter_k as 512, cleanup * Use 1200 as blackwell instead of 1000 * address review comments * mmq: fix stride * quantize.cu: use reference impl of e8m0 scale * address review comments * add 120f-virtual + minor fixes --------- Co-authored-by: Aman Gupta <aman>
42 lines
1.7 KiB
Plaintext
42 lines
1.7 KiB
Plaintext
#pragma once
|
|
|
|
#include "common.cuh"
|
|
#include "mmq.cuh"
|
|
|
|
#include <cstdint>
|
|
|
|
#define CUDA_QUANTIZE_BLOCK_SIZE 256
|
|
#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128
|
|
|
|
static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access.");
|
|
static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access.");
|
|
|
|
typedef void (*quantize_cuda_t)(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_row_q8_1_cuda(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_mmq_q8_1_cuda(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_mmq_mxfp4_cuda(const float * x,
|
|
const int32_t * ids,
|
|
void * vy,
|
|
ggml_type type_src0,
|
|
int64_t ne00,
|
|
int64_t s01,
|
|
int64_t s02,
|
|
int64_t s03,
|
|
int64_t ne0,
|
|
int64_t ne1,
|
|
int64_t ne2,
|
|
int64_t ne3,
|
|
cudaStream_t stream);
|