Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ggml : update ggml_prec specification (#26675)

Browse Source 
* ggml : update ggml_prec specification

[no ci]

* cont : add GGML_PREC_BF16

* cont : rework API

* cont : use new API

* cont : swap arg order

* cont : support for MUL_MAT_ID

* cont : fix accidental remove of "break;"

* cont : return bools, add doc TAG_GGML_PREC, clean-up

* cont : add search tag

* cont : ws
This commit is contained in:
Georgi Gerganov
2026-09-08 09:06:24 +03:00
committed by GitHub
parent 9dcf84e5ae
commit 5a6caa05fc
10 changed files with 134 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/include/ggml.h
+56 -7
View File
@@ -433,10 +433,21 @@ extern "C" {
GGML_TYPE_COUNT = 43,
};
// precision
// [TAG_GGML_PREC]
// this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op
// the declared types can be:
// - result accumulation type
// - source tensor data representation type
// - etc.
// the precision parameters are stored as ggml_tensor.op_params to the respective ops
enum ggml_prec {
GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default
GGML_PREC_F32 = 10,
GGML_PREC_UNDEFINED = 0,
GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED
GGML_PREC_F32 = 10,
GGML_PREC_BF16 = 15,
GGML_PREC_F16 = 20,
GGML_PREC_Q8 = 30,
GGML_PREC_Q4 = 40,
};
// op hint
@@ -1429,6 +1440,42 @@ extern "C" {
struct ggml_tensor * b,
float eps);
// [TAG_GGML_PREC]
// set the minimum required accumulator type for the implementation to use during the compute
// for example:
// - GGML_PREC_F32 - requires accumulation of the results in F32
// - GGML_PREC_BF16 - can accumulate the results in BF16, F32
// - GGML_PREC_F16 - can accumulate the results in F16, F32
// - GGML_PREC_Q8 - not allowed
// - GGML_PREC_Q4 - not allowed
//
// return false on faliure
GGML_API bool ggml_prec_set_acc(
struct ggml_tensor * a,
enum ggml_prec prec);
// [TAG_GGML_PREC]
// set the smallest rank that the implementation can use to internally convert the src[idx] data to
// ranks in decreasing order:
// - GGML_PREC_F32 - GGML_TYPE_F32
// - GGML_PREC_BF16 - GGML_TYPE_BF16
// - GGML_PREC_F16 - GGML_TYPE_F16,
// - GGML_PREC_Q8 - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc.
// - GGML_PREC_Q4 - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc.
//
// for example:
// - ggml_prec_set_src(a, GGML_PREC_Q8, 1):
// - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0
// - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4
// - ggml_prec_set_src(a, GGML_PREC_Q4, 1):
// - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc.
//
// return false on faliure
GGML_API bool ggml_prec_set_src(
struct ggml_tensor * a,
enum ggml_prec prec,
int idx);
// A: k columns, n rows => [ne03, ne02, n, k]
// B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
// result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
@@ -1439,9 +1486,10 @@ extern "C" {
// change the precision of a matrix multiplication
// set to GGML_PREC_F32 for higher precision (useful for phi-2)
GGML_API void ggml_mul_mat_set_prec(
GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec(
struct ggml_tensor * a,
enum ggml_prec prec);
enum ggml_prec prec),
"use ggml_prec_set_acc() instead");
// change the hint of a matrix multiplication
GGML_API void ggml_mul_mat_set_hint(
@@ -2446,9 +2494,10 @@ extern "C" {
float max_bias,
float logit_softcap);
GGML_API void ggml_flash_attn_ext_set_prec(
GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(
struct ggml_tensor * a,
enum ggml_prec prec);
enum ggml_prec prec),
"use ggml_prec_set_acc() instead");
GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(
const struct ggml_tensor * a);
ggml/src/ggml-impl.h
+12
View File
@@ -160,6 +160,18 @@ static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t
return ((const float *)(tensor->op_params))[i];
}
// [TAG_GGML_PREC]
// - GGML_OP_MUL_MAT
// 0 - acc
// 1 - hint
// 2 - src0 precision
// 3 - src1 precision
//
// - GGML_OP_MUL_MAT_ID
// 0 - acc
// 1 - hint
// 2 - src0 precision
// 3 - src1 precision
static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {
assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));
((int32_t *)(tensor->op_params))[i] = value;
ggml/src/ggml.c
+51
View File
@@ -3277,6 +3277,57 @@ struct ggml_tensor * ggml_l2_norm_inplace(
return ggml_l2_norm_impl(ctx, a, eps, true);
}
// ggml_prec
bool ggml_prec_set_acc(
struct ggml_tensor * a,
enum ggml_prec prec) {
switch (a->op) {
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
{
const int32_t prec_i32 = (int32_t) prec;
ggml_set_op_params_i32(a, 0, prec_i32);
}
break;
case GGML_OP_FLASH_ATTN_EXT:
{
const int32_t prec_i32 = (int32_t) prec;
ggml_set_op_params_i32(a, 3, prec_i32);
}
break;
default:
return false;
};
return true;
}
bool ggml_prec_set_src(
struct ggml_tensor * a,
enum ggml_prec prec,
int idx) {
GGML_ASSERT(idx >= 0 && idx < GGML_MAX_SRC);
switch (a->op) {
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
{
if (idx != 1) {
return false;
}
const int32_t prec_i32 = (int32_t) prec;
ggml_set_op_params_i32(a, 2 + idx, prec_i32);
}
break;
default:
return false;
};
return true;
}
// ggml_mul_mat
static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
src/llama-graph.cpp
+6 -6
View File
@@ -1926,7 +1926,7 @@ ggml_tensor * llm_graph_context::build_ffn(
cur = build_lora_mm(down, cur);
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
}
}
@@ -2024,7 +2024,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (probs_in == nullptr) {
logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {
ggml_mul_mat_set_prec(logits, GGML_PREC_F32);
ggml_prec_set_acc(logits, GGML_PREC_F32);
}
cb(logits, "ffn_moe_logits", il);
} else {
@@ -2636,7 +2636,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
ggml_flash_attn_ext_add_sinks(cur, sinks);
GGML_ASSERT(n_kv_max >= 0 && n_kv_max <= INT32_MAX);
ggml_flash_attn_ext_set_n_kv_max(cur, static_cast<int32_t>(n_kv_max));
ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (v_mla) {
#if 0
@@ -2662,7 +2662,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
// note: this op tends to require high floating point range
// while for some models F16 is enough, for others it is not, so we default to F32 here
ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
ggml_prec_set_acc(kq, GGML_PREC_F32);
if (arch == LLM_ARCH_GROK) {
// need to do the following:
@@ -2895,7 +2895,7 @@ ggml_tensor * llm_graph_context::build_attn(
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators
cur = build_lora_mm(wo, cur);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (wo_s) {
cur = ggml_mul(ctx0, cur, wo_s);
}
@@ -2982,7 +2982,7 @@ ggml_tensor * llm_graph_context::build_attn(
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
// GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators
cur = build_lora_mm(wo, cur);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (wo_s) {
cur = ggml_mul(ctx0, cur, wo_s);
}
src/models/minimax-m3.cpp
+3 -3
View File
@@ -191,7 +191,7 @@ ggml_tensor * llama_model_minimax_m3::graph::build_attn_msa_fa(
ggml_tensor * o = ggml_flash_attn_ext(ctx0, q, k, v, mask, kq_scale,
hparams.f_max_alibi_bias, 0.0f);
ggml_flash_attn_ext_set_prec(o, GGML_PREC_F32);
ggml_prec_set_acc(o, GGML_PREC_F32);
cb(o, "msa_fattn", il);
// [D, Gp, R, C] -> [D, Gp, C, R] -> [n_embd, T]
@@ -389,7 +389,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
ggml_tensor * iq4 = ggml_reshape_4d(ctx0, iq, n_idx_dim, Hd, 1, ns);
ggml_tensor * sc = ggml_mul_mat(ctx0,
ggml_reshape_4d(ctx0, ikp, n_idx_dim, n_ps, 1, ns), iq4);
ggml_mul_mat_set_prec(sc, GGML_PREC_F32);
ggml_prec_set_acc(sc, GGML_PREC_F32);
// unmapped positions come out -inf, so they can never rank into the top-k
sc = ggml_add_inplace(ctx0, sc,
ggml_reshape_4d(ctx0, msa->pos_mask, n_ps, 1, 1, ns));
@@ -471,7 +471,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
ggml_tensor * sc = ggml_mul_mat(ctx0, ikp,
ggml_reshape_2d(ctx0, iq_s, n_idx_dim, Hd*n_tps));
// indexer scores run in F32
ggml_mul_mat_set_prec(sc, GGML_PREC_F32);
ggml_prec_set_acc(sc, GGML_PREC_F32);
sc = ggml_reshape_3d(ctx0, sc, n_ps, Hd, n_tps);
// unmapped positions (holes, padding, empty cells) come out -inf
sc = ggml_add_inplace(ctx0, sc, pm_s);
tests/test-backend-ops.cpp
+1 -1
View File
@@ -7659,7 +7659,7 @@ struct test_flash_attn_ext : public test_case {
ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hsk), max_bias, logit_softcap);
ggml_flash_attn_ext_add_sinks(out, s);
ggml_flash_attn_ext_set_n_kv_max(out, n_kv_max);
ggml_flash_attn_ext_set_prec (out, prec);
ggml_prec_set_acc(out, prec);
ggml_set_name(out, "out");
return out;
tools/mtmd/clip.cpp
+2 -2
View File
@@ -780,7 +780,7 @@ ggml_tensor * clip_graph::build_attn(
}
cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (sinks != nullptr) {
ggml_flash_attn_ext_add_sinks(cur, sinks);
}
@@ -793,7 +793,7 @@ ggml_tensor * clip_graph::build_attn(
ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
// F32 may not needed for vision encoders?
// ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
// ggml_prec_set_acc(kq, GGML_PREC_F32);
kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f);
if (sinks != nullptr) {
tools/mtmd/models/mimovl.cpp
+1 -1
View File
@@ -2,7 +2,7 @@
ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const {
ggml_tensor * cur = ggml_mul_mat(ctx0, w, x);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
return cur;
}
tools/mtmd/models/qwen3tts-spkenc.cpp
+1 -1
View File
@@ -27,7 +27,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens
ggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC);
ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out]
ggml_mul_mat_set_prec(y, GGML_PREC_F32);
ggml_prec_set_acc(y, GGML_PREC_F32);
ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1);
y = ggml_add(ctx0, y, b2d);
tools/tuning/fa-vec.cpp
+1 -1
View File
@@ -56,7 +56,7 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) {
ggml_set_name(m, "m");
ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f / sqrtf((float) s.dk), 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32);
ggml_prec_set_acc(out, GGML_PREC_F32);
ggml_set_name(out, "out");
return out;