Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

model : BailingMoE3 Support (#26608)

Browse Source 
* Adding support for bailingmoe3

* Adds speculative decoding support

* Make BailingMoE3 safe gate metadata optional

* bailingmoe3: apply trained SwiGLU clamps

* common: fix Bailing V3 tool argument parsing

* llama-model-saver, instantiate float vector metadata writer

* bailingmoe3: support Q-LoRA (Ling-3.0-tiny)

Ling-3.0-flash sets q_lora_rank: None and projects Q directly, so the current
implementation loads a single ATTN_Q tensor. Ling-3.0-tiny sets q_lora_rank: 256
and routes Q through a LoRA bottleneck instead:

    q_a_proj -> q_a_layernorm -> q_b_proj

Conversion therefore failed with:

    ValueError: Can not map tensor 'model.layers.3.attention.q_a_layernorm.weight'

Add the missing path, mirroring the existing deepseek2 MLA implementation:

  * constants.py       - add ATTN_Q_A / ATTN_Q_B / ATTN_Q_A_NORM to BAILINGMOE3
  * tensor_mapping.py  - map model.layers.{bid}.attention.q_{a,b}_proj and
                         q_a_layernorm
  * conversion         - emit attention.q_lora_rank when the config has it
  * bailingmoe3.cpp    - read n_lora_q; create the Q-LoRA tensors and build Q
                         through the bottleneck when q_lora_rank > 0

Everything is gated on q_lora_rank > 0. Ling-3.0-flash's config has no
q_lora_rank, the converter only emits the key when present, hparams.n_lora_q
defaults to 0, and get_key(..., required=false) leaves the target untouched when
the key is absent - so flash keeps taking the existing direct-Q branch.

The LoRA path produces the same shape as the direct projection, so the
nope/rope split, RoPE application and wk_b absorption downstream are unchanged.

* small mtp change

* bailingmoe3: support separate MTP GGUF and Q-LoRA MTP

* gguf: remove duplicate add_kda_gate_lower_bound definition

---------

Co-authored-by: bloomer <bloomer@booper.brushtail.me>
Co-authored-by: Dyluhn <dylanranejohnston1@gmail.com>
This commit is contained in:
Toby
2026-08-17 09:49:49 +02:00
committed by GitHub
co-authored by bloomer Dyluhn
parent 4197155add
commit 3733366720
18 changed files with 938 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/constants.py
+50 -1
View File
@@ -261,6 +261,7 @@ class Keys:
class KDA:
HEAD_DIM = "{arch}.kda.head_dim"
SAFE_GATE = "{arch}.kda.safe_gate"
GATE_LOWER_BOUND = "{arch}.kda.gate_lower_bound"
class WKV:
@@ -552,6 +553,7 @@ class MODEL_ARCH(IntEnum):
PLM = auto()
BAILINGMOE = auto()
BAILINGMOE2 = auto()
BAILINGMOE3 = auto()
DOTS1 = auto()
ARCEE = auto()
AFMOE = auto()
@@ -1267,6 +1269,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.PLM: "plm",
MODEL_ARCH.BAILINGMOE: "bailingmoe",
MODEL_ARCH.BAILINGMOE2: "bailingmoe2",
MODEL_ARCH.BAILINGMOE3: "bailingmoe3",
MODEL_ARCH.DOTS1: "dots1",
MODEL_ARCH.ARCEE: "arcee",
MODEL_ARCH.AFMOE: "afmoe",
@@ -4234,6 +4237,50 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
MODEL_TENSOR.LAYER_OUT_NORM,
],
MODEL_ARCH.BAILINGMOE3: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
MODEL_TENSOR.OUTPUT,
MODEL_TENSOR.ATTN_NORM,
MODEL_TENSOR.ATTN_Q,
MODEL_TENSOR.ATTN_Q_A,
MODEL_TENSOR.ATTN_Q_B,
MODEL_TENSOR.ATTN_Q_A_NORM,
MODEL_TENSOR.ATTN_K,
MODEL_TENSOR.ATTN_V,
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.ATTN_GATE,
MODEL_TENSOR.ATTN_KV_A_MQA,
MODEL_TENSOR.ATTN_KV_B,
MODEL_TENSOR.ATTN_K_B,
MODEL_TENSOR.ATTN_V_B,
MODEL_TENSOR.ATTN_KV_A_NORM,
MODEL_TENSOR.FFN_NORM,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FFN_GATE_INP,
MODEL_TENSOR.FFN_GATE_EXP,
MODEL_TENSOR.FFN_DOWN_EXP,
MODEL_TENSOR.FFN_UP_EXP,
MODEL_TENSOR.FFN_GATE_SHEXP,
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
MODEL_TENSOR.FFN_EXP_PROBS_B,
MODEL_TENSOR.SSM_CONV1D_Q,
MODEL_TENSOR.SSM_CONV1D_K,
MODEL_TENSOR.SSM_CONV1D_V,
MODEL_TENSOR.SSM_F_A,
MODEL_TENSOR.SSM_BETA,
MODEL_TENSOR.SSM_A,
MODEL_TENSOR.SSM_G_A,
MODEL_TENSOR.SSM_DT,
MODEL_TENSOR.SSM_NORM,
MODEL_TENSOR.NEXTN_EH_PROJ,
MODEL_TENSOR.NEXTN_ENORM,
MODEL_TENSOR.NEXTN_HNORM,
MODEL_TENSOR.LAYER_OUT_NORM,
],
MODEL_ARCH.DOTS1: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -5487,7 +5534,9 @@ KEY_SSM_GROUP_COUNT = Keys.SSM.GROUP_COUNT
KEY_SSM_DT_B_C_RMS = Keys.SSM.DT_B_C_RMS
# KDA
KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM
KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM
KEY_KDA_SAFE_GATE = Keys.KDA.SAFE_GATE
KEY_KDA_GATE_LOWER_BOUND = Keys.KDA.GATE_LOWER_BOUND
# tokenization
KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL
gguf-py/gguf/gguf_writer.py
+6 -3
View File
@@ -1103,9 +1103,6 @@ class GGUFWriter:
def add_ssm_dt_b_c_rms(self, value: bool) -> None:
self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value)
def add_kda_gate_lower_bound(self, value: float) -> None:
self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value)
def add_expert_latent_length(self, value: int) -> None:
self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value)
@@ -1121,6 +1118,12 @@ class GGUFWriter:
def add_kda_head_dim(self, value: int) -> None:
self.add_uint32(Keys.KDA.HEAD_DIM.format(arch=self.arch), value)
def add_kda_safe_gate(self, value: bool) -> None:
self.add_bool(Keys.KDA.SAFE_GATE.format(arch=self.arch), value)
def add_kda_gate_lower_bound(self, value: float) -> None:
self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value)
def add_tokenizer_model(self, model: str) -> None:
self.add_string(Keys.Tokenizer.MODEL, model)
gguf-py/gguf/tensor_mapping.py
+20
View File
@@ -255,6 +255,7 @@ class TensorNameMap:
# Attention query
MODEL_TENSOR.ATTN_Q: (
"model.layers.{bid}.self_attn.q_proj", # llama-hf nemotron olmoe olmo2 phimoe
"model.layers.{bid}.attention.q_proj", # bailingmoe3
"layers.{bid}.self_attn.q_proj", # embeddinggemma
"model.layers.{bid}.self_attn.q_proj_no_perm", # llama-custom
"layers.{bid}.attention.wq", # llama-pth
@@ -275,6 +276,7 @@ class TensorNameMap:
# Attention key
MODEL_TENSOR.ATTN_K: (
"model.layers.{bid}.self_attn.k_proj", # llama-hf nemotron olmoe olmo2 phimoe
"model.layers.{bid}.attention.k_proj", # bailingmoe3
"layers.{bid}.self_attn.k_proj", # embeddinggemma
"model.layers.{bid}.self_attn.k_proj_no_perm", # llama-custom
"layers.{bid}.attention.wk", # llama-pth
@@ -296,6 +298,7 @@ class TensorNameMap:
# Attention value
MODEL_TENSOR.ATTN_V: (
"model.layers.{bid}.self_attn.v_proj", # llama-hf nemotron olmoe olmo2 phimoe
"model.layers.{bid}.attention.v_proj", # bailingmoe3
"layers.{bid}.self_attn.v_proj", # embeddinggemma
"layers.{bid}.attention.wv", # llama-pth
"encoder.layer.{bid}.attention.self.value", # bert
@@ -321,6 +324,8 @@ class TensorNameMap:
"transformer.h.{bid}.self_attention.dense", # falcon
"h.{bid}.self_attention.dense", # bloom
"model.layers.{bid}.self_attn.o_proj", # llama-hf nemotron olmoe olmo2 phimoe
"model.layers.{bid}.attention.o_proj", # bailingmoe3
"model.layers.{bid}.attention.dense", # bailingmoe3 MLA
"layers.{bid}.self_attn.o_proj", # embeddinggemma
"model.layers.{bid}.self_attn.out_proj", # lfm2 minimax-01
"model.layers.{bid}.self_attn.linear_attn", # deci
@@ -834,6 +839,7 @@ class TensorNameMap:
"model.layers.{bid}.linear_attn.dt_proj", # qwen3next
"backbone.layers.{bid}.mixer.dt", # nemotron-h-moe
"model.layers.{bid}.self_attn.dt_proj", # kimi
"model.layers.{bid}.attention.dt_proj", # bailingmoe3
),
MODEL_TENSOR.SSM_DT_NORM: (
@@ -848,6 +854,7 @@ class TensorNameMap:
"model.layers.layers.{bid}.mixer.A_log", # plamo2
"model.layers.{bid}.linear_attn.A_log", # qwen3next
"model.layers.{bid}.self_attn.A_log", # kimi
"model.layers.{bid}.attention.A_log", # bailingmoe3
),
MODEL_TENSOR.SSM_B_NORM: (
@@ -874,6 +881,7 @@ class TensorNameMap:
"model.layers.{bid}.linear_attn.norm", # qwen3next
"backbone.layers.{bid}.mixer.norm", # mamba2
"model.layers.{bid}.self_attn.o_norm", # kimi
"model.layers.{bid}.attention.o_norm", # bailingmoe3
),
MODEL_TENSOR.SSM_OUT: (
@@ -895,12 +903,15 @@ class TensorNameMap:
# Kimi Linear KDA (using SSM_ prefix for consistency)
MODEL_TENSOR.SSM_CONV1D_Q: (
"model.layers.{bid}.self_attn.q_conv1d",
"model.layers.{bid}.attention.q_conv1d",
),
MODEL_TENSOR.SSM_CONV1D_K: (
"model.layers.{bid}.self_attn.k_conv1d",
"model.layers.{bid}.attention.k_conv1d",
),
MODEL_TENSOR.SSM_CONV1D_V: (
"model.layers.{bid}.self_attn.v_conv1d",
"model.layers.{bid}.attention.v_conv1d",
),
MODEL_TENSOR.SSM_F_A: (
"model.layers.{bid}.self_attn.f_a_proj",
@@ -911,6 +922,7 @@ class TensorNameMap:
MODEL_TENSOR.SSM_BETA: (
"model.layers.{bid}.linear_attn.in_proj_b", # qwen3.5
"model.layers.{bid}.self_attn.b_proj", # Kimi Linear
"model.layers.{bid}.attention.b_proj", # bailingmoe3
),
# Kimi K3 latent MoE: routed experts operate in a down-projected space
MODEL_TENSOR.FFN_ROUTED_DOWN: (
@@ -1103,40 +1115,48 @@ class TensorNameMap:
MODEL_TENSOR.ATTN_Q_A: (
"model.layers.{bid}.self_attn.q_a_proj", # deepseek2
"model.layers.{bid}.attention.q_a_proj", # bailingmoe3 (Ling-3.0-tiny)
"layers.{bid}.attention.wq_a", # mistral-large
),
MODEL_TENSOR.ATTN_Q_B: (
"model.layers.{bid}.self_attn.q_b_proj", # deepseek2
"model.layers.{bid}.attention.q_b_proj", # bailingmoe3 (Ling-3.0-tiny)
"layers.{bid}.attention.wq_b", # mistral-large
),
MODEL_TENSOR.ATTN_KV_A_MQA: (
"model.layers.{bid}.self_attn.kv_a_proj_with_mqa", # deepseek2
"model.layers.{bid}.attention.kv_a_proj_with_mqa", # bailingmoe3
"layers.{bid}.attention.wkv_a_with_mqa", # mistral-large
),
MODEL_TENSOR.ATTN_KV_B: (
"model.layers.{bid}.self_attn.kv_b_proj", # deepseek2
"model.layers.{bid}.attention.kv_b_proj", # bailingmoe3
),
MODEL_TENSOR.ATTN_K_B: (
"model.layers.{bid}.self_attn.k_b_proj", # deepseek2
"model.layers.{bid}.attention.k_b_proj", # bailingmoe3
"layers.{bid}.attention.k_b_proj", # mistral-large
),
MODEL_TENSOR.ATTN_V_B: (
"model.layers.{bid}.self_attn.v_b_proj", # deepseek2
"model.layers.{bid}.attention.v_b_proj", # bailingmoe3
"layers.{bid}.attention.v_b_proj", # mistral-large
),
MODEL_TENSOR.ATTN_Q_A_NORM: (
"model.layers.{bid}.self_attn.q_a_layernorm", # deepseek2
"model.layers.{bid}.attention.q_a_layernorm", # bailingmoe3 (Ling-3.0-tiny)
"layers.{bid}.attention.q_a_norm", # mistral-large
),
MODEL_TENSOR.ATTN_KV_A_NORM: (
"model.layers.{bid}.self_attn.kv_a_layernorm", # deepseek2
"model.layers.{bid}.attention.kv_a_layernorm", # bailingmoe3
"layers.{bid}.attention.kv_a_norm", # mistral-large
),