Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

model : BailingMoE3 Support (#26608)

Browse Source 
* Adding support for bailingmoe3

* Adds speculative decoding support

* Make BailingMoE3 safe gate metadata optional

* bailingmoe3: apply trained SwiGLU clamps

* common: fix Bailing V3 tool argument parsing

* llama-model-saver, instantiate float vector metadata writer

* bailingmoe3: support Q-LoRA (Ling-3.0-tiny)

Ling-3.0-flash sets q_lora_rank: None and projects Q directly, so the current
implementation loads a single ATTN_Q tensor. Ling-3.0-tiny sets q_lora_rank: 256
and routes Q through a LoRA bottleneck instead:

    q_a_proj -> q_a_layernorm -> q_b_proj

Conversion therefore failed with:

    ValueError: Can not map tensor 'model.layers.3.attention.q_a_layernorm.weight'

Add the missing path, mirroring the existing deepseek2 MLA implementation:

  * constants.py       - add ATTN_Q_A / ATTN_Q_B / ATTN_Q_A_NORM to BAILINGMOE3
  * tensor_mapping.py  - map model.layers.{bid}.attention.q_{a,b}_proj and
                         q_a_layernorm
  * conversion         - emit attention.q_lora_rank when the config has it
  * bailingmoe3.cpp    - read n_lora_q; create the Q-LoRA tensors and build Q
                         through the bottleneck when q_lora_rank > 0

Everything is gated on q_lora_rank > 0. Ling-3.0-flash's config has no
q_lora_rank, the converter only emits the key when present, hparams.n_lora_q
defaults to 0, and get_key(..., required=false) leaves the target untouched when
the key is absent - so flash keeps taking the existing direct-Q branch.

The LoRA path produces the same shape as the direct projection, so the
nope/rope split, RoPE application and wk_b absorption downstream are unchanged.

* small mtp change

* bailingmoe3: support separate MTP GGUF and Q-LoRA MTP

* gguf: remove duplicate add_kda_gate_lower_bound definition

---------

Co-authored-by: bloomer <bloomer@booper.brushtail.me>
Co-authored-by: Dyluhn <dylanranejohnston1@gmail.com>
This commit is contained in:
Toby
2026-08-17 09:49:49 +02:00
committed by GitHub
co-authored by bloomer Dyluhn
parent 4197155add
commit 3733366720
18 changed files with 938 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-chat-auto-parser.cpp
+64 -1
View File
@@ -90,6 +90,7 @@ static void test_normalize_quotes_with_embedded_quotes(testing & t);
// TAG_WITH_TAGGED argument parsing tests
static void test_tagged_args_with_embedded_quotes(testing & t);
static void test_bailing_v3_tool_format(testing & t);
static void test_role_markers_all_templates(testing & t);
@@ -118,6 +119,7 @@ int main(int argc, char * argv[]) {
t.test("standard_json_tools", test_standard_json_tools_formats);
t.test("normalize_quotes_to_json", test_normalize_quotes_to_json);
t.test("tagged_args_embedded_quotes", test_tagged_args_with_embedded_quotes);
t.test("bailing_v3", test_bailing_v3_tool_format);
t.test("role_markers_all_templates", test_role_markers_all_templates);
return t.summary();
@@ -2081,6 +2083,68 @@ static void test_role_markers_all_templates(testing & t) {
}
}
static void test_bailing_v3_tool_format(testing & t) {
const std::string template_source = R"JINJA(
{# Bailing V3 chat template #}
{%- if tools %}{{ tools | tojson }}{%- endif %}
{%- for message in messages %}
{%- if message.role == "user" %}
{{- '<role>HUMAN</role>' + message.content + '<|role_end|>' }}
{%- elif message.role == "assistant" %}
{{- '<role>ASSISTANT</role>' }}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- set tc = tool_call.function %}
{{- '<tool_call>' + tc.name }}
{%- for k, v in tc.arguments.items() %}
{{- '<arg_key>' + k + '</arg_key>' }}
{{- '\n<arg_value>' + v + '</arg_value>' }}
{%- endfor %}
{{- '\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|role_end|>' }}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}{{- '<role>ASSISTANT</role>' }}{%- endif %}
)JINJA";
common_chat_template tmpl(template_source, "", "");
struct autoparser analysis;
analysis.analyze_template(tmpl);
t.assert_equal("arg_value_suffix", "</arg_value>", analysis.tools.arguments.value_suffix);
t.assert_true("intertag whitespace", analysis.tools.arguments.tolerate_intertag_whitespace);
generation_params inputs;
inputs.tools = json::array({
{
{ "type", "function" },
{ "function", {
{ "name", "test_function_name" },
{ "parameters", {
{ "type", "object" },
{ "properties", {
{ "param1", { { "type", "string" } } },
{ "param2", { { "type", "string" } } },
} },
} },
} },
},
});
inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE;
auto parser = analysis.build_parser(inputs, "");
const std::string output =
"<tool_call>test_function_name\n"
"<arg_key>param1</arg_key>\n"
"<arg_value>value1</arg_value>"
"<arg_key>param2</arg_key>\n"
"<arg_value>value2</arg_value>\n"
"</tool_call>";
common_peg_parse_context ctx(output, COMMON_PEG_PARSE_FLAG_LENIENT);
t.assert_true("multi-argument tool call", parser.parse(ctx).success());
}
// Test that reproduces the Seed-OSS template issue with embedded quotes
static void test_tagged_args_with_embedded_quotes(testing & t) {
json tools = build_edit_tool();
@@ -2198,4 +2262,3 @@ static void test_tagged_args_with_embedded_quotes(testing & t) {
}
}
}
tests/test-llama-archs.cpp
+14 -1
View File
@@ -105,6 +105,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_KIMI_LINEAR
|| arch == LLM_ARCH_BAILINGMOE3
|| arch == LLM_ARCH_KIMI_K3
|| arch == LLM_ARCH_MISTRAL4) {
n_embd = 128;
@@ -146,7 +147,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_KIMI_K3) {
arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR ||
arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3) {
GGML_ASSERT(n_layer >= 2);
std::vector<uint32_t> n_head_per_layer;
n_head_per_layer.reserve(n_layer);
@@ -165,6 +167,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_KIMI_LINEAR
|| arch == LLM_ARCH_BAILINGMOE3
|| arch == LLM_ARCH_KIMI_K3
|| arch == LLM_ARCH_MISTRAL4) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, uint32_t(576));
@@ -244,6 +247,12 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));
ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f}));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f}));
}
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
@@ -361,6 +370,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_EXAONE_MOE:
case LLM_ARCH_BAILINGMOE:
case LLM_ARCH_BAILINGMOE2:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_DOTS1:
case LLM_ARCH_AFMOE:
case LLM_ARCH_ERNIE4_5:
@@ -610,6 +620,9 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg
}
const std::string config_name = moe ? "MoE" : "Dense";
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
if (arch == LLM_ARCH_BAILINGMOE3) {
GGML_ASSERT(gguf_remove_key(gguf_ctx.get(), "bailingmoe3.kda.safe_gate") >= 0);
}
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_cpu;
std::vector<float> logits_cpu;
for (device_config & dc : dev_configs) {