From a68799c6094020a020cee72895b8df36b3b81098 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sun, 15 Nov 2026 17:39:59 +0100 Subject: [PATCH] model: support MiniMax-M3.1 (#32301) --- conversion/__init__.py | 1 + conversion/base.py | 11 +++++--- conversion/minimax.py | 55 ++++++++++++++++++++++++++++++++++++++- src/llama-model.cpp | 4 +-- src/models/dflash.cpp | 2 ++ src/models/minimax-m3.cpp | 24 ++++++++++++++--- 6 files changed, 88 insertions(+), 9 deletions(-) diff --git a/conversion/__init__.py b/conversion/__init__.py index afdd73e4c..fb6b8464c 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -220,6 +220,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "MiniMaxM2ForCausalLM": "minimax", "MiniMaxM3SparseForCausalLM": "minimax", "MiniMaxM3SparseForConditionalGeneration": "minimax", + "DSparkMiniMaxDraftModel": "minimax", "Ministral3ForCausalLM": "mistral3", "Ministral3Model": "mistral3", "Mistral3ForConditionalGeneration": "mistral3", diff --git a/conversion/base.py b/conversion/base.py index ac231ac94..7558fdcb4 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -398,6 +398,9 @@ class ModelBase: return data / scale.float() def dequant_simple(weight: Tensor, scale: Tensor, block_size: Sequence[int] | None = None) -> Tensor: + if scale.dtype == torch.uint8: + # E8M0 exponent stored as raw bytes (MXFP8) + scale = (scale.float() - 127.0).exp2() scale = scale.float() # ModelOpt block scales are pre-shaped for broadcasting, e.g. [out/128, 1, in/128, 1] @@ -538,7 +541,7 @@ class ModelBase: s = self.model_tensors[name] self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s()) tensors_to_remove.append(name) - elif quant_method in ("fp8", "bsgemm-moe"): # bsgemm-moe: drama-3, same block layout + elif quant_method in ("fp8", "bsgemm-moe", "mxfp8"): # bsgemm-moe: drama-3, same block layout block_size = quant_config.get("weight_block_size") for name in self.model_tensors.keys(): if name.endswith("_scale_inv"): @@ -1357,10 +1360,10 @@ class ModelBase: consumed.append(input_scale_name) # Check if this is a per-expert tensor - m = re.search(r'\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$', name) + m = re.search(r'\.experts\.(\d+)\.(gate_proj|up_proj|down_proj|w1|w2|w3)\.weight$', name) if m: expert_id = int(m.group(1)) - proj_type = m.group(2) + proj_type = {"w1": "gate_proj", "w2": "down_proj", "w3": "up_proj"}.get(m.group(2), m.group(2)) bid_m = re.search(r'\.layers\.(\d+)\.', name) bid = int(bid_m.group(1)) if bid_m else 0 key = (bid, proj_type) @@ -1451,6 +1454,8 @@ class ModelBase: quant_layers = quant_config.get("quantized_layers", quant_layers) or {} nvfp4_compressed_tensors = self._is_nvfp4_compressed_tensors(quant_method, quant_format, quant_groups) + # e.g. MXFP8 dense weights with compressed-tensors NVFP4 experts + nvfp4_compressed_tensors |= quantization_config.get("moe_quant_format") == "nvfp4-pack-quantized" self._nvfp4_global_algo = quant_algo diff --git a/conversion/minimax.py b/conversion/minimax.py index 4d856fab8..c9f512f5f 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + from typing import Iterable, Sequence, TYPE_CHECKING import torch @@ -8,6 +10,7 @@ if TYPE_CHECKING: from torch import Tensor from .base import ModelBase, TextModel, MmprojModel, gguf, logger +from .qwen import DSparkModel @ModelBase.register("MiniMaxText01ForCausalLM") @@ -170,6 +173,15 @@ class MiniMaxM2Model(TextModel): class MiniMaxM3Model(MiniMaxM2Model): model_arch = gguf.MODEL_ARCH.MINIMAXM3 + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # transformers 5.x folds sparse_attention_config and moe_layer_freq into other keys + with open(self.dir_model / "config.json", "r", encoding="utf-8") as f: + raw_hparams = json.load(f) + for key, value in raw_hparams.get("text_config", raw_hparams).items(): + self.hparams.setdefault(key, value) + def tensor_force_quant(self, name, new_name, bid, n_dims): if ".indexer." in new_name: return gguf.GGMLQuantizationType.F32 @@ -188,6 +200,7 @@ class MiniMaxM3Model(MiniMaxM2Model): self.gguf_writer.add_indexer_top_k(sac["sparse_topk_blocks"]) self.gguf_writer.add_indexer_block_size(sac["sparse_block_size"]) self.gguf_writer.add_indexer_local_blocks(sac["sparse_local_block"]) + self.gguf_writer.add_indexer_types([bool(v) for v in sac["sparse_attention_freq"]]) moe_layer_freq = self.find_hparam(["moe_layer_freq"]) n_dense = 0 @@ -201,7 +214,47 @@ class MiniMaxM3Model(MiniMaxM2Model): def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None): # Gemma-style (1 + w) RMSNorm: bake the +1 in so llama.cpp can use plain RMSNorm if name.endswith("norm.weight"): - data_torch = data_torch + 1.0 + data_torch = data_torch.float() + 1.0 + + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("DSparkMiniMaxDraftModel") +class MiniMaxM3DSparkModel(DSparkModel): + model_arch = gguf.MODEL_ARCH.DFLASH + + def __init__(self, dir_model, *args, **kwargs): + # transformers drops the draft keys when it parses the minimax_m3_vl config + with open(dir_model / "config.json", "r", encoding="utf-8") as f: + kwargs["hparams"] = json.load(f) + super().__init__(dir_model, *args, **kwargs) + + # draft layers use a dense FFN, the MoE keys are copied from the target config + self.hparams["intermediate_size"] = self.hparams["dspark_ffn_hidden_size"] + for key in ("num_local_experts", "num_experts_per_tok", "scoring_func"): + self.hparams.pop(key, None) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + self.gguf_writer.add_hidden_act(self.hparams["hidden_act"]) + + layer_types = self.hparams["layer_types"] + self.gguf_writer.add_sliding_window(self.hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in layer_types]) + + @classmethod + def filter_tensors(cls, item): + name, gen = item + name = name.removeprefix("language_model.model.dspark.").replace(".decoder_layer.", ".") + if name == "final_norm.weight": + name = "norm.weight" + return super().filter_tensors((name, gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None): + # Gemma-style (1 + w) RMSNorm, same as the target + if name.endswith("norm.weight"): + data_torch = data_torch.float() + 1.0 yield from super().modify_tensors(data_torch, name, bid) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 42924b938..194bbb92b 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2970,9 +2970,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } break; case LLM_ARCH_MINIMAX_M3: { - // sparse (MSA) layers carry an indexer key cache, but leading dense layers do not + // only sparse (MSA) layers carry an indexer key cache llama_kv_cache::layer_filter_cb filter_idx = - [&](int32_t il) { return (uint32_t) il >= hparams.n_layer_dense_lead; }; + [&](int32_t il) { return (uint32_t) il < hparams.n_layer() && hparams.is_indexer_full(il); }; res = new llama_kv_cache_msa( *this, diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 96360e404..000222f4c 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -15,6 +15,8 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { if (ml.get_key(LLM_KV_HIDDEN_ACT, hidden_act, false)) { if (hidden_act == "gelu" || hidden_act == "gelu_pytorch_tanh") { hparams.llm_ffn_op = LLM_FFN_GELU; + } else if (hidden_act == "swigluoai") { + hparams.llm_ffn_op = LLM_FFN_SWIGLU_OAI_MOE; } else if (hidden_act != "silu") { throw std::runtime_error("unsupported DFlash hidden activation: " + hidden_act); } diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 466985cc4..4f1184590 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -24,6 +24,12 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size); ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks); + // BC for MiniMax-M3 GGUFs without indexer_types metadata: leading dense layers have no indexer + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + hparams.is_indexer_full_impl[il] = il >= hparams.n_layer_dense_lead; + } + ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false); + msa_p = { /*.blk =*/ (int) hparams.indexer_block_size, /*.topk_blocks =*/ (int) hparams.indexer_top_k, @@ -79,8 +85,9 @@ void llama_model_minimax_m3::load_arch_tensors(llama_model_loader &) { layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, 0); layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + } - // indexer + if (hparams.is_indexer_full(i)) { layer.index_q_proj = create_tensor(tn(LLM_TENSOR_INDEXER_Q_PROJ, "weight", i), {n_embd, hparams.indexer_n_head * hparams.indexer_head_size}, 0); layer.index_k_proj = create_tensor(tn(LLM_TENSOR_INDEXER_K_PROJ, "weight", i), {n_embd, hparams.indexer_head_size}, 0); layer.index_q_norm = create_tensor(tn(LLM_TENSOR_INDEXER_Q_NORM, "weight", i), {hparams.indexer_head_size}, 0); @@ -300,7 +307,11 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap ggml_tensor * inp_out_ids = build_inp_out_ids(); + const bool extract_last = (size_t) n_layer < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[n_layer]; + for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; // self-attention @@ -325,7 +336,7 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - const bool is_sparse = msa_enabled && il >= (int) hparams.n_layer_dense_lead; + const bool is_sparse = msa_enabled && hparams.is_indexer_full(il); if (!is_sparse) { cur = build_attn(inp_attn, model.layers[il].wo, NULL, model.layers[il].wo_s, @@ -529,7 +540,7 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap } } - if (il == n_layer - 1 && inp_out_ids) { + if (il == n_layer - 1 && inp_out_ids && !extract_last) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } @@ -589,6 +600,13 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap inpL = cur; } + if (extract_last) { + res->t_layer_inp[n_layer] = inpL; + if (inp_out_ids) { + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + } + cur = inpL; cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);