From d3064fddce83046ef66eded524cf00947d5166f3 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Tue, 17 Nov 2026 22:25:01 +0800 Subject: [PATCH] convert : fix FP8 qwen4exp checkpoints (n-gram table scale, block-scaled FP8) (#32476) * convert : fix FP8 n-gram table export for qwen4exp * convert : dequantize block-scaled FP8 weight_scale_inv in ModelOpt checkpoints --- conversion/base.py | 22 ++++++++ conversion/qwen4exp.py | 38 ++++++++++--- src/llama-model-saver.cpp | 1 + src/llama-model.h | 1 + src/models/qwen4exp.cpp | 5 ++ tests/test_conversion_fp8.py | 101 +++++++++++++++++++++++++++++++++++ 6 files changed, 160 insertions(+), 8 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index a95ecc008..7cfbff321 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -653,6 +653,28 @@ class ModelBase: raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported") elif quant_method in ("modelopt", "modelopt_mixed"): dequant_fp8() + # FP8_PB_WO: one scale per block, the block size follows from the two shapes + for name in self.model_tensors.keys(): + if not name.endswith(".weight_scale_inv"): + continue + weight_name = name.removesuffix("_scale_inv") + if weight_name not in self.model_tensors: + tensors_to_remove.append(name) + continue + w = self.model_tensors[weight_name] + s = self.model_tensors[name] + if w().dtype != torch.float8_e4m3fn: + continue + w_shape, s_shape = w().shape, s().shape + block_size = None + if prod(s_shape) != 1: + if len(s_shape) != len(w_shape) or any(ws % ss for ws, ss in zip(w_shape, s_shape)): + raise ValueError(f"FP8 weight {weight_name!r} {list(w_shape)} is not tiled by its scale {list(s_shape)}") + block_size = [ws // ss for ws, ss in zip(w_shape, s_shape)] + self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs) + tensors_to_remove.append(name) + if self._fp8_as_q8: + self._fp8_dequantized.add(weight_name) elif quant_method is not None: raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}") diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py index 9680d2937..fe49cc03d 100644 --- a/conversion/qwen4exp.py +++ b/conversion/qwen4exp.py @@ -8,7 +8,7 @@ from torch import Tensor import gguf import numpy as np -from .base import ModelBase +from .base import LazyTorchTensor, ModelBase from .qwen import _LinearAttentionVReorderBase, _Qwen35MRopeMixin from .qwen3vl import Qwen3VLVisionModel @@ -41,6 +41,18 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): self._ple_shards: dict[int, str] = {} self._ple_row_dim: int | None = None self._mtp_fc: dict[str, Tensor] = {} + self._ple_scale: float | None = None + + def dequant_model(self): + # FP8 checkpoints: all n-gram shards share one scale, stored under the parent module name + # there is no "ngram_embedding.weight", so the generic FP8 pass drops it or cannot map it + for name in list(self.model_tensors): + if name.endswith(".ngram_embedding.weight_scale"): + scale = LazyTorchTensor.to_eager(self.model_tensors.pop(name)()).float().flatten() + if scale.numel() != 1: + raise ValueError(f"PLE scale {name!r} must be a scalar, got {scale.numel()} values") + self._ple_scale = float(scale[0]) + super().dequant_model() @classmethod def filter_tensors(cls, item): @@ -203,21 +215,31 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): f"PLE shard {shard} has row dim {int(shape[-1])}, expected {self._ple_row_dim}") rows += int(shape[0]) + # an FP8 table keeps its bytes and gets a scale tensor, any other type has the scale folded in + keep_fp8 = name in self._fp8_e4m3_preserved table = gguf.LazyChunkedTensor( - [self._load_ple_shard(shard) for shard in shards], + [self._load_ple_shard(shard, keep_fp8) for shard in shards], shape=(rows, self._ple_row_dim), - dtype=np.float32, + dtype=np.uint8 if keep_fp8 else np.float32, ) gguf_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD] - return [(gguf_name + ".weight", cast(Tensor, table))] + out = [(gguf_name + ".weight", cast(Tensor, table))] + if keep_fp8 and self._ple_scale is not None: + out.append((gguf_name + ".scale", torch.tensor([self._ple_scale], dtype=torch.float32))) + return out - def _load_ple_shard(self, name: str): + def _load_ple_shard(self, name: str, keep_fp8: bool): def load() -> np.ndarray: - from .base import LazyTorchTensor - # a fresh lazy tensor every call, or to_eager() memoizes every shard eager = LazyTorchTensor.to_eager(self.model_tensors[name]()) - return eager.to(torch.float32).contiguous().numpy() + if keep_fp8: + if eager.dtype != torch.float8_e4m3fn: + raise ValueError(f"PLE shard {name} is {eager.dtype}, expected FP8 like the other shards") + return eager.contiguous().view(torch.uint8).numpy() + data = eager.to(torch.float32) + if self._ple_scale is not None: + data = data * self._ple_scale + return data.contiguous().numpy() return load def prepare_tensors(self): diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 4e9e23d10..338baa8f7 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -571,6 +571,7 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->hc_head_base); add_tensor(model->hc_head_scale); add_tensor(model->per_layer_tok_embd); + add_tensor(model->per_layer_tok_embd_s); add_tensor(model->per_layer_model_proj); add_tensor(model->per_layer_proj_norm); add_tensor(model->value_embd); diff --git a/src/llama-model.h b/src/llama-model.h index 19b4c4288..f3a665410 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -881,6 +881,7 @@ struct llama_model { struct ggml_tensor * altup_proj = nullptr; struct ggml_tensor * altup_unembd_proj = nullptr; struct ggml_tensor * per_layer_tok_embd = nullptr; + struct ggml_tensor * per_layer_tok_embd_s = nullptr; // optional scalar scale of an FP8 table // limite value embeddings struct ggml_tensor * value_embd = nullptr; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 181e4dca4..faa4a928f 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -215,6 +215,8 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), { hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY); + // an FP8 table keeps its checkpoint bytes, so its per-tensor scale is applied after the gather + per_layer_tok_embd_s = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "scale"), { 1 }, TENSOR_NOT_REQUIRED); } auto load_block = [&](int il, int flags) { @@ -1408,6 +1410,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple( // gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows); + if (model.per_layer_tok_embd_s) { + emb = ggml_mul(ctx0, emb, model.per_layer_tok_embd_s); + } emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens); cb(emb, "ple_embd", -1); diff --git a/tests/test_conversion_fp8.py b/tests/test_conversion_fp8.py index 96387ad88..e191c8a55 100644 --- a/tests/test_conversion_fp8.py +++ b/tests/test_conversion_fp8.py @@ -25,6 +25,7 @@ from conversion.llama import LlamaModel from conversion.nemotron import NemotronHModel from conversion.phi import PhiMoeModel from conversion.qwen import Qwen2MoeModel, Qwen3NextModel +from conversion.qwen4exp import Qwen4ExpTextModel class ForcedF16Model(Qwen2MoeModel): @@ -366,6 +367,27 @@ class TestFP8Conversion(unittest.TestCase): self.assertEqual(dtype, gguf.GGMLQuantizationType.Q8_0) np.testing.assert_allclose(gguf.dequantize(data, dtype), np.full((4, 32), 0.25), atol=1e-4) + def test_modelopt_block_scale_inv(self): + name = "model.layers.0.self_attn.q_proj.weight" + weight = (torch.arange(32, dtype=torch.float32).reshape(4, 8) - 16).to(torch.float8_e4m3fn) + scale = torch.tensor([[0.25, 0.5], [1.0, 2.0]]) + expected = weight.float() * scale.repeat_interleave(2, 0).repeat_interleave(4, 1) + for eager in (False, True): + with self.subTest(eager=eager): + result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": scale}, eager=eager, check_lazy=not eager) + self.assertEqual(set(result), {"blk.0.attn_q.weight"}) + dtype, data = result["blk.0.attn_q.weight"] + self.assertEqual(dtype, gguf.GGMLQuantizationType.F16) + np.testing.assert_array_equal(data, expected.numpy()) + + with self.assertRaisesRegex(ValueError, "not tiled"): + self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.ones((3, 2))}) + + # a scalar scale keeps the FP8 bytes, as with weight_scale + result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.tensor([0.25])}, check_lazy=True) + self.assertEqual(result["blk.0.attn_q.weight"][0], gguf.GGMLQuantizationType.F8_E4M3) + np.testing.assert_array_equal(result["blk.0.attn_q.scale"][1], [0.25]) + class TestInkling2PackedConversion(unittest.TestCase): """Conversion of the packed Inkling2 expert weights (see _dequant_expert_chunk in conversion/inkling2.py). @@ -675,5 +697,84 @@ class TestInkling2TextNames(unittest.TestCase): np.testing.assert_array_equal(renamed[name], actual[name], name) +class TestQwen4ExpPleScale(unittest.TestCase): + """FP8 checkpoints store the n-gram table as shards with one shared scale under the parent module name.""" + + PREFIX = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding." + SHARDS = (torch.tensor([[1., -2.], [3., 4.]]), torch.tensor([[5., 6.], [-7., 8.]])) + TABLE = "per_layer_token_embd.weight" + SCALE = "per_layer_token_embd.scale" + + def convert(self, ftype=gguf.LlamaFileType.GUESSED, scaled=True, quant_method="modelopt", **kwargs): + # shard 1 first: the row order must follow the shard index + tensors = {self.PREFIX + f"shard_{i}.weight": self.SHARDS[i].to(torch.float8_e4m3fn) for i in (1, 0)} + if scaled: + tensors[self.PREFIX + "weight_scale"] = torch.tensor([0.25], dtype=torch.bfloat16) + hparams = { + "model_type": "qwen4exp", + "architectures": ["Qwen4ExpForCausalLM"], + "num_hidden_layers": 2, + "mtp_num_hidden_layers": 1, + "split_ngram_parts": 2, + "quantization_config": {"quant_method": quant_method}, + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "config.json").write_text(json.dumps(hparams)) + save_file(tensors, str(path / "model.safetensors")) + model = Qwen4ExpTextModel(path, ftype, path / "out.gguf", hparams=hparams, **kwargs) + model.prepare_tensors() + result = {} + for name, info in model.gguf_writer.tensors[0].items(): + if isinstance(info.tensor, gguf.LazyChunkedTensor): + # written like the GGUF writer does, a chunked tensor has no other read path + with (path / "tensor.bin").open("wb") as f: + info.tensor.tofile(f) + data = np.frombuffer((path / "tensor.bin").read_bytes(), dtype=info.tensor.dtype).reshape(info.tensor.shape) + else: + data = np.asarray(gguf.LazyNumpyTensor.to_eager(info.tensor)) + result[name] = (info.dtype, data) + return result + + def test_fp8_table_keeps_bytes_and_scale(self): + raw = torch.cat(self.SHARDS).to(torch.float8_e4m3fn).view(torch.uint8).numpy() + for kwargs, names in ( + ({}, {self.TABLE, self.SCALE}), + ({"quant_method": "fp8"}, {self.TABLE, self.SCALE}), + ({"ple_only": True}, {self.TABLE}), + ({"no_ple": True}, {self.SCALE}), + ): + with self.subTest(**kwargs): + result = self.convert(**kwargs) + self.assertEqual(set(result), names) + if self.TABLE in result: + dtype, data = result[self.TABLE] + self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3) + np.testing.assert_array_equal(data, raw) + if self.SCALE in result: + dtype, data = result[self.SCALE] + self.assertEqual(dtype, gguf.GGMLQuantizationType.F32) + np.testing.assert_array_equal(data, [0.25]) + + result = self.convert(scaled=False) + self.assertEqual(set(result), {self.TABLE}) + np.testing.assert_array_equal(result[self.TABLE][1], raw) + + def test_scale_is_folded_into_other_types(self): + values = torch.cat(self.SHARDS).numpy() + for kwargs, qtype in ( + ({"ftype": gguf.LlamaFileType.ALL_F32}, gguf.GGMLQuantizationType.F32), + ({"ftype": gguf.LlamaFileType.MOSTLY_F16}, gguf.GGMLQuantizationType.F16), + ({"fp8_as_q8": True}, gguf.GGMLQuantizationType.F16), + ): + for scaled in (False, True): + with self.subTest(scaled=scaled, **kwargs): + result = self.convert(scaled=scaled, **kwargs) + self.assertEqual(set(result), {self.TABLE}) + dtype, data = result[self.TABLE] + self.assertEqual(dtype, qtype) + np.testing.assert_array_equal(data, values * (0.25 if scaled else 1.)) + + if __name__ == "__main__": unittest.main()