convert : fix FP8 qwen4exp checkpoints (n-gram table scale, block-scaled FP8) (#32476)
* convert : fix FP8 n-gram table export for qwen4exp * convert : dequantize block-scaled FP8 weight_scale_inv in ModelOpt checkpoints
This commit is contained in:
1 parent
7f106b5459
commit
d3064fddce
6 files changed
+160
-8
No files matched your search
@@ -653,6 +653,28 @@ class ModelBase:
|
||||
raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
|
||||
elif quant_method in ("modelopt", "modelopt_mixed"):
|
||||
dequant_fp8()
|
||||
# FP8_PB_WO: one scale per block, the block size follows from the two shapes
|
||||
for name in self.model_tensors.keys():
|
||||
if not name.endswith(".weight_scale_inv"):
|
||||
continue
|
||||
weight_name = name.removesuffix("_scale_inv")
|
||||
if weight_name not in self.model_tensors:
|
||||
tensors_to_remove.append(name)
|
||||
continue
|
||||
w = self.model_tensors[weight_name]
|
||||
s = self.model_tensors[name]
|
||||
if w().dtype != torch.float8_e4m3fn:
|
||||
continue
|
||||
w_shape, s_shape = w().shape, s().shape
|
||||
block_size = None
|
||||
if prod(s_shape) != 1:
|
||||
if len(s_shape) != len(w_shape) or any(ws % ss for ws, ss in zip(w_shape, s_shape)):
|
||||
raise ValueError(f"FP8 weight {weight_name!r} {list(w_shape)} is not tiled by its scale {list(s_shape)}")
|
||||
block_size = [ws // ss for ws, ss in zip(w_shape, s_shape)]
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
|
||||
tensors_to_remove.append(name)
|
||||
if self._fp8_as_q8:
|
||||
self._fp8_dequantized.add(weight_name)
|
||||
elif quant_method is not None:
|
||||
raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")
|
||||
|
||||
|
||||
+30
-8
@@ -8,7 +8,7 @@ from torch import Tensor
|
||||
import gguf
|
||||
import numpy as np
|
||||
|
||||
from .base import ModelBase
|
||||
from .base import LazyTorchTensor, ModelBase
|
||||
from .qwen import _LinearAttentionVReorderBase, _Qwen35MRopeMixin
|
||||
from .qwen3vl import Qwen3VLVisionModel
|
||||
|
||||
@@ -41,6 +41,18 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
self._ple_shards: dict[int, str] = {}
|
||||
self._ple_row_dim: int | None = None
|
||||
self._mtp_fc: dict[str, Tensor] = {}
|
||||
self._ple_scale: float | None = None
|
||||
|
||||
def dequant_model(self):
|
||||
# FP8 checkpoints: all n-gram shards share one scale, stored under the parent module name
|
||||
# there is no "ngram_embedding.weight", so the generic FP8 pass drops it or cannot map it
|
||||
for name in list(self.model_tensors):
|
||||
if name.endswith(".ngram_embedding.weight_scale"):
|
||||
scale = LazyTorchTensor.to_eager(self.model_tensors.pop(name)()).float().flatten()
|
||||
if scale.numel() != 1:
|
||||
raise ValueError(f"PLE scale {name!r} must be a scalar, got {scale.numel()} values")
|
||||
self._ple_scale = float(scale[0])
|
||||
super().dequant_model()
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
@@ -203,21 +215,31 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
f"PLE shard {shard} has row dim {int(shape[-1])}, expected {self._ple_row_dim}")
|
||||
rows += int(shape[0])
|
||||
|
||||
# an FP8 table keeps its bytes and gets a scale tensor, any other type has the scale folded in
|
||||
keep_fp8 = name in self._fp8_e4m3_preserved
|
||||
table = gguf.LazyChunkedTensor(
|
||||
[self._load_ple_shard(shard) for shard in shards],
|
||||
[self._load_ple_shard(shard, keep_fp8) for shard in shards],
|
||||
shape=(rows, self._ple_row_dim),
|
||||
dtype=np.float32,
|
||||
dtype=np.uint8 if keep_fp8 else np.float32,
|
||||
)
|
||||
gguf_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD]
|
||||
return [(gguf_name + ".weight", cast(Tensor, table))]
|
||||
out = [(gguf_name + ".weight", cast(Tensor, table))]
|
||||
if keep_fp8 and self._ple_scale is not None:
|
||||
out.append((gguf_name + ".scale", torch.tensor([self._ple_scale], dtype=torch.float32)))
|
||||
return out
|
||||
|
||||
def _load_ple_shard(self, name: str):
|
||||
def _load_ple_shard(self, name: str, keep_fp8: bool):
|
||||
def load() -> np.ndarray:
|
||||
from .base import LazyTorchTensor
|
||||
|
||||
# a fresh lazy tensor every call, or to_eager() memoizes every shard
|
||||
eager = LazyTorchTensor.to_eager(self.model_tensors[name]())
|
||||
return eager.to(torch.float32).contiguous().numpy()
|
||||
if keep_fp8:
|
||||
if eager.dtype != torch.float8_e4m3fn:
|
||||
raise ValueError(f"PLE shard {name} is {eager.dtype}, expected FP8 like the other shards")
|
||||
return eager.contiguous().view(torch.uint8).numpy()
|
||||
data = eager.to(torch.float32)
|
||||
if self._ple_scale is not None:
|
||||
data = data * self._ple_scale
|
||||
return data.contiguous().numpy()
|
||||
return load
|
||||
|
||||
def prepare_tensors(self):
|
||||
|
||||
@@ -571,6 +571,7 @@ void llama_model_saver::add_tensors_from_model() {
|
||||
add_tensor(model->hc_head_base);
|
||||
add_tensor(model->hc_head_scale);
|
||||
add_tensor(model->per_layer_tok_embd);
|
||||
add_tensor(model->per_layer_tok_embd_s);
|
||||
add_tensor(model->per_layer_model_proj);
|
||||
add_tensor(model->per_layer_proj_norm);
|
||||
add_tensor(model->value_embd);
|
||||
|
||||
@@ -881,6 +881,7 @@ struct llama_model {
|
||||
struct ggml_tensor * altup_proj = nullptr;
|
||||
struct ggml_tensor * altup_unembd_proj = nullptr;
|
||||
struct ggml_tensor * per_layer_tok_embd = nullptr;
|
||||
struct ggml_tensor * per_layer_tok_embd_s = nullptr; // optional scalar scale of an FP8 table
|
||||
|
||||
// limite value embeddings
|
||||
struct ggml_tensor * value_embd = nullptr;
|
||||
|
||||
@@ -215,6 +215,8 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
|
||||
|
||||
per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),
|
||||
{ hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY);
|
||||
// an FP8 table keeps its checkpoint bytes, so its per-tensor scale is applied after the gather
|
||||
per_layer_tok_embd_s = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "scale"), { 1 }, TENSOR_NOT_REQUIRED);
|
||||
}
|
||||
|
||||
auto load_block = [&](int il, int flags) {
|
||||
@@ -1408,6 +1410,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(
|
||||
|
||||
// gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does
|
||||
ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
|
||||
if (model.per_layer_tok_embd_s) {
|
||||
emb = ggml_mul(ctx0, emb, model.per_layer_tok_embd_s);
|
||||
}
|
||||
emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens);
|
||||
cb(emb, "ple_embd", -1);
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from conversion.llama import LlamaModel
|
||||
from conversion.nemotron import NemotronHModel
|
||||
from conversion.phi import PhiMoeModel
|
||||
from conversion.qwen import Qwen2MoeModel, Qwen3NextModel
|
||||
from conversion.qwen4exp import Qwen4ExpTextModel
|
||||
|
||||
|
||||
class ForcedF16Model(Qwen2MoeModel):
|
||||
@@ -366,6 +367,27 @@ class TestFP8Conversion(unittest.TestCase):
|
||||
self.assertEqual(dtype, gguf.GGMLQuantizationType.Q8_0)
|
||||
np.testing.assert_allclose(gguf.dequantize(data, dtype), np.full((4, 32), 0.25), atol=1e-4)
|
||||
|
||||
def test_modelopt_block_scale_inv(self):
|
||||
name = "model.layers.0.self_attn.q_proj.weight"
|
||||
weight = (torch.arange(32, dtype=torch.float32).reshape(4, 8) - 16).to(torch.float8_e4m3fn)
|
||||
scale = torch.tensor([[0.25, 0.5], [1.0, 2.0]])
|
||||
expected = weight.float() * scale.repeat_interleave(2, 0).repeat_interleave(4, 1)
|
||||
for eager in (False, True):
|
||||
with self.subTest(eager=eager):
|
||||
result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": scale}, eager=eager, check_lazy=not eager)
|
||||
self.assertEqual(set(result), {"blk.0.attn_q.weight"})
|
||||
dtype, data = result["blk.0.attn_q.weight"]
|
||||
self.assertEqual(dtype, gguf.GGMLQuantizationType.F16)
|
||||
np.testing.assert_array_equal(data, expected.numpy())
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "not tiled"):
|
||||
self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.ones((3, 2))})
|
||||
|
||||
# a scalar scale keeps the FP8 bytes, as with weight_scale
|
||||
result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.tensor([0.25])}, check_lazy=True)
|
||||
self.assertEqual(result["blk.0.attn_q.weight"][0], gguf.GGMLQuantizationType.F8_E4M3)
|
||||
np.testing.assert_array_equal(result["blk.0.attn_q.scale"][1], [0.25])
|
||||
|
||||
|
||||
class TestInkling2PackedConversion(unittest.TestCase):
|
||||
"""Conversion of the packed Inkling2 expert weights (see _dequant_expert_chunk in conversion/inkling2.py).
|
||||
@@ -675,5 +697,84 @@ class TestInkling2TextNames(unittest.TestCase):
|
||||
np.testing.assert_array_equal(renamed[name], actual[name], name)
|
||||
|
||||
|
||||
class TestQwen4ExpPleScale(unittest.TestCase):
|
||||
"""FP8 checkpoints store the n-gram table as shards with one shared scale under the parent module name."""
|
||||
|
||||
PREFIX = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding."
|
||||
SHARDS = (torch.tensor([[1., -2.], [3., 4.]]), torch.tensor([[5., 6.], [-7., 8.]]))
|
||||
TABLE = "per_layer_token_embd.weight"
|
||||
SCALE = "per_layer_token_embd.scale"
|
||||
|
||||
def convert(self, ftype=gguf.LlamaFileType.GUESSED, scaled=True, quant_method="modelopt", **kwargs):
|
||||
# shard 1 first: the row order must follow the shard index
|
||||
tensors = {self.PREFIX + f"shard_{i}.weight": self.SHARDS[i].to(torch.float8_e4m3fn) for i in (1, 0)}
|
||||
if scaled:
|
||||
tensors[self.PREFIX + "weight_scale"] = torch.tensor([0.25], dtype=torch.bfloat16)
|
||||
hparams = {
|
||||
"model_type": "qwen4exp",
|
||||
"architectures": ["Qwen4ExpForCausalLM"],
|
||||
"num_hidden_layers": 2,
|
||||
"mtp_num_hidden_layers": 1,
|
||||
"split_ngram_parts": 2,
|
||||
"quantization_config": {"quant_method": quant_method},
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory)
|
||||
(path / "config.json").write_text(json.dumps(hparams))
|
||||
save_file(tensors, str(path / "model.safetensors"))
|
||||
model = Qwen4ExpTextModel(path, ftype, path / "out.gguf", hparams=hparams, **kwargs)
|
||||
model.prepare_tensors()
|
||||
result = {}
|
||||
for name, info in model.gguf_writer.tensors[0].items():
|
||||
if isinstance(info.tensor, gguf.LazyChunkedTensor):
|
||||
# written like the GGUF writer does, a chunked tensor has no other read path
|
||||
with (path / "tensor.bin").open("wb") as f:
|
||||
info.tensor.tofile(f)
|
||||
data = np.frombuffer((path / "tensor.bin").read_bytes(), dtype=info.tensor.dtype).reshape(info.tensor.shape)
|
||||
else:
|
||||
data = np.asarray(gguf.LazyNumpyTensor.to_eager(info.tensor))
|
||||
result[name] = (info.dtype, data)
|
||||
return result
|
||||
|
||||
def test_fp8_table_keeps_bytes_and_scale(self):
|
||||
raw = torch.cat(self.SHARDS).to(torch.float8_e4m3fn).view(torch.uint8).numpy()
|
||||
for kwargs, names in (
|
||||
({}, {self.TABLE, self.SCALE}),
|
||||
({"quant_method": "fp8"}, {self.TABLE, self.SCALE}),
|
||||
({"ple_only": True}, {self.TABLE}),
|
||||
({"no_ple": True}, {self.SCALE}),
|
||||
):
|
||||
with self.subTest(**kwargs):
|
||||
result = self.convert(**kwargs)
|
||||
self.assertEqual(set(result), names)
|
||||
if self.TABLE in result:
|
||||
dtype, data = result[self.TABLE]
|
||||
self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3)
|
||||
np.testing.assert_array_equal(data, raw)
|
||||
if self.SCALE in result:
|
||||
dtype, data = result[self.SCALE]
|
||||
self.assertEqual(dtype, gguf.GGMLQuantizationType.F32)
|
||||
np.testing.assert_array_equal(data, [0.25])
|
||||
|
||||
result = self.convert(scaled=False)
|
||||
self.assertEqual(set(result), {self.TABLE})
|
||||
np.testing.assert_array_equal(result[self.TABLE][1], raw)
|
||||
|
||||
def test_scale_is_folded_into_other_types(self):
|
||||
values = torch.cat(self.SHARDS).numpy()
|
||||
for kwargs, qtype in (
|
||||
({"ftype": gguf.LlamaFileType.ALL_F32}, gguf.GGMLQuantizationType.F32),
|
||||
({"ftype": gguf.LlamaFileType.MOSTLY_F16}, gguf.GGMLQuantizationType.F16),
|
||||
({"fp8_as_q8": True}, gguf.GGMLQuantizationType.F16),
|
||||
):
|
||||
for scaled in (False, True):
|
||||
with self.subTest(scaled=scaled, **kwargs):
|
||||
result = self.convert(scaled=scaled, **kwargs)
|
||||
self.assertEqual(set(result), {self.TABLE})
|
||||
dtype, data = result[self.TABLE]
|
||||
self.assertEqual(dtype, qtype)
|
||||
np.testing.assert_array_equal(data, values * (0.25 if scaled else 1.))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user