From 9c9074e14e029a16fc3298e6bc6a3ef00a0b9c11 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Thu, 5 Nov 2026 01:05:06 -0800 Subject: [PATCH] conversion : resolve quantized_layers from source tensor names (#31611) * conversion : resolve quantized_layers from source tensor names This commit changes how the per layer NVFP4 quant_algo are resolved from hf_quant_config.json quantized_layers. The keys in quantized_layers are module paths in the source checkpoint, and these can be either be modules like 'lm_head' or parent modules like 'layers.0.ffn.experts'. Currently these keys are passed through filter_tensors, but they are not tensor names. So a filter_tensors that keeps an explicit list of tensor names, like Nemotron's with --mtp would drop them. That means there was no `_nvfp4_layer_algo entry`, and `_tag_prec_a4` fell back to `_nvfp4_global_algo`. This commit records the mapping from source tensor name to filtered tensor name in index_tensors. Then in prepare_tensors each source tensor is resolved to its closest quantized_layers parent, and the algo is stored under the filtered tensor name, which is what _tag_prec_a4 later looks up. Converting Nemotron 3.5 Lightning NVFP4 with --mtp now produces: ```console general.tensor_extra.name = ['output.weight'] general.tensor_extra.prec_a4 = [False] ``` whereas on master no general.tensor_extra fields are written. For the non-MTP conversion, the general.tensor_extra fields are identical to master. Also manually verified (using pdb) that this works for DeepSeek-V4.1-Flash-1017-NVFP4. Refs: https://github.com/ggml-org/llama.cpp/pull/31602#issuecomment-6603337403 * conversion : shorten comment * conversion : also remove '.bias' suffixes --- conversion/base.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index 7028d633b..841efe729 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -167,6 +167,8 @@ class ModelBase: self._fusable_qkv_weight_layers: set[int] = set() self._fusable_qkv_bias_layers: set[int] = set() self.hparams = ModelBase.load_hparams(self.dir_model, self.is_mistral_format) if hparams is None else hparams + # source tensor name to filtered tensor name mapping. + self._source_tensor_names: dict[str, str | None] = {} self.model_tensors = self.index_tensors(remote_hf_model_id=remote_hf_model_id) self.metadata_override = metadata_override self.model_name = model_name @@ -229,7 +231,9 @@ class ModelBase: remote_tensors = gguf.utility.SafetensorRemote.get_list_tensors_hf_model(remote_hf_model_id) for name, remote_tensor in remote_tensors.items(): data_gen = lambda r=remote_tensor: LazyTorchTensor.from_remote_tensor(r) # noqa: E731 - if titem := self.filter_tensors((name, data_gen)): + titem = self.filter_tensors((name, data_gen)) + self._source_tensor_names[name] = titem[0] if titem else None + if titem: tname, tgen = titem tensors[tname] = tgen @@ -294,7 +298,9 @@ class ModelBase: data_gen = lambda data=data_torch: LazyTorchTensor.from_eager(data) # noqa: E731 else: data_gen = lambda data=data_torch: data # noqa: E731 - if titem := self.filter_tensors((name, data_gen)): + titem = self.filter_tensors((name, data_gen)) + self._source_tensor_names[name] = titem[0] if titem else None + if titem: tname, tgen = titem tensors[tname] = tgen @@ -1417,18 +1423,25 @@ class ModelBase: # Per-tensor NVFP4 precision. self._nvfp4_layer_algo = {} if quant_layers: - # store all possible module paths and assert if a quantized layer is not in the model - modules: set[str] = set() - for name in self.model_tensors: - while name := name.rpartition(".")[0]: - modules.add(name) - - for layer_name, entry in quant_layers.items(): - if not isinstance(entry, dict): + # quantized_layers keys are module paths in the source checkpoint, not tensor names + # resolve each source tensor to the closest quantized_layers parent and record the algo under its filtered name for _tag_prec_a4 + layer_algo = {k: v.get("quant_algo") for k, v in quant_layers.items() if isinstance(v, dict)} + matched: set[str] = set() + for src_name, filtered_name in self._source_tensor_names.items(): + module = src_name + while module := module.rpartition(".")[0]: + if module in layer_algo: + break + if not module: continue - if titem := self.filter_tensors((layer_name, lambda: torch.empty(0))): - assert titem[0] in modules, f"quantized_layers entry {layer_name!r} is not in the model tensors" - self._nvfp4_layer_algo[titem[0]] = entry.get("quant_algo") + + matched.add(module) + # filter_tensors may drop a tensors so we need to check for that here. + if filtered_name is not None: + self._nvfp4_layer_algo[filtered_name.removesuffix(".weight").removesuffix(".bias")] = layer_algo[module] + + unmatched = layer_algo.keys() - matched + assert not unmatched, f"quantized_layers entries not in the model tensors: {sorted(unmatched)}" # NVFP4 weights are repacked and written directly to gguf_writer. # This must run before dequant_model so NVFP4 tensors are removed