* convert : add Inkling2 text GGUF conversion Assisted-by: Codex * llama : implement Inkling2 text inference Assisted-by: Codex * tests : cover Inkling2 text support Assisted-by: Codex * conversion : dequantize packed Inkling2 experts in row chunks Decode per-layer NVFP4 and MXFP8 schemes, validate companions, and use stored scale shapes with a warning for the declared MXFP8 [32,32] grid. Reorder packed w13 rows and their scale rows together. Keep normal GGUF outtype selection and 3D expert tensors. Stream 128 rows per expert through LazyChunkedTensor. Real w13 (384x4608x4608) needs a 2.25 MiB F32 chunk instead of a 30.375 GiB expanded layer. Real-shaped sparse probes peak below 5.8 MiB of Python/NumPy chunk allocations, plus 5.2 MiB of callback metadata: budget about 16 MiB per projection including mapped pages, excluding runtime, other tensors and OS cache. Full-model chunk metadata is roughly 370 MiB. Packed synthetic GGUF weights match the independent pinned vLLM decode bit-for-bit. CPU prefill/decode logits remain within 3.1e-6 absolute error, with 100% argmax agreement. Float fixtures, original-name conversion and CPU architecture checks pass. Real checkpoint conversion remains untested. Assisted-by: Codex * convert : add Inkling2 vision mmproj support Assisted-by: Codex * mtmd : implement Inkling2 still-image encoder Assisted-by: Codex * inkling2 : scope image loading Assisted-by: Codex * inkling2 : enable flash attention with folded bias and sinks Assisted-by: Codex * cuda : route per-head flash attention masks through vec Assisted-by: Codex * mtmd : support Inkling2 temporal frame means Assisted-by: Codex * style : restore shared image comment Assisted-by: Codex * tests : enlarge Inkling2 context fixture Assisted-by: Codex * inkling2 : load per-layer expert feed-forward widths Assisted-by: Codex * tests : add Inkling2 Metal fusion baselines Assisted-by: Codex * inkling2 : group and align registry entries Assisted-by: Fable * convert : regenerate the Inkling2 tokenizer hash with the update script Assisted-by: Fable * conversion : revert the print_registered_models change Assisted-by: Fable * conversion : sort the Inkling2 model map entries Assisted-by: Fable * convert : map Inkling2 vision checkpoint names in the tensor table Assisted-by: Fable * convert : map Inkling2 text checkpoint names in the tensor table Assisted-by: Fable * tests : document the Inkling2 packed expert conversion test Assisted-by: Fable * tests : cover Inkling2 checkpoint and transformers tensor names Assisted-by: Fable * llama : place Inkling2 fields and list entries by file convention Assisted-by: Fable * gguf-py : align Inkling2 tensor mapping comments Assisted-by: Fable * mtmd : place Inkling2 entries by file convention Assisted-by: Fable * tests : place Inkling2 cases after the existing ones Assisted-by: Fable * cuda : index per-head flash attention masks in the tile and mma kernels Assisted-by: Claude Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q * inkling2 : project the relative bias before the gather, drop arch branches Assisted-by: Claude Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q * convert : require Inkling2 hparams, add inkling2 pre-tokenizer Assisted-by: Claude Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q * mtmd : cap Inkling2 image size, drop video frame averaging Assisted-by: Claude Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q * inkling2 : fix CI build, Meta device assert and Windows conversion test Assisted-by: Claude * tests : use the default sliding window for the Inkling2 test model Assisted-by: Claude * simplify mtmd part * give _v arch for vision * remove alpha channel handling, it should be a follow up * explicit rule about audio + vision models * add audio support * rm audio * rm gguf mtmd rope base * address review comment * nits * do not normalize embd from mtmd --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
328 lines
19 KiB
Python
328 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from functools import partial
|
|
from typing import Callable, Iterable, TYPE_CHECKING, cast
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
if TYPE_CHECKING:
|
|
from torch import Tensor
|
|
|
|
from .base import LazyTorchTensor, ModelBase, MmprojModel, TextModel, gguf, logger
|
|
|
|
|
|
def _e4m3_table() -> np.ndarray:
|
|
codes = np.arange(256, dtype=np.uint8)
|
|
exponent = ((codes & 0x7F) >> 3).astype(np.int32)
|
|
mantissa = (codes & 7).astype(np.float32)
|
|
values = np.where(exponent == 0, np.ldexp(mantissa, -9), np.ldexp(1 + mantissa / 8, exponent - 7))
|
|
values = np.where(codes & 0x80, -values, values)
|
|
values[(codes & 0x7F) == 0x7F] = np.nan
|
|
return values.astype(np.float32)
|
|
|
|
|
|
_E4M3 = _e4m3_table()
|
|
_E2M1 = np.array([0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6], dtype=np.float32)
|
|
_EXPERT_CHUNK_ROWS = 128
|
|
|
|
|
|
def _dequant_expert_chunk(weight: Callable[[], Tensor], scale: Callable[[], Tensor], amax: Callable[[], Tensor] | None, denominator: Callable[[], Tensor] | None, name: str, expert: int, rows: slice) -> np.ndarray:
|
|
# Fresh lazy tensors release their mappings after each chunk instead of caching the layer.
|
|
packed = LazyTorchTensor.to_eager(weight().view(torch.uint8)[expert, rows]).numpy()
|
|
scales = LazyTorchTensor.to_eager(scale().view(torch.uint8)[expert, rows]).numpy()
|
|
if amax is not None:
|
|
block_scale = _E4M3[scales]
|
|
if np.isnan(block_scale).any():
|
|
raise ValueError(f"Inkling2: {name}_scale expert {expert}: E4M3FN NaN")
|
|
values = np.empty((packed.shape[0], packed.shape[1] * 2), dtype=np.float32)
|
|
values[:, 0::2] = _E2M1[packed & 15]
|
|
values[:, 1::2] = _E2M1[packed >> 4]
|
|
values.reshape(values.shape[0], -1, 16)[:] *= block_scale[:, :, None]
|
|
assert denominator is not None
|
|
a = LazyTorchTensor.to_eager(amax()[expert]).item()
|
|
d = LazyTorchTensor.to_eager(denominator()).item()
|
|
if not np.isfinite(a) or a < 0:
|
|
raise ValueError(f"Inkling2: {name}_amax expert {expert}: expected finite nonnegative value")
|
|
if not np.isfinite(d) or d <= 0:
|
|
raise ValueError(f"Inkling2: {name}_outer_scale_denominator expert {expert}: expected finite positive value")
|
|
with np.errstate(over="ignore", invalid="ignore"):
|
|
values *= np.float32(a) / np.float32(d)
|
|
else:
|
|
if np.any(scales == 255):
|
|
raise ValueError(f"Inkling2: {name}_scale expert {expert}: UE8M0 byte 255 is NaN")
|
|
values = _E4M3[packed]
|
|
if np.isnan(values).any():
|
|
raise ValueError(f"Inkling2: {name} expert {expert}: E4M3FN NaN")
|
|
block_scale = np.ldexp(np.ones(scales.shape, dtype=np.float32), scales.astype(np.int32) - 127)
|
|
with np.errstate(over="ignore", invalid="ignore"):
|
|
values.reshape(values.shape[0], -1, 32)[:] *= block_scale[:, :, None]
|
|
if not np.isfinite(values).all():
|
|
raise ValueError(f"Inkling2: {name} expert {expert}: non-finite dequantized weight")
|
|
return values
|
|
|
|
|
|
def _is_inkling2(dir_model: Path) -> bool:
|
|
if not (dir_model / "config.json").is_file():
|
|
return False
|
|
with open(dir_model / "config.json", encoding="utf-8") as f:
|
|
return json.load(f).get("model_type") in ("inkling2", "inkling2_text")
|
|
|
|
|
|
@ModelBase.register_hparams_loader(_is_inkling2)
|
|
def _load_inkling2_hparams(dir_model: Path):
|
|
# AutoConfig injects MXFP8 defaults and can replace the original layer schedule.
|
|
with open(dir_model / "config.json", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
@ModelBase.register("Inkling2ForConditionalGeneration", "Inkling2ForCausalLM")
|
|
@ModelBase.example("thinkingmachines/Inkling-2-Small")
|
|
class Inkling2Model(TextModel):
|
|
model_arch = gguf.MODEL_ARCH.INKLING2
|
|
|
|
def __init__(self, dir_model: Path, *args, **kwargs):
|
|
self._expert_quant: dict[str, dict] = {}
|
|
self._packed_experts: dict[str, gguf.LazyChunkedTensor] = {}
|
|
quant_path = dir_model / "hf_quant_config.json"
|
|
if quant_path.is_file():
|
|
with open(quant_path, encoding="utf-8") as f:
|
|
groups = json.load(f).get("model", {}).get("parameter_groups", [])
|
|
for group in groups:
|
|
for name in group.get("tensors", []):
|
|
item = self.filter_tensors((name, lambda: torch.empty(0)))
|
|
if item is not None and item[0].endswith((".w13_weight", ".w2_weight")):
|
|
name = item[0]
|
|
if name in self._expert_quant:
|
|
raise ValueError(f"Inkling2: duplicate hf_quant_config.json scheme for {name}")
|
|
self._expert_quant[name] = group
|
|
super().__init__(dir_model, *args, **kwargs)
|
|
logger.warning("Inkling2: converting the text backbone only; vision and MTP tensors are skipped")
|
|
hp = self.hparams
|
|
for name, default in (("q_bias", False), ("o_bias", False), ("n_shared_experts", 0), ("use_value_norm_no_gamma", True), ("attn_res_normalize_values", True), ("scoring_func", "sigmoid")):
|
|
if hp.get(name, default) != default:
|
|
raise ValueError(f"Inkling2: unsupported {name}={hp[name]!r}")
|
|
types = hp.get("layer_types")
|
|
if types is None:
|
|
local = hp["local_layer_ids"]
|
|
types = ["hybrid_sliding" if i in local else "hybrid" for i in range(self.block_count)]
|
|
if len(types) != self.block_count or set(types) - {"hybrid", "hybrid_sliding"}:
|
|
raise ValueError("Inkling2: invalid layer_types")
|
|
self.is_swa = [t == "hybrid_sliding" for t in types]
|
|
|
|
def dequant_model(self):
|
|
for name in list(self.model_tensors):
|
|
if not name.endswith((".w13_weight", ".w2_weight")):
|
|
continue
|
|
weight = self.model_tensors[name]
|
|
tensor = weight()
|
|
companions = [name + suffix for suffix in ("_scale", "_amax", "_outer_scale_denominator")]
|
|
if tensor.dtype.is_floating_point and tensor.element_size() > 1:
|
|
if any(n in self.model_tensors for n in companions):
|
|
raise ValueError(f"Inkling2: floating-point {name} has packed quantization companions")
|
|
continue
|
|
config = self._expert_quant.get(name, {})
|
|
scheme = config.get("weight_transform")
|
|
if scheme not in ("nvfp4_nearest", "mxfp8_nearest"):
|
|
raise ValueError(f"Inkling2: {name}: missing or unsupported hf_quant_config.json scheme {scheme!r}")
|
|
nvfp4 = scheme == "nvfp4_nearest"
|
|
if nvfp4 and config.get("weight_scale_format") != "amax_denominator":
|
|
raise ValueError(f"Inkling2: {name}: expected NVFP4 amax_denominator scale format")
|
|
required = companions if nvfp4 else companions[:1]
|
|
for companion in required:
|
|
if companion not in self.model_tensors:
|
|
raise ValueError(f"Inkling2: {name}: missing companion {companion}")
|
|
if not nvfp4 and any(n in self.model_tensors for n in companions[1:]):
|
|
raise ValueError(f"Inkling2: {name}: MXFP8 must not have amax or outer_scale_denominator")
|
|
scale = self.model_tensors[companions[0]]
|
|
scale_tensor = scale()
|
|
weight_dtype, scale_dtype = (torch.uint8, torch.float8_e4m3fn) if nvfp4 else (torch.float8_e4m3fn, torch.uint8)
|
|
if tensor.dtype != weight_dtype or scale_tensor.dtype != scale_dtype:
|
|
raise ValueError(f"Inkling2: {name}: expected {weight_dtype} weights and {scale_dtype} scales, got {tensor.dtype} and {scale_tensor.dtype}")
|
|
experts = self.hparams["n_routed_experts"]
|
|
hidden = self.hparams["hidden_size"]
|
|
inter = self.find_hparam(["moe_intermediate_size", "intermediate_size"])
|
|
w13 = name.endswith(".w13_weight")
|
|
rows, cols = (2 * inter, hidden) if w13 else (hidden, inter)
|
|
shape = (experts, rows, cols)
|
|
if min(shape) <= 0 or cols % (16 if nvfp4 else 32) or tuple(tensor.shape) != (experts, rows, cols // (2 if nvfp4 else 1)):
|
|
raise ValueError(f"Inkling2: {name}: invalid packed shape {tuple(tensor.shape)} for logical shape {shape}")
|
|
ss = tuple(scale_tensor.shape)
|
|
if len(ss) != 3 or ss[0] != experts or min(ss) <= 0 or rows % ss[1] or cols % ss[2]:
|
|
raise ValueError(f"Inkling2: {name}_scale: shape {ss} does not divide logical shape {shape}")
|
|
block = (rows // ss[1], cols // ss[2])
|
|
if block != (1, 16 if nvfp4 else 32):
|
|
raise ValueError(f"Inkling2: {name}_scale: unsupported stored block shape {block}")
|
|
if list(block) != config.get("weight_block_shape"):
|
|
logger.warning("Inkling2: %s: hf_quant_config.json weight_block_shape %s conflicts with stored weight/scale shapes %s/%s; using %s", name, config.get("weight_block_shape"), tuple(tensor.shape), ss, list(block))
|
|
amax = denominator = None
|
|
if nvfp4:
|
|
amax, denominator = (self.model_tensors[n] for n in companions[1:])
|
|
for companion, gen, expected in ((companions[1], amax, (experts,)), (companions[2], denominator, (1,))):
|
|
t = gen()
|
|
if t.dtype != torch.float32 or tuple(t.shape) != expected:
|
|
raise ValueError(f"Inkling2: {companion}: expected F32 {expected}, got {t.dtype} {tuple(t.shape)}")
|
|
chunks: list[Callable[[], np.ndarray]] = []
|
|
for expert in range(experts):
|
|
for half in range(2 if w13 else 1):
|
|
for start in range(0, rows // 2 if w13 else rows, _EXPERT_CHUNK_ROWS):
|
|
stop = min(start + _EXPERT_CHUNK_ROWS, rows // 2 if w13 else rows)
|
|
# The same packed row slice selects the weight and its scale groups.
|
|
row_slice = slice(2 * start + half, 2 * stop + half, 2) if w13 else slice(start, stop)
|
|
chunks.append(partial(_dequant_expert_chunk, weight, scale, amax, denominator, name, expert, row_slice))
|
|
self._packed_experts[name] = gguf.LazyChunkedTensor(chunks, shape, np.float32)
|
|
self.model_tensors[name] = lambda shape=shape: torch.empty(shape, dtype=torch.float32, device="meta")
|
|
for companion in required:
|
|
del self.model_tensors[companion]
|
|
for name, gen in self.model_tensors.items():
|
|
t = gen()
|
|
if not t.dtype.is_floating_point or t.element_size() == 1 or name.endswith(("_weight_scale", "_weight_amax", "_weight_outer_scale_denominator")):
|
|
raise ValueError(f"Inkling2: unexpected packed tensor or orphan companion {name}")
|
|
|
|
@classmethod
|
|
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
|
name, gen = item
|
|
if any(p in name for p in ("visual.", "vision_tower.", "mtp.", "markov_head", "nextn_predict")):
|
|
return None
|
|
name = name.replace("model.language_model.", "model.").replace("model.llm.", "model.")
|
|
# the mapping table cannot shift a block index: attn_queries.{i} feeds block i + 1
|
|
name = re.sub(r"attn_res\.attn_queries\.(\d+)", lambda m: f"layers.{int(m[1]) + 1}.attention_input.query", name)
|
|
# the table maps a module, not its bias: gate.weight is the router, gate.bias the score correction
|
|
name = name.replace("mlp.gate.bias", "mlp.gate.e_score_correction_bias")
|
|
return name, gen
|
|
|
|
def get_vocab_base(self):
|
|
tokens, types, pre = super().get_vocab_base()
|
|
n = self.hparams["unpadded_vocab_size"]
|
|
return tokens[:n], types[:n], pre
|
|
|
|
def set_vocab(self):
|
|
tokens, types, pre = self.get_vocab_base()
|
|
self.gguf_writer.add_tokenizer_model("gpt2")
|
|
self.gguf_writer.add_tokenizer_pre(pre)
|
|
self.gguf_writer.add_token_list(tokens)
|
|
self.gguf_writer.add_token_types(types)
|
|
special = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
|
eos = self.hparams["eos_token_id"]
|
|
special.special_token_ids["eos"] = eos
|
|
special.special_token_ids["eot"] = eos
|
|
special.add_special_token["bos"] = False
|
|
special.add_to_gguf(self.gguf_writer)
|
|
|
|
def set_gguf_parameters(self):
|
|
hp = self.hparams
|
|
hp["num_attention_heads"] = [hp.get("swa_num_attention_heads", hp["num_attention_heads"]) if s else hp["num_attention_heads"] for s in self.is_swa]
|
|
hp["num_key_value_heads"] = [hp.get("swa_num_key_value_heads", hp["num_key_value_heads"]) if s else hp["num_key_value_heads"] for s in self.is_swa]
|
|
for name in ("n_routed_experts", "num_experts_per_tok"):
|
|
if name not in hp:
|
|
raise KeyError(f"Inkling2: missing {name}")
|
|
super().set_gguf_parameters()
|
|
w = self.gguf_writer
|
|
w.add_vocab_size(hp["vocab_size"])
|
|
w.add_vocab_size_unpadded(hp["unpadded_vocab_size"])
|
|
w.add_expert_feed_forward_length(self.find_hparam(["moe_intermediate_size", "intermediate_size"]))
|
|
w.add_key_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
|
|
w.add_value_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
|
|
w.add_rope_dimension_count(0)
|
|
w.add_sliding_window(hp["sliding_window_size"])
|
|
w.add_sliding_window_pattern(self.is_swa)
|
|
w.add_shortconv_l_cache(self.find_hparam(["conv_kernel_size", "sconv_kernel_size"]))
|
|
w.add_expert_weights_scale(hp["route_scale"])
|
|
w.add_activation_situ_beta(hp["mlp_gate_softcap"])
|
|
w.add_activation_situ_linear_beta(hp["mlp_up_softcap"])
|
|
w.add_attn_res_block_size(hp["n_layers_per_attn_res_block"])
|
|
w.add_final_logit_softcapping(hp["final_logit_softcapping"])
|
|
w.add_relative_length(hp["d_rel"])
|
|
w.add_relative_extent(hp["rel_extent"])
|
|
w.add_qk_clip(hp["qk_l2_clip_threshold"])
|
|
w.add_relative_clip(hp["rel_l2_clip_threshold"])
|
|
w.add_log_scale_floor(hp["log_scaling_n_floor"])
|
|
w.add_log_scale_alpha(hp["log_scaling_alpha"])
|
|
|
|
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
|
if new_name.endswith(("_conv.weight", "_clip.weight", "_res_query.weight", "attn_rel_bias.weight")):
|
|
return gguf.GGMLQuantizationType.F32
|
|
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
|
|
|
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
|
if name in self._packed_experts:
|
|
target = name.replace(".w13_weight", ".gate_up_proj").replace(".w2_weight", ".down_proj")
|
|
yield self.map_tensor_name(target, try_suffixes=()) + ".weight", cast("Tensor", self._packed_experts[name])
|
|
return
|
|
if name.endswith(".w13_weight"):
|
|
shape = data_torch.shape
|
|
data_torch = data_torch.reshape(shape[0], shape[1] // 2, 2, shape[2]).transpose(1, 2).reshape(shape)
|
|
name = name.replace(".w13_weight", ".gate_up_proj")
|
|
name = name.replace(".w2_weight", ".down_proj")
|
|
for static, conv in ((".static_weight", ".conv1d.weight"), (".static_w", ".weight")):
|
|
if name.endswith(static):
|
|
data_torch = data_torch.flip(0).T.contiguous()
|
|
name = name.removesuffix(static) + conv
|
|
if name.endswith((".conv1d.weight", "sconv.weight")):
|
|
data_torch = data_torch.squeeze(1) if data_torch.ndim == 3 else data_torch
|
|
elif name.endswith(".rel_logits_proj.proj"):
|
|
data_torch = data_torch.T.contiguous()
|
|
# a name the table maps without a suffix is a bare parameter
|
|
new_name = self.tensor_map.get_name(name)
|
|
if new_name is not None:
|
|
suffix = ".bias" if name.endswith(".e_score_correction_bias") else ".weight"
|
|
yield new_name + suffix, data_torch
|
|
else:
|
|
yield from super().modify_tensors(data_torch, name, bid)
|
|
|
|
|
|
@ModelBase.register("Inkling2ForConditionalGeneration")
|
|
class Inkling2VisionModel(MmprojModel):
|
|
def get_vision_config(self):
|
|
config = self.global_config.get("vision_config")
|
|
if not config:
|
|
return None
|
|
config = dict(config)
|
|
config["patch_size"] = config.get("native_patch_size", config.get("patch_size", 20))
|
|
config["spatial_merge_size"] = config.get("adapter_merge_size", config.get("spatial_merge_size", 2))
|
|
config["intermediate_size"] = config.get("intermediate_size") or int(config["hidden_size"] * config.get("mlp_ratio", 4.0))
|
|
config["image_size"] = config["posemb_grid_size"] * config["patch_size"]
|
|
if config.get("num_channels", 3) != 3 or config.get("hidden_act", "gelu") != "gelu":
|
|
raise ValueError("Inkling2 vision requires RGB patches and exact GELU")
|
|
return config
|
|
|
|
def set_gguf_parameters(self):
|
|
self.preprocessor_config.setdefault("image_mean", [0.5, 0.5, 0.5])
|
|
self.preprocessor_config.setdefault("image_std", [0.5, 0.5, 0.5])
|
|
super().set_gguf_parameters()
|
|
config = self.hparams
|
|
rope = config.get("rope_parameters") or {"rope_type": "axial", "rope_theta": 10000.0}
|
|
if rope["rope_type"] != "axial" or rope["rope_theta"] != 10000.0:
|
|
raise ValueError("Inkling2 vision requires axial RoPE with theta 10000")
|
|
w = self.gguf_writer
|
|
w.add_clip_vision_projector_type(gguf.VisionProjectorType.INKLING2V)
|
|
w.add_vision_attention_layernorm_eps(config["layer_norm_eps"])
|
|
w.add_vision_spatial_merge_size(config["spatial_merge_size"])
|
|
|
|
@classmethod
|
|
def filter_tensors(cls, item):
|
|
name, gen = item
|
|
if not name.startswith(("model.visual.", "model.vision_tower.")):
|
|
return None
|
|
return name, gen
|
|
|
|
# mm.{bid} takes the projector layer index, the mapping table cannot derive it from these names
|
|
_projector_layers = {"model.visual.proj_in": 1, "model.visual.proj_out": 2, "model.vision_tower.merger.linear_1": 1, "model.vision_tower.merger.linear_2": 2}
|
|
|
|
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
|
module, _, suffix = name.rpartition(".")
|
|
if module in self._projector_layers:
|
|
yield self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, self._projector_layers[module], "." + suffix), data_torch
|
|
return
|
|
# lay out the linear patch embedding (HWC patches) as a conv2d for build_inp()
|
|
if name.endswith(".patch_embedding.weight"):
|
|
ps = self.hparams["patch_size"]
|
|
data_torch = data_torch.reshape(data_torch.shape[0], ps, ps, 3).permute(0, 3, 1, 2)
|
|
# these are bare parameters, they have no .weight
|
|
if name.endswith(("position_embedding", ".lambda_1", ".lambda_2", ".gamma")):
|
|
name = name.removesuffix(".gamma") + ".weight"
|
|
yield from super().modify_tensors(data_torch, name, bid)
|