* mtmd: support MageVL-Cowork model * mtmd: keep magevl rope_freqs at model level and check its shape The rope factors are shared by every layer, so they live in clip_model instead of a pointer copied into each clip_layer. The loader now checks that v.rope_freqs is F32 with d_head elements and that d_head splits into the 4:6:6 pairs the graph expects. --------- Co-authored-by: Pascal <admin@serveurperso.com>
78 lines
3.2 KiB
Python
78 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Callable, Iterable, TYPE_CHECKING
|
|
|
|
import torch
|
|
|
|
if TYPE_CHECKING:
|
|
from torch import Tensor
|
|
|
|
from .base import MmprojModel, ModelBase, gguf
|
|
|
|
|
|
@ModelBase.register("LlavaOnevision2ForConditionalGeneration")
|
|
@ModelBase.example("microsoft/Mage-VL-Cowork")
|
|
class MageVLVisionModel(MmprojModel):
|
|
# a rope pair uses 2 different angles: the graph runs 2 ropes on heads permuted to [evens | odds]
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
assert self.hparams_vision is not None
|
|
|
|
self.n_embd = self.find_vparam(["hidden_size"])
|
|
self.n_head = self.find_vparam(["num_attention_heads"])
|
|
self.d_head = self.n_embd // self.n_head
|
|
|
|
d_half = self.d_head // 2
|
|
assert d_half % 16 == 0, "the 4:6:6 split of the rope requires d_head to be a multiple of 32"
|
|
|
|
# 4:6:6 split for t:h:w
|
|
n_t = 4 * (d_half // 16)
|
|
n_s = 6 * (d_half // 16)
|
|
theta = float(self.find_vparam(["rope_theta"]))
|
|
freq_t = 1.0 / theta ** (torch.arange(n_t, dtype=torch.float64) / n_t)
|
|
freq_s = 1.0 / theta ** (torch.arange(n_s, dtype=torch.float64) / n_s)
|
|
freq = torch.cat([freq_t, freq_s, freq_s])
|
|
self.rope_freq = torch.cat([freq, freq])
|
|
|
|
def pair_axis(i: int) -> int:
|
|
dim = (2 * i) % d_half
|
|
return 0 if dim < n_t else 1 if dim < n_t + n_s else 2
|
|
|
|
self.rope_pairs = sorted(range(d_half), key=pair_axis)
|
|
|
|
def set_gguf_parameters(self):
|
|
super().set_gguf_parameters()
|
|
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MAGEVL)
|
|
self.gguf_writer.add_vision_attention_layernorm_eps(self.find_vparam(["layer_norm_eps"]))
|
|
self.gguf_writer.add_vision_spatial_merge_size(self.find_vparam(["spatial_merge_size"]))
|
|
self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"]))
|
|
self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"]))
|
|
|
|
@classmethod
|
|
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
|
name, _ = item
|
|
|
|
if not name.startswith("visual."):
|
|
return None
|
|
|
|
return super().filter_tensors(item)
|
|
|
|
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
|
# ggml uses theta = pos / rope_freqs
|
|
pairs = torch.tensor(self.rope_pairs)
|
|
freq_a = self.rope_freq[2 * pairs]
|
|
freq_b = self.rope_freq[2 * pairs + 1]
|
|
yield ("visual.rope_freqs.weight", (1.0 / torch.cat([freq_a, freq_b])).to(torch.float32))
|
|
|
|
def permute_qk(self, data_torch: Tensor) -> Tensor:
|
|
rows = [2 * i for i in self.rope_pairs] + [2 * i + 1 for i in self.rope_pairs]
|
|
return data_torch.reshape(self.n_head, self.d_head, -1)[:, rows].reshape(data_torch.shape)
|
|
|
|
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
|
if ".self_attn.qkv." in name:
|
|
q, k, v = data_torch.split(self.n_embd, dim=0)
|
|
data_torch = torch.cat([self.permute_qk(q), self.permute_qk(k), v], dim=0)
|
|
|
|
yield from super().modify_tensors(data_torch, name, bid)
|