Files
llama.cpp/conversion/metis.py
T
2026-11-03 16:24:45 +01:00

205 lines
11 KiB
Python

from __future__ import annotations
import math
from typing import Callable, Iterable, TYPE_CHECKING
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import LazyTorchTensor, ModelBase, TextModel, gguf
@ModelBase.register("Metis16ForCausalLM")
class Metis16Model(TextModel):
model_arch = gguf.MODEL_ARCH.METIS16
# read by generate_extra_tensors() or not used at inference: dropped in modify_tensors()
_folded = {
"pass_embeddings", "lm_head.weight", "depth_budget.multiplier",
"depth_memory.pass_embeddings", "depth_memory.anchor_embeddings",
"depth_memory.state_write.bias", "depth_memory.metadata_write.bias",
}
_folded_layer = {
"mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits",
"mixer_connection.read_logits", "mixer_connection.write_logits",
"moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits",
"moe_connection.read_logits", "moe_connection.write_logits",
"moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier",
}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
hp = self.hparams
self.n_pass = hp["num_passes"]
self.attn_layers = list(hp["attention_layer_indices"])
self._generated: set[str] = set()
# one cache slot per pass and layer, while the tensors exist once per physical layer
self.block_count = self.n_pass * hp["num_hidden_layers"]
def set_vocab(self):
self._set_vocab_gpt2()
# turns end on the first eos id, so llama.cpp must stop on it too
eos = self.hparams.get("eos_token_id")
if isinstance(eos, list) and eos:
self.gguf_writer.add_eot_token_id(eos[0])
def set_gguf_parameters(self):
hp = self.hparams
super().set_gguf_parameters()
n_layer = hp["num_hidden_layers"]
self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)])
self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"])
self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"])
self.gguf_writer.add_expert_shared_count(1)
self.gguf_writer.add_moe_latent_size(hp["latent_dim"])
d_inner = hp["mamba_expand"] * hp["hidden_size"]
self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"])
self.gguf_writer.add_ssm_inner_size(d_inner)
self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"])
self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"])
self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"])
self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"])
self.gguf_writer.add_hyper_connection_count(hp["n_streams"])
self.gguf_writer.add_metis_pass_count(self.n_pass)
self.gguf_writer.add_metis_memory_length(hp["memory_dim"])
self.gguf_writer.add_metis_route_length(hp["route_feature_dim"])
self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"])
self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"])
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
# the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it
if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")):
name += ".bias"
elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates"))
or name in ("depth_memory.stream_gate", "stream_embeddings")):
name += ".weight"
elif name == "depth_memory.stream_gate_bias":
name = "depth_memory.stream_gate.bias"
return super().filter_tensors((name, gen))
def _load(self, name: str) -> Tensor:
return LazyTorchTensor.to_eager(self.model_tensors[name]())
def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]:
self._generated.add(name)
return name, data
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
hp = self.hparams
r = hp["route_feature_dim"]
n_stream = hp["n_streams"]
T = gguf.MODEL_TENSOR
passes = self._load("pass_embeddings")[: self.n_pass]
# the mixes depend only on the weights and the pass, so Sinkhorn runs once here
# the controller runs in the checkpoint dtype, as the reference does
for il in range(hp["num_hidden_layers"]):
for src, (mix, read, write) in (
("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)),
("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)),
):
prefix = f"layers.{il}.{src}."
ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias")
mixes, reads, writes = [], [], []
for p in range(self.n_pass):
delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float()
m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream])
log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream)
for _ in range(hp["mhc_sinkhorn_iterations"]):
log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True)
log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True)
mixes.append(log_m.exp())
reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1))
writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1))
act = ctrl_w.dtype
yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float())
yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float())
yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float())
# one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width
meta_w = self._load("depth_memory.metadata_write.weight").float()
mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"]
base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float()
+ mean_k * meta_w[:, r])
pass_e = self._load("depth_memory.pass_embeddings").float()
anchor_e = self._load("depth_memory.anchor_embeddings").float()
n_anchor = len(self.attn_layers) + 1
entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)])
yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias)
def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# the graph reads the concatenated features one block at a time; each block is its own tensor
for i, part in enumerate(parts):
piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:]
yield from super().modify_tensors(piece, template.format(part), bid)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name in self._generated:
yield name, data_torch
return
if name in self._folded:
return
hp = self.hparams
d = hp["hidden_size"]
if bid is None:
if name == "stream_embeddings.weight":
# the graph keeps the streams innermost: [n_stream, n_embd, n_tokens]
yield from super().modify_tensors(data_torch.T.contiguous(), name, bid)
return
if name == "depth_memory.metadata_write.weight":
# metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized
r = hp["route_feature_dim"]
meta_w = data_torch.float()
entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"])
data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1)
elif name == "depth_memory.stream_gate.weight":
data_torch = data_torch.float() / math.sqrt(d)
elif name == "depth_memory.route_projection.weight":
yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid)
return
elif name == "continuation.hidden.weight":
yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid)
return
yield from super().modify_tensors(data_torch, name, bid)
return
local = name.removeprefix(f"layers.{bid}.")
if local in self._folded_layer:
return
if local == "mixer.impl.mixer.A_log":
data_torch = -torch.exp(data_torch).unsqueeze(-1)
elif local == "mixer.impl.mixer.D":
data_torch = data_torch.unsqueeze(-1)
elif local == "mixer.impl.mixer.conv1d.weight":
data_torch = data_torch.squeeze(1)
elif local == "mixer.impl.mixer.norm.weight":
data_torch = data_torch.reshape(hp["mamba_n_groups"], -1)
elif local == "mixer.pass_gates.weight":
# the reference applies tanh(gate) in the checkpoint dtype
data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float()
elif local == "moe.shared_expert.gate_up.weight":
gate, up = data_torch.chunk(2, dim=0)
yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid)
yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid)
return
yield from super().modify_tensors(data_torch, name, bid)
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
# FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32
if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate"))
or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight",
"hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))):
return gguf.GGMLQuantizationType.F32
return super().tensor_force_quant(name, new_name, bid, n_dims)