diff --git a/conversion/__init__.py b/conversion/__init__.py index dcbb97c1f..6cea1ec78 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -197,6 +197,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "MambaLMHeadModel": "mamba", "MapleForCausalLM": "maple", "MellumForCausalLM": "mellum", + "Metis16ForCausalLM": "metis", "MiMoV2FlashForCausalLM": "mimo", "MiMoV2ForCausalLM": "mimo", "MiniCPM3ForCausalLM": "minicpm", diff --git a/conversion/base.py b/conversion/base.py index fd880bbab..9d9d6e875 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -2116,6 +2116,9 @@ class TextModel(ModelBase): if chkhsh == "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4": # ref: https://huggingface.co/evilfreelancer/ruGPT3XL res = "gpt-2" + if chkhsh == "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9": + # ref: https://huggingface.co/Lernex/Metis-1.6-Logos + res = "gpt-2" if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7": # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B res = "lfm2" diff --git a/conversion/metis.py b/conversion/metis.py new file mode 100644 index 000000000..a073ed4df --- /dev/null +++ b/conversion/metis.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import math +from typing import Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import LazyTorchTensor, ModelBase, TextModel, gguf + + +@ModelBase.register("Metis16ForCausalLM") +class Metis16Model(TextModel): + model_arch = gguf.MODEL_ARCH.METIS16 + + # read by generate_extra_tensors() or not used at inference: dropped in modify_tensors() + _folded = { + "pass_embeddings", "lm_head.weight", "depth_budget.multiplier", + "depth_memory.pass_embeddings", "depth_memory.anchor_embeddings", + "depth_memory.state_write.bias", "depth_memory.metadata_write.bias", + } + _folded_layer = { + "mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits", + "mixer_connection.read_logits", "mixer_connection.write_logits", + "moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits", + "moe_connection.read_logits", "moe_connection.write_logits", + "moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier", + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.hparams + self.n_pass = hp["num_passes"] + self.attn_layers = list(hp["attention_layer_indices"]) + self._generated: set[str] = set() + # one cache slot per pass and layer, while the tensors exist once per physical layer + self.block_count = self.n_pass * hp["num_hidden_layers"] + + def set_vocab(self): + self._set_vocab_gpt2() + # turns end on the first eos id, so llama.cpp must stop on it too + eos = self.hparams.get("eos_token_id") + if isinstance(eos, list) and eos: + self.gguf_writer.add_eot_token_id(eos[0]) + + def set_gguf_parameters(self): + hp = self.hparams + super().set_gguf_parameters() + + n_layer = hp["num_hidden_layers"] + self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)]) + + self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"]) + self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"]) + self.gguf_writer.add_expert_shared_count(1) + self.gguf_writer.add_moe_latent_size(hp["latent_dim"]) + + d_inner = hp["mamba_expand"] * hp["hidden_size"] + self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"]) + self.gguf_writer.add_ssm_inner_size(d_inner) + self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"]) + self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"]) + self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"]) + self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"]) + + self.gguf_writer.add_hyper_connection_count(hp["n_streams"]) + self.gguf_writer.add_metis_pass_count(self.n_pass) + self.gguf_writer.add_metis_memory_length(hp["memory_dim"]) + self.gguf_writer.add_metis_route_length(hp["route_feature_dim"]) + self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"]) + self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"]) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + # the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it + if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")): + name += ".bias" + elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates")) + or name in ("depth_memory.stream_gate", "stream_embeddings")): + name += ".weight" + elif name == "depth_memory.stream_gate_bias": + name = "depth_memory.stream_gate.bias" + return super().filter_tensors((name, gen)) + + def _load(self, name: str) -> Tensor: + return LazyTorchTensor.to_eager(self.model_tensors[name]()) + + def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]: + self._generated.add(name) + return name, data + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + hp = self.hparams + r = hp["route_feature_dim"] + n_stream = hp["n_streams"] + T = gguf.MODEL_TENSOR + passes = self._load("pass_embeddings")[: self.n_pass] + + # the mixes depend only on the weights and the pass, so Sinkhorn runs once here + # the controller runs in the checkpoint dtype, as the reference does + for il in range(hp["num_hidden_layers"]): + for src, (mix, read, write) in ( + ("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)), + ("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)), + ): + prefix = f"layers.{il}.{src}." + ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias") + mixes, reads, writes = [], [], [] + for p in range(self.n_pass): + delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float() + m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream]) + log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream) + for _ in range(hp["mhc_sinkhorn_iterations"]): + log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True) + log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True) + mixes.append(log_m.exp()) + reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1)) + writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1)) + act = ctrl_w.dtype + yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float()) + yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float()) + yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float()) + + # one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width + meta_w = self._load("depth_memory.metadata_write.weight").float() + mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"] + base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float() + + mean_k * meta_w[:, r]) + pass_e = self._load("depth_memory.pass_embeddings").float() + anchor_e = self._load("depth_memory.anchor_embeddings").float() + n_anchor = len(self.attn_layers) + 1 + entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)]) + yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias) + + def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # the graph reads the concatenated features one block at a time; each block is its own tensor + for i, part in enumerate(parts): + piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:] + yield from super().modify_tensors(piece, template.format(part), bid) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name in self._generated: + yield name, data_torch + return + if name in self._folded: + return + hp = self.hparams + d = hp["hidden_size"] + + if bid is None: + if name == "stream_embeddings.weight": + # the graph keeps the streams innermost: [n_stream, n_embd, n_tokens] + yield from super().modify_tensors(data_torch.T.contiguous(), name, bid) + return + if name == "depth_memory.metadata_write.weight": + # metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized + r = hp["route_feature_dim"] + meta_w = data_torch.float() + entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"]) + data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1) + elif name == "depth_memory.stream_gate.weight": + data_torch = data_torch.float() / math.sqrt(d) + elif name == "depth_memory.route_projection.weight": + yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid) + return + elif name == "continuation.hidden.weight": + yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid) + return + yield from super().modify_tensors(data_torch, name, bid) + return + + local = name.removeprefix(f"layers.{bid}.") + if local in self._folded_layer: + return + + if local == "mixer.impl.mixer.A_log": + data_torch = -torch.exp(data_torch).unsqueeze(-1) + elif local == "mixer.impl.mixer.D": + data_torch = data_torch.unsqueeze(-1) + elif local == "mixer.impl.mixer.conv1d.weight": + data_torch = data_torch.squeeze(1) + elif local == "mixer.impl.mixer.norm.weight": + data_torch = data_torch.reshape(hp["mamba_n_groups"], -1) + elif local == "mixer.pass_gates.weight": + # the reference applies tanh(gate) in the checkpoint dtype + data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float() + elif local == "moe.shared_expert.gate_up.weight": + gate, up = data_torch.chunk(2, dim=0) + yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid) + yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: + # FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32 + if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate")) + or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight", + "hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 37df033ef..3b8a33597 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -198,6 +198,7 @@ pre_computed_hashes = [ # jina-v2-de variants {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"}, {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"}, + {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Lernex/Metis-1.6-Logos", "chkhsh": "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9"}, # lfm2 variants {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"}, # hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ad4e675f6..c4c4a4c65 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -203,6 +203,7 @@ class Keys: ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank" VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank" GATE_LORA_RANK = "{arch}.attention.gate_lora_rank" + PASS_LORA_RANK = "{arch}.attention.pass_lora_rank" REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count" SLIDING_WINDOW = "{arch}.attention.sliding_window" SCALE = "{arch}.attention.scale" @@ -315,6 +316,12 @@ class Keys: L_CYCLES = "{arch}.hrm.l_cycles" PREFIX_LM = "{arch}.hrm.prefix_lm" + class Metis: + PASS_COUNT = "{arch}.metis.pass_count" + MEMORY_LENGTH = "{arch}.metis.memory_length" + ROUTE_LENGTH = "{arch}.metis.route_length" + CONTINUATION_LOGIT_LIMIT = "{arch}.metis.continuation_logit_limit" + class SSM: CONV_KERNEL = "{arch}.ssm.conv_kernel" INNER_SIZE = "{arch}.ssm.inner_size" @@ -322,6 +329,7 @@ class Keys: TIME_STEP_RANK = "{arch}.ssm.time_step_rank" GROUP_COUNT = "{arch}.ssm.group_count" DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms" + NORM_EPSILON = "{arch}.ssm.norm_epsilon" class KDA: HEAD_DIM = "{arch}.kda.head_dim" @@ -681,6 +689,7 @@ class MODEL_ARCH(IntEnum): NEMOTRON = auto() NEMOTRON_H = auto() NEMOTRON_H_MOE = auto() + METIS16 = auto() EXAONE = auto() EXAONE4 = auto() EXAONE_MOE = auto() @@ -830,6 +839,36 @@ class MODEL_TENSOR(IntEnum): FFN_GATE_TID2EID = auto() MOE_LATENT_DOWN = auto() # nemotron 3 super MOE_LATENT_UP = auto() # nemotron 3 super + STREAM_EMBD = auto() + DEPTH_MEM_STATE_WRITE = auto() + DEPTH_MEM_META_WRITE = auto() + DEPTH_MEM_ENTRY_BIAS = auto() + DEPTH_MEM_QUERY = auto() + DEPTH_MEM_KEY = auto() + DEPTH_MEM_VALUE = auto() + DEPTH_MEM_OUT = auto() + DEPTH_MEM_ROUTE_STATE = auto() + DEPTH_MEM_ROUTE_RETR = auto() + DEPTH_MEM_ROUTE_DIFF = auto() + DEPTH_MEM_ROUTE_HIST = auto() + DEPTH_MEM_ROUTE = auto() + DEPTH_MEM_STREAM_GATE = auto() + CONT_HIDDEN_STATE = auto() + CONT_HIDDEN_RETR = auto() + CONT_HIDDEN_DIFF = auto() + CONT_HIDDEN_ROUTE = auto() + CONT_HIDDEN = auto() + CONT_OUTPUT = auto() + HC_ATTN_MIX = auto() + HC_ATTN_READ = auto() + HC_ATTN_WRITE = auto() + HC_FFN_MIX = auto() + HC_FFN_READ = auto() + HC_FFN_WRITE = auto() + ATTN_PASS_LORA_DOWN = auto() + ATTN_PASS_LORA_UP = auto() + ATTN_PASS_GATE = auto() + FFN_EXP_EMBD = auto() FFN_RESID_ALPHA = auto() FFN_RESID_BETA = auto() FFN_PRE_EXP_NORM = auto() @@ -1558,6 +1597,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.NEMOTRON: "nemotron", MODEL_ARCH.NEMOTRON_H: "nemotron_h", MODEL_ARCH.NEMOTRON_H_MOE: "nemotron_h_moe", + MODEL_ARCH.METIS16: "metis16", MODEL_ARCH.EXAONE: "exaone", MODEL_ARCH.EXAONE4: "exaone4", MODEL_ARCH.EXAONE_MOE: "exaone-moe", @@ -1708,6 +1748,36 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid", MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super + MODEL_TENSOR.STREAM_EMBD: "stream_embd", + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: "depth_mem_state_write", + MODEL_TENSOR.DEPTH_MEM_META_WRITE: "depth_mem_meta_write", + MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS: "depth_mem_entry_bias", + MODEL_TENSOR.DEPTH_MEM_QUERY: "depth_mem_query", + MODEL_TENSOR.DEPTH_MEM_KEY: "depth_mem_key", + MODEL_TENSOR.DEPTH_MEM_VALUE: "depth_mem_value", + MODEL_TENSOR.DEPTH_MEM_OUT: "depth_mem_out", + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: "depth_mem_route_state", + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: "depth_mem_route_retr", + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: "depth_mem_route_diff", + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: "depth_mem_route_hist", + MODEL_TENSOR.DEPTH_MEM_ROUTE: "depth_mem_route", + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: "depth_mem_stream_gate", + MODEL_TENSOR.CONT_HIDDEN_STATE: "cont_hidden_state", + MODEL_TENSOR.CONT_HIDDEN_RETR: "cont_hidden_retr", + MODEL_TENSOR.CONT_HIDDEN_DIFF: "cont_hidden_diff", + MODEL_TENSOR.CONT_HIDDEN_ROUTE: "cont_hidden_route", + MODEL_TENSOR.CONT_HIDDEN: "cont_hidden", + MODEL_TENSOR.CONT_OUTPUT: "cont_output", + MODEL_TENSOR.HC_ATTN_MIX: "blk.{bid}.hc_attn_mix", + MODEL_TENSOR.HC_ATTN_READ: "blk.{bid}.hc_attn_read", + MODEL_TENSOR.HC_ATTN_WRITE: "blk.{bid}.hc_attn_write", + MODEL_TENSOR.HC_FFN_MIX: "blk.{bid}.hc_ffn_mix", + MODEL_TENSOR.HC_FFN_READ: "blk.{bid}.hc_ffn_read", + MODEL_TENSOR.HC_FFN_WRITE: "blk.{bid}.hc_ffn_write", + MODEL_TENSOR.ATTN_PASS_LORA_DOWN: "blk.{bid}.attn_pass_lora_down", + MODEL_TENSOR.ATTN_PASS_LORA_UP: "blk.{bid}.attn_pass_lora_up", + MODEL_TENSOR.ATTN_PASS_GATE: "blk.{bid}.attn_pass_gate", + MODEL_TENSOR.FFN_EXP_EMBD: "blk.{bid}.ffn_exp_embd", MODEL_TENSOR.FFN_RESID_ALPHA: "blk.{bid}.ffn_resid_alpha", MODEL_TENSOR.FFN_RESID_BETA: "blk.{bid}.ffn_resid_beta", MODEL_TENSOR.FFN_PRE_EXP_NORM: "blk.{bid}.ffn_pre_exp_norm", @@ -4917,6 +4987,61 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.METIS16: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.STREAM_EMBD, + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE, + MODEL_TENSOR.DEPTH_MEM_META_WRITE, + MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS, + MODEL_TENSOR.DEPTH_MEM_QUERY, + MODEL_TENSOR.DEPTH_MEM_KEY, + MODEL_TENSOR.DEPTH_MEM_VALUE, + MODEL_TENSOR.DEPTH_MEM_OUT, + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE, + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR, + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF, + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST, + MODEL_TENSOR.DEPTH_MEM_ROUTE, + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE, + MODEL_TENSOR.CONT_HIDDEN_STATE, + MODEL_TENSOR.CONT_HIDDEN_RETR, + MODEL_TENSOR.CONT_HIDDEN_DIFF, + MODEL_TENSOR.CONT_HIDDEN_ROUTE, + MODEL_TENSOR.CONT_HIDDEN, + MODEL_TENSOR.CONT_OUTPUT, + MODEL_TENSOR.HC_ATTN_MIX, + MODEL_TENSOR.HC_ATTN_READ, + MODEL_TENSOR.HC_ATTN_WRITE, + MODEL_TENSOR.HC_FFN_MIX, + MODEL_TENSOR.HC_FFN_READ, + MODEL_TENSOR.HC_FFN_WRITE, + MODEL_TENSOR.ATTN_PASS_LORA_DOWN, + MODEL_TENSOR.ATTN_PASS_LORA_UP, + MODEL_TENSOR.ATTN_PASS_GATE, + MODEL_TENSOR.FFN_EXP_EMBD, + MODEL_TENSOR.SSM_IN, + MODEL_TENSOR.SSM_CONV1D, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_D, + MODEL_TENSOR.SSM_NORM, + MODEL_TENSOR.SSM_OUT, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.MOE_LATENT_DOWN, + MODEL_TENSOR.MOE_LATENT_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_UP_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + ], MODEL_ARCH.NEMOTRON_H: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 56e8938c4..0e7aa5012 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -975,6 +975,18 @@ class GGUFWriter: def add_hrm_prefix_lm(self, value: bool) -> None: self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value) + def add_metis_pass_count(self, value: int) -> None: + self.add_uint32(Keys.Metis.PASS_COUNT.format(arch=self.arch), value) + + def add_metis_memory_length(self, value: int) -> None: + self.add_uint32(Keys.Metis.MEMORY_LENGTH.format(arch=self.arch), value) + + def add_metis_route_length(self, value: int) -> None: + self.add_uint32(Keys.Metis.ROUTE_LENGTH.format(arch=self.arch), value) + + def add_metis_continuation_logit_limit(self, value: float) -> None: + self.add_float32(Keys.Metis.CONTINUATION_LOGIT_LIMIT.format(arch=self.arch), value) + def add_adapter_count(self, count: int) -> None: self.add_uint32(Keys.Adapters.COUNT.format(arch=self.arch), count) @@ -1086,6 +1098,9 @@ class GGUFWriter: def add_attention_output_lora_rank(self, length: int) -> None: self.add_uint32(Keys.Attention.OUTPUT_LORA_RANK.format(arch=self.arch), length) + def add_attention_pass_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.PASS_LORA_RANK.format(arch=self.arch), length) + def add_attention_compress_ratios(self, values: Sequence[int]) -> None: self.add_array(Keys.Attention.COMPRESS_RATIOS.format(arch=self.arch), values) @@ -1267,6 +1282,9 @@ class GGUFWriter: def add_ssm_dt_b_c_rms(self, value: bool) -> None: self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value) + def add_ssm_norm_epsilon(self, value: float) -> None: + self.add_float32(Keys.SSM.NORM_EPSILON.format(arch=self.arch), value) + def add_expert_latent_length(self, value: int) -> None: self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 2602bfb61..ff22a60bc 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -35,6 +35,7 @@ class TensorNameMap: "model.transformer.wte", # llada "embed_tokens", # qwen3-embedding "model.embed", # talkie + "embedding", # metis16 ), # Masked embeddings @@ -224,6 +225,7 @@ class TensorNameMap: block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = { # Attention norm MODEL_TENSOR.ATTN_NORM: ( + "layers.{bid}.mixer_connection.norm", # metis16 "model.lilicorr.layers.{bid}.attn_norm", # lilicorr "gpt_neox.layers.{bid}.input_layernorm", # gptneox "transformer.h.{bid}.ln_1", # gpt2 gpt-j refact qwen jais exaone @@ -294,6 +296,7 @@ class TensorNameMap: "model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 "head.layers.{bid}.self_attn.in_proj", # laya "layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 text + "layers.{bid}.mixer.qkv", # metis16 ), # Attention query @@ -401,6 +404,7 @@ class TensorNameMap: "model.layers.{bid}.self_attn.language_expert_dense", # cogvlm "model.blocks.{bid}.attn.attn_resid", # talkie "head.layers.{bid}.self_attn.out_proj", # laya + "layers.{bid}.mixer.out", # metis16 ), # Attention output norm @@ -449,6 +453,7 @@ class TensorNameMap: # Feed-forward norm MODEL_TENSOR.FFN_NORM: ( + "layers.{bid}.moe_connection.norm", # metis16 "model.lilicorr.layers.{bid}.mlp_norm", # lilicorr "model.lilicorr.feature_mlp.{bid}.norm", # lilicorr "gpt_neox.layers.{bid}.post_attention_layernorm", # gptneox @@ -536,6 +541,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.gate", # nemotron-h-moe "model.layers.{bid}.moe.gate", # step3.5 "model.layers.{bid}.router.proj", # gemma4 + "layers.{bid}.moe.expert_router", # metis16 ), MODEL_TENSOR.FFN_GATE_INP_SHEXP: ( @@ -543,6 +549,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_EXP_PROBS_B: ( + "layers.{bid}.moe.selection_bias", # metis16 "model.layers.{bid}.mlp.gate.e_score_correction", # deepseek-v3 dots1 "model.layers.{bid}.mlp.gate.e_score_correction_bias", # muse-spark "model.layers.{bid}.mlp.moe_statics.e_score_correction", # ernie4.5-moe @@ -617,6 +624,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_UP_SHEXP: ( + "layers.{bid}.moe.shared_expert.up", # metis16 "model.layers.{bid}.mlp.shared_expert.up_proj", # qwen2moe "model.layers.{bid}.mlp.shared_experts.up_proj", # deepseek deepseek2 "model.layers.{bid}.feed_forward.shared_expert.up_proj", # llama4 @@ -670,6 +678,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_GATE_SHEXP: ( + "layers.{bid}.moe.shared_expert.gate", # metis16 "model.layers.{bid}.mlp.shared_expert.gate_proj", # qwen2moe "model.layers.{bid}.mlp.shared_experts.gate_proj", # deepseek deepseek2 "model.layers.{bid}.feed_forward.shared_expert.gate_proj", # llama4 @@ -686,18 +695,94 @@ class TensorNameMap: MODEL_TENSOR.FFN_GATE_UP_EXP: ( "model.layers.{bid}.mlp.experts.gate_up_proj", "model.layers.{bid}.experts.gate_up_proj", # gemma4 + "layers.{bid}.moe.local_experts.gate_up", # metis16 ), MODEL_TENSOR.MOE_LATENT_DOWN: ( "backbone.layers.{bid}.mixer.fc1_latent_proj", # nemotron 3 super "model.layers.{bid}.mlp.pre_expert_proj", # muse-spark + "layers.{bid}.moe.latent_down", # metis16 ), MODEL_TENSOR.MOE_LATENT_UP: ( "backbone.layers.{bid}.mixer.fc2_latent_proj", # nemotron 3 super "model.layers.{bid}.mlp.post_expert_proj", # muse-spark + "layers.{bid}.moe.latent_up", # metis16 ), + MODEL_TENSOR.STREAM_EMBD: ( + "stream_embeddings", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: ( + "depth_memory.state_write", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_META_WRITE: ( + "depth_memory.metadata_write", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_QUERY: ( + "depth_memory.query", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_KEY: ( + "depth_memory.key", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_VALUE: ( + "depth_memory.value", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_OUT: ( + "depth_memory.output", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: ( + "depth_memory.route_state", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: ( + "depth_memory.route_retr", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: ( + "depth_memory.route_diff", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: ( + "depth_memory.route_hist", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE: ( + "depth_memory.route_projection", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: ( + "depth_memory.stream_gate", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_STATE: ( + "continuation.hidden_state", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_RETR: ( + "continuation.hidden_retr", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_DIFF: ( + "continuation.hidden_diff", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_ROUTE: ( + "continuation.hidden_route", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN: ( + "continuation.hidden", # metis16 + ), + MODEL_TENSOR.CONT_OUTPUT: ( + "continuation.output", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_LORA_DOWN: ( + "layers.{bid}.mixer.pass_lora_down", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_LORA_UP: ( + "layers.{bid}.mixer.pass_lora_up", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_GATE: ( + "layers.{bid}.mixer.pass_gates", # metis16 + ), + MODEL_TENSOR.FFN_EXP_EMBD: ( + "layers.{bid}.moe.expert_embeddings", # metis16 + ), + + + + MODEL_TENSOR.FFN_PRE_EXP_NORM: ( "model.layers.{bid}.mlp.pre_expert_norm", # muse-spark ), @@ -759,6 +844,7 @@ class TensorNameMap: "model.layers.{bid}.block_sparse_moe.experts.down", # smallthinker "model.layers.{bid}.moe.down_proj", # step3.5 "model.layers.{bid}.experts.down_proj", # gemma4 + "layers.{bid}.moe.local_experts.down", # metis16 ), MODEL_TENSOR.FFN_DOWN_SHEXP: ( @@ -771,6 +857,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.shared_experts.down_proj", # nemotron-h-moe "model.layers.{bid}.block_sparse_moe.shared_experts.down_proj", # kimi "model.layers.{bid}.share_expert.down_proj", # step3.5 + "layers.{bid}.moe.shared_expert.down", # metis16 ), MODEL_TENSOR.FFN_DOWN_CHEXP: ( @@ -906,6 +993,7 @@ class TensorNameMap: "model.layers.{bid}.mamba.in_proj", # jamba falcon-h1 granite-hybrid "model.layers.layers.{bid}.mixer.in_proj", # plamo2 "model.layers.{bid}.linear_attn.in_proj_qkvz", # qwen3next + "layers.{bid}.mixer.impl.mixer.in_proj", # metis16 ), MODEL_TENSOR.SSM_CONV1D: ( @@ -915,6 +1003,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mixer.conv1d", # plamo2 "model.layers.{bid}.linear_attn.conv1d", # qwen3next "layers.{bid}.linear_attn.conv1d", # qwen3.5 text + "layers.{bid}.mixer.impl.mixer.conv1d", # metis16 ), MODEL_TENSOR.SSM_X: ( @@ -925,6 +1014,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_DT: ( + "layers.{bid}.mixer.impl.mixer.dt_bias", # metis16 "model.layers.{bid}.dt_proj", # mamba-hf "backbone.layers.{bid}.mixer.dt_proj", # mamba "model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid @@ -942,6 +1032,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_A: ( + "layers.{bid}.mixer.impl.mixer.A_log", # metis16 "model.layers.{bid}.A_log", # mamba-hf "backbone.layers.{bid}.mixer.A_log", # mamba "model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid @@ -965,6 +1056,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_D: ( + "layers.{bid}.mixer.impl.mixer.D", # metis16 "model.layers.{bid}.D", # mamba-hf "backbone.layers.{bid}.mixer.D", # mamba "model.layers.{bid}.mamba.D", # jamba falcon-h1 granite-hybrid @@ -972,6 +1064,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_NORM: ( + "layers.{bid}.mixer.impl.mixer.norm", # metis16 "model.layers.{bid}.mamba.norm", # falcon-h1 granite-hybrid "model.layers.{bid}.linear_attn.norm", # qwen3next "layers.{bid}.linear_attn.norm", # qwen3.5 text @@ -987,6 +1080,7 @@ class TensorNameMap: "model.layers.{bid}.linear_attn.out_proj", # qwen3next "layers.{bid}.linear_attn.out_proj", # qwen3.5 text "model.layers.layers.{bid}.mixer.out_proj", # plamo2 + "layers.{bid}.mixer.impl.mixer.out_proj", # metis16 ), MODEL_TENSOR.SSM_ALPHA: ( diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 525f91031..92a13e985 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -102,6 +102,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_NEMOTRON, "nemotron" }, { LLM_ARCH_NEMOTRON_H, "nemotron_h" }, { LLM_ARCH_NEMOTRON_H_MOE, "nemotron_h_moe" }, + { LLM_ARCH_METIS16, "metis16" }, { LLM_ARCH_EXAONE, "exaone" }, { LLM_ARCH_EXAONE4, "exaone4" }, { LLM_ARCH_EXAONE_MOE, "exaone-moe" }, @@ -272,6 +273,10 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" }, { LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" }, { LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" }, + { LLM_KV_METIS_PASS_COUNT, "%s.metis.pass_count" }, + { LLM_KV_METIS_MEMORY_LENGTH, "%s.metis.memory_length" }, + { LLM_KV_METIS_ROUTE_LENGTH, "%s.metis.route_length" }, + { LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, "%s.metis.continuation_logit_limit" }, { LLM_KV_MUDD_FEED_FORWARD_LENGTH, "%s.mudd.feed_forward_length" }, { LLM_KV_MUDD_TAP_COUNT, "%s.mudd.tap_count" }, { LLM_KV_MUDD_TAP_INDICES, "%s.mudd.tap_indices" }, @@ -329,6 +334,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_KV_SOURCE_LAYERS, "%s.attention.kv_source_layers" }, { LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" }, { LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" }, + { LLM_KV_ATTENTION_PASS_LORA_RANK, "%s.attention.pass_lora_rank" }, { LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" }, { LLM_KV_ATTENTION_COMPRESS_RATIOS, "%s.attention.compress_ratios" }, { LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" }, @@ -387,6 +393,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" }, { LLM_KV_SSM_GROUP_COUNT, "%s.ssm.group_count" }, { LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" }, + { LLM_KV_SSM_NORM_EPS, "%s.ssm.norm_epsilon" }, { LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" }, { LLM_KV_KDA_SAFE_GATE, "%s.kda.safe_gate" }, @@ -548,6 +555,36 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" }, { LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" }, { LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" }, + { LLM_TENSOR_STREAM_EMBD, "stream_embd" }, + { LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "depth_mem_state_write" }, + { LLM_TENSOR_DEPTH_MEM_META_WRITE, "depth_mem_meta_write" }, + { LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "depth_mem_entry_bias" }, + { LLM_TENSOR_DEPTH_MEM_QUERY, "depth_mem_query" }, + { LLM_TENSOR_DEPTH_MEM_KEY, "depth_mem_key" }, + { LLM_TENSOR_DEPTH_MEM_VALUE, "depth_mem_value" }, + { LLM_TENSOR_DEPTH_MEM_OUT, "depth_mem_out" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "depth_mem_route_state" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "depth_mem_route_retr" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "depth_mem_route_diff" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "depth_mem_route_hist" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE, "depth_mem_route" }, + { LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "depth_mem_stream_gate" }, + { LLM_TENSOR_CONT_HIDDEN_STATE, "cont_hidden_state" }, + { LLM_TENSOR_CONT_HIDDEN_RETR, "cont_hidden_retr" }, + { LLM_TENSOR_CONT_HIDDEN_DIFF, "cont_hidden_diff" }, + { LLM_TENSOR_CONT_HIDDEN_ROUTE, "cont_hidden_route" }, + { LLM_TENSOR_CONT_HIDDEN, "cont_hidden" }, + { LLM_TENSOR_CONT_OUTPUT, "cont_output" }, + { LLM_TENSOR_HC_ATTN_MIX, "blk.%d.hc_attn_mix" }, + { LLM_TENSOR_HC_ATTN_READ, "blk.%d.hc_attn_read" }, + { LLM_TENSOR_HC_ATTN_WRITE, "blk.%d.hc_attn_write" }, + { LLM_TENSOR_HC_FFN_MIX, "blk.%d.hc_ffn_mix" }, + { LLM_TENSOR_HC_FFN_READ, "blk.%d.hc_ffn_read" }, + { LLM_TENSOR_HC_FFN_WRITE, "blk.%d.hc_ffn_write" }, + { LLM_TENSOR_ATTN_PASS_LORA_DOWN, "blk.%d.attn_pass_lora_down" }, + { LLM_TENSOR_ATTN_PASS_LORA_UP, "blk.%d.attn_pass_lora_up" }, + { LLM_TENSOR_ATTN_PASS_GATE, "blk.%d.attn_pass_gate" }, + { LLM_TENSOR_FFN_EXP_EMBD, "blk.%d.ffn_exp_embd" }, { LLM_TENSOR_FFN_PRE_EXP_NORM, "blk.%d.ffn_pre_exp_norm" }, { LLM_TENSOR_FFN_POST_EXP_NORM, "blk.%d.ffn_post_exp_norm" }, { LLM_TENSOR_FFN_RESID_ALPHA, "blk.%d.ffn_resid_alpha" }, @@ -1158,6 +1195,36 @@ static const std::map LLM_TENSOR_INFOS = { // latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU {LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_FFN_LATENT_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_STREAM_EMBD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}}, + {LLM_TENSOR_DEPTH_MEM_STATE_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_META_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_KEY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_VALUE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_STREAM_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_LORA_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_LORA_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_FFN_EXP_EMBD, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}}, {LLM_TENSOR_FFN_PRE_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_FFN_POST_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_FFN_RESID_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, @@ -1286,6 +1353,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) { case LLM_ARCH_LFM3: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_METIS16: case LLM_ARCH_QWEN3NEXT: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_BAILINGMOE3: diff --git a/src/llama-arch.h b/src/llama-arch.h index b98be40cf..f811607b6 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -106,6 +106,7 @@ enum llm_arch { LLM_ARCH_NEMOTRON, LLM_ARCH_NEMOTRON_H, LLM_ARCH_NEMOTRON_H_MOE, + LLM_ARCH_METIS16, LLM_ARCH_EXAONE, LLM_ARCH_EXAONE4, LLM_ARCH_EXAONE_MOE, @@ -276,6 +277,10 @@ enum llm_kv { LLM_KV_HRM_H_CYCLES, LLM_KV_HRM_L_CYCLES, LLM_KV_HRM_PREFIX_LM, + LLM_KV_METIS_PASS_COUNT, + LLM_KV_METIS_MEMORY_LENGTH, + LLM_KV_METIS_ROUTE_LENGTH, + LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, LLM_KV_REALTIME_DELAY_TOKENS, LLM_KV_MUDD_FEED_FORWARD_LENGTH, LLM_KV_MUDD_TAP_COUNT, @@ -333,6 +338,7 @@ enum llm_kv { LLM_KV_ATTENTION_KV_SOURCE_LAYERS, LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, LLM_KV_ATTENTION_OUTPUT_LORA_RANK, + LLM_KV_ATTENTION_PASS_LORA_RANK, LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, LLM_KV_ATTENTION_COMPRESS_RATIOS, LLM_KV_ATTENTION_SHARED_KV_LAYERS, @@ -391,6 +397,7 @@ enum llm_kv { LLM_KV_SSM_TIME_STEP_RANK, LLM_KV_SSM_GROUP_COUNT, LLM_KV_SSM_DT_B_C_RMS, + LLM_KV_SSM_NORM_EPS, LLM_KV_KDA_HEAD_DIM, LLM_KV_KDA_SAFE_GATE, @@ -566,6 +573,36 @@ enum llm_tensor { LLM_TENSOR_FFN_EXP_PROBS_B_VL, LLM_TENSOR_FFN_LATENT_DOWN, LLM_TENSOR_FFN_LATENT_UP, + LLM_TENSOR_STREAM_EMBD, + LLM_TENSOR_DEPTH_MEM_STATE_WRITE, + LLM_TENSOR_DEPTH_MEM_META_WRITE, + LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, + LLM_TENSOR_DEPTH_MEM_QUERY, + LLM_TENSOR_DEPTH_MEM_KEY, + LLM_TENSOR_DEPTH_MEM_VALUE, + LLM_TENSOR_DEPTH_MEM_OUT, + LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, + LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, + LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, + LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, + LLM_TENSOR_DEPTH_MEM_ROUTE, + LLM_TENSOR_DEPTH_MEM_STREAM_GATE, + LLM_TENSOR_CONT_HIDDEN_STATE, + LLM_TENSOR_CONT_HIDDEN_RETR, + LLM_TENSOR_CONT_HIDDEN_DIFF, + LLM_TENSOR_CONT_HIDDEN_ROUTE, + LLM_TENSOR_CONT_HIDDEN, + LLM_TENSOR_CONT_OUTPUT, + LLM_TENSOR_HC_ATTN_MIX, + LLM_TENSOR_HC_ATTN_READ, + LLM_TENSOR_HC_ATTN_WRITE, + LLM_TENSOR_HC_FFN_MIX, + LLM_TENSOR_HC_FFN_READ, + LLM_TENSOR_HC_FFN_WRITE, + LLM_TENSOR_ATTN_PASS_LORA_DOWN, + LLM_TENSOR_ATTN_PASS_LORA_UP, + LLM_TENSOR_ATTN_PASS_GATE, + LLM_TENSOR_FFN_EXP_EMBD, LLM_TENSOR_FFN_PRE_EXP_NORM, LLM_TENSOR_FFN_POST_EXP_NORM, LLM_TENSOR_FFN_RESID_ALPHA, diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0cd795a56..eb9756996 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2703,6 +2703,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { res = std::max(n_tokens * 80, 64u * model.n_tensors()); } else if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || + model.arch == LLM_ARCH_METIS16 || model.arch == LLM_ARCH_BAILINGMOE3 || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE || diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 4d2cb8931..95daf482d 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -209,6 +209,7 @@ struct llama_hparams { uint32_t ssm_d_state = 0; uint32_t ssm_dt_rank = 0; uint32_t ssm_n_group = 0; + float ssm_norm_eps = 0.0f; // 0 = use f_norm_rms_eps // for MiniMax-Text-01 linear attention uint32_t n_embd_head_la = 0; @@ -230,6 +231,13 @@ struct llama_hparams { uint32_t n_hrm_l_cycles = 0; bool hrm_prefix_lm = false; + // metis16 (passes over shared layers, depth memory) + uint32_t metis_n_pass = 0; + uint32_t metis_n_mem = 0; + uint32_t metis_n_route = 0; + uint32_t metis_pass_lora_rank = 0; + float metis_cont_limit = 0.0f; + // limite float f_xsa_eps = 0.0f; std::array final_logit_sigmoid_capping = {}; // scale, shift, temperature diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 36f3850d5..bfa1b547c 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -247,6 +247,10 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles); add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles); add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm); + add_kv(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass); + add_kv(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem); + add_kv(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route); + add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit); if (hparams.final_logit_sigmoid_capping[0] != 0.0f) { add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector( hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end())); @@ -323,6 +327,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true); add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count); add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank); + add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank); add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base); if (model->arch == LLM_ARCH_DEEPSEEK4 || hparams.dsv4_hc_mult > 0) { // the loader requires one compress ratio per layer, including nextn layers @@ -419,6 +424,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); add_kv(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms); + add_kv(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps); add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda); add_kv(LLM_KV_KDA_SAFE_GATE, hparams.kda_safe_gate); @@ -521,6 +527,32 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->cls_out_b); add_tensor(model->cls_norm); add_tensor(model->hrm_z_l_init); + add_tensor(model->stream_embd); + add_tensor(model->depth_mem_state_write); + add_tensor(model->depth_mem_meta_write); + add_tensor(model->depth_mem_entry_bias); + add_tensor(model->depth_mem_query); + add_tensor(model->depth_mem_query_b); + add_tensor(model->depth_mem_key); + add_tensor(model->depth_mem_key_b); + add_tensor(model->depth_mem_value); + add_tensor(model->depth_mem_value_b); + add_tensor(model->depth_mem_out); + add_tensor(model->depth_mem_out_b); + add_tensor(model->depth_mem_route_state); + add_tensor(model->depth_mem_route_retr); + add_tensor(model->depth_mem_route_diff); + add_tensor(model->depth_mem_route_hist); + add_tensor(model->depth_mem_route_b); + add_tensor(model->depth_mem_stream_gate); + add_tensor(model->depth_mem_stream_gate_b); + add_tensor(model->cont_hidden_state); + add_tensor(model->cont_hidden_retr); + add_tensor(model->cont_hidden_diff); + add_tensor(model->cont_hidden_route); + add_tensor(model->cont_hidden_b); + add_tensor(model->cont_output); + add_tensor(model->cont_output_b); add_tensor(model->hc_head_fn); add_tensor(model->hc_head_base); add_tensor(model->hc_head_scale); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 87893bf67..3c4e598f1 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -242,6 +242,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_nemotron_h(params); case LLM_ARCH_NEMOTRON_H_MOE: return new llama_model_nemotron_h_moe(params); + case LLM_ARCH_METIS16: + return new llama_model_metis16(params); case LLM_ARCH_EXAONE: return new llama_model_exaone(params); case LLM_ARCH_EXAONE4: @@ -515,7 +517,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str }; auto get_tensor_config = [&]() -> tensor_config { - if (ud->model->arch == LLM_ARCH_HRM_TEXT) { + if (ud->model->arch == LLM_ARCH_HRM_TEXT || ud->model->arch == LLM_ARCH_METIS16) { // aliased cache slots cannot satisfy the meta-split invariants, so replicate all tensors return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, tensor, 0, 0}; } @@ -1049,6 +1051,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_7B: return "7B"; case LLM_TYPE_8B: return "8B"; case LLM_TYPE_9B: return "9B"; + case LLM_TYPE_10B: return "10B"; case LLM_TYPE_11B: return "11B"; case LLM_TYPE_12B: return "12B"; case LLM_TYPE_13B: return "13B"; @@ -3618,6 +3621,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_WAVTOKENIZER_DEC: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_METIS16: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_KIMI_K3: case LLM_ARCH_GLM5_NEXT: diff --git a/src/llama-model.h b/src/llama-model.h index da4964c98..45d2b8666 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -76,6 +76,7 @@ enum llm_type { LLM_TYPE_7B, LLM_TYPE_8B, LLM_TYPE_9B, + LLM_TYPE_10B, LLM_TYPE_11B, LLM_TYPE_12B, LLM_TYPE_13B, @@ -646,6 +647,18 @@ struct llama_layer { // gemma4 layer output scale, reused for talkie embedding skip scale struct ggml_tensor * out_scale = nullptr; + // metis16 per-pass hyper-connection mixes, attention pass LoRA, expert route embeddings + struct ggml_tensor * hc_attn_mix = nullptr; + struct ggml_tensor * hc_attn_read = nullptr; + struct ggml_tensor * hc_attn_write = nullptr; + struct ggml_tensor * hc_ffn_mix = nullptr; + struct ggml_tensor * hc_ffn_read = nullptr; + struct ggml_tensor * hc_ffn_write = nullptr; + struct ggml_tensor * attn_pass_lora_down = nullptr; + struct ggml_tensor * attn_pass_lora_up = nullptr; + struct ggml_tensor * attn_pass_gate = nullptr; + struct ggml_tensor * ffn_exp_embd = nullptr; + // limite struct ggml_tensor * attn_ve_gate = nullptr; struct ggml_tensor * attn_xsa_alpha = nullptr; @@ -779,6 +792,34 @@ struct llama_model { // hrm-text initial low-cycle state struct ggml_tensor * hrm_z_l_init = nullptr; + // metis16 hyper-connection stream embeddings, depth memory and continuation head + struct ggml_tensor * stream_embd = nullptr; + struct ggml_tensor * depth_mem_state_write = nullptr; + struct ggml_tensor * depth_mem_meta_write = nullptr; + struct ggml_tensor * depth_mem_entry_bias = nullptr; + struct ggml_tensor * depth_mem_query = nullptr; + struct ggml_tensor * depth_mem_query_b = nullptr; + struct ggml_tensor * depth_mem_key = nullptr; + struct ggml_tensor * depth_mem_key_b = nullptr; + struct ggml_tensor * depth_mem_value = nullptr; + struct ggml_tensor * depth_mem_value_b = nullptr; + struct ggml_tensor * depth_mem_out = nullptr; + struct ggml_tensor * depth_mem_out_b = nullptr; + struct ggml_tensor * depth_mem_route_state = nullptr; + struct ggml_tensor * depth_mem_route_retr = nullptr; + struct ggml_tensor * depth_mem_route_diff = nullptr; + struct ggml_tensor * depth_mem_route_hist = nullptr; + struct ggml_tensor * depth_mem_route_b = nullptr; + struct ggml_tensor * depth_mem_stream_gate = nullptr; + struct ggml_tensor * depth_mem_stream_gate_b = nullptr; + struct ggml_tensor * cont_hidden_state = nullptr; + struct ggml_tensor * cont_hidden_retr = nullptr; + struct ggml_tensor * cont_hidden_diff = nullptr; + struct ggml_tensor * cont_hidden_route = nullptr; + struct ggml_tensor * cont_hidden_b = nullptr; + struct ggml_tensor * cont_output = nullptr; + struct ggml_tensor * cont_output_b = nullptr; + // DeepSeek-V4 struct ggml_tensor * hc_head_fn = nullptr; struct ggml_tensor * hc_head_base = nullptr; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 34c247858..d62f7d5e9 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -361,6 +361,16 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param quantize &= name.find("indexer.k_proj.weight") == std::string::npos; quantize &= name.find("indexer.q_proj.weight") == std::string::npos; + // metis16: small control tensors that steer every layer; the stream and hc_ tables (ne[0] = 4) would fall back to F16 + if (arch == LLM_ARCH_METIS16) { + quantize &= name.rfind("depth_mem_", 0) == std::string::npos; + quantize &= name.rfind("cont_", 0) == std::string::npos; + quantize &= name.find("stream_embd") == std::string::npos; + quantize &= name.find("attn_pass_lora") == std::string::npos; + quantize &= name.find("ffn_exp_embd") == std::string::npos; + quantize &= name.find("hc_") == std::string::npos; + } + // glm5-next if (arch == LLM_ARCH_GLM5_NEXT) { quantize &= name.find("hc_") == std::string::npos; diff --git a/src/models/mamba-base.cpp b/src/models/mamba-base.cpp index 370de0f01..63af3113d 100644 --- a/src/models/mamba-base.cpp +++ b/src/models/mamba-base.cpp @@ -290,7 +290,11 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp, // grouped RMS norm if (model.layers[il].ssm_norm) { y = ggml_reshape_4d(ctx0, y, d_inner / n_group, n_group, n_seq_tokens, n_seqs); - y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il); + if (hparams.ssm_norm_eps > 0.0f) { + y = ggml_mul(ctx0, ggml_rms_norm(ctx0, y, hparams.ssm_norm_eps), model.layers[il].ssm_norm); + } else { + y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il); + } } y = ggml_reshape_2d(ctx0, y, d_inner, n_seq_tokens * n_seqs); diff --git a/src/models/metis16.cpp b/src/models/metis16.cpp new file mode 100644 index 000000000..4e4b032b9 --- /dev/null +++ b/src/models/metis16.cpp @@ -0,0 +1,401 @@ +#include "models.h" + +#include + +void llama_model_metis16::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); + ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); + ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state); + ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); + ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); + ml.get_key(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps); + + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); + ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size); + + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); + ml.get_key(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass); + ml.get_key(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem); + ml.get_key(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route); + ml.get_key(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank); + ml.get_key(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit); + + n_pass = hparams.metis_n_pass; + n_stream = hparams.dsv4_hc_mult; + n_mem = hparams.metis_n_mem; + n_route = hparams.metis_n_route; + cont_limit = hparams.metis_cont_limit; + + if (n_pass < 1 || hparams.n_layer() % n_pass != 0) { + throw std::runtime_error(format("metis16: %u layers do not split into %u passes", hparams.n_layer(), n_pass)); + } + n_layer_phys = hparams.n_layer() / n_pass; + + bool has_attn = false; + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (hparams.n_head_kv(il) != hparams.n_head_kv(il % n_layer_phys)) { + throw std::runtime_error(format("metis16: layer %u does not repeat the layout of layer %u", il, il % n_layer_phys)); + } + hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0; + has_attn = has_attn || !hparams.is_recr_impl[il]; + } + if (!has_attn) { + throw std::runtime_error("metis16: at least one attention layer is required"); + } + + switch (n_layer_phys) { + case 12: type = LLM_TYPE_3B; break; // Praxis + case 20: type = LLM_TYPE_10B; break; // Logos + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_metis16::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = hparams.ssm_d_inner; + const int64_t d_state = hparams.ssm_d_state; + const int64_t n_ssm_head = hparams.ssm_dt_rank; + const int64_t n_group = hparams.ssm_n_group; + const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_ssm_head; + const int64_t n_latent = hparams.moe_latent_size; + const int64_t n_ff_shexp = hparams.n_ff_shexp; + const int64_t n_s = n_stream; + const int64_t n_p = n_pass; + const int64_t n_m = n_mem; + const int64_t n_r = n_route; + + int64_t n_attn = 0; + for (uint32_t il = 0; il < n_layer_phys; ++il) { + n_attn += !hparams.is_recr(il); + } + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + stream_embd = create_tensor(tn(LLM_TENSOR_STREAM_EMBD, "weight"), {n_s, n_embd}, 0); + + depth_mem_state_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "weight"), {n_embd, n_m}, 0); + depth_mem_meta_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_META_WRITE, "weight"), {n_r + 3, n_m}, 0); + depth_mem_entry_bias = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "weight"), {n_m, n_p*(n_attn + 1)}, 0); + depth_mem_query = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "weight"), {n_embd, n_m}, 0); + depth_mem_query_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "bias"), {n_m}, 0); + depth_mem_key = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "weight"), {n_m, n_m}, 0); + depth_mem_key_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "bias"), {n_m}, 0); + depth_mem_value = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "weight"), {n_m, n_m}, 0); + depth_mem_value_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "bias"), {n_m}, 0); + depth_mem_out = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "weight"), {n_m, n_embd}, 0); + depth_mem_out_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "bias"), {n_embd}, 0); + depth_mem_route_state = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "weight"), {n_embd, n_r}, 0); + depth_mem_route_retr = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "weight"), {n_embd, n_r}, 0); + depth_mem_route_diff = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "weight"), {n_embd, n_r}, 0); + depth_mem_route_hist = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "weight"), {n_r, n_r}, 0); + depth_mem_route_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE, "bias"), {n_r}, 0); + depth_mem_stream_gate = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "weight"), {n_embd, n_s}, 0); + depth_mem_stream_gate_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "bias"), {n_s}, 0); + + cont_hidden_state = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_STATE, "weight"), {n_embd, n_r}, 0); + cont_hidden_retr = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_RETR, "weight"), {n_embd, n_r}, 0); + cont_hidden_diff = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_DIFF, "weight"), {n_embd, n_r}, 0); + cont_hidden_route = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_ROUTE, "weight"), {n_r, n_r}, 0); + cont_hidden_b = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN, "bias"), {n_r}, 0); + cont_output = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "weight"), {n_r, 1}, 0); + cont_output_b = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "bias"), {1}, 0); + + for (int il = 0; il < (int) n_layer_phys; ++il) { + auto & layer = layers[il]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), {n_embd}, 0); + layer.hc_attn_mix = create_tensor(tn(LLM_TENSOR_HC_ATTN_MIX, "weight", il), {n_s, n_s, n_p}, 0); + layer.hc_attn_read = create_tensor(tn(LLM_TENSOR_HC_ATTN_READ, "weight", il), {n_s, n_p}, 0); + layer.hc_attn_write = create_tensor(tn(LLM_TENSOR_HC_ATTN_WRITE, "weight", il), {n_s, n_p}, 0); + layer.hc_ffn_mix = create_tensor(tn(LLM_TENSOR_HC_FFN_MIX, "weight", il), {n_s, n_s, n_p}, 0); + layer.hc_ffn_read = create_tensor(tn(LLM_TENSOR_HC_FFN_READ, "weight", il), {n_s, n_p}, 0); + layer.hc_ffn_write = create_tensor(tn(LLM_TENSOR_HC_FFN_WRITE, "weight", il), {n_s, n_p}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", il), {n_embd}, 0); + + if (hparams.is_recr(il)) { + layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", il), {n_embd, d_in_proj}, 0); + layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), {d_conv, d_inner + 2*n_group*d_state}, 0); + layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", il), {d_inner + 2*n_group*d_state}, 0); + layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), {n_ssm_head}, 0); + layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), {d_inner / n_group, n_group}, 0); + layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), {d_inner, n_embd}, 0); + + // no "weight" suffix for these + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, il), {1, n_ssm_head}, 0); + layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, il), {1, n_ssm_head}, 0); + } else { + const int64_t n_qkv = n_embd_head_k * (hparams.n_head(il) + 2*hparams.n_head_kv(il)); + const int64_t rank = hparams.metis_pass_lora_rank; + + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), {n_embd, n_qkv}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), {n_embd_head_k * hparams.n_head(il), n_embd}, 0); + layer.attn_pass_lora_down = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_DOWN, "weight", il), {n_embd, rank}, 0); + layer.attn_pass_lora_up = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_UP, "weight", il), {rank, n_qkv}, 0); + layer.attn_pass_gate = create_tensor(tn(LLM_TENSOR_ATTN_PASS_GATE, "weight", il), {n_p}, 0); + } + + const int64_t n_ff_exp = hparams.n_ff_exp(il); + + layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", il), {n_embd, n_latent}, 0); + layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", il), {n_latent, n_embd}, 0); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), {n_latent + n_r, n_expert}, 0); + layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", il), {n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), {n_expert}, 0); + layer.ffn_exp_embd = create_tensor(tn(LLM_TENSOR_FFN_EXP_EMBD, "weight", il), {n_r, n_expert}, 0); + layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", il), {n_latent, 2*n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), {n_ff_exp, n_latent, n_expert}, 0); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), {n_ff_shexp, n_latent}, 0); + } + + // the virtual layers of later passes share the physical layer's weights + for (int il = n_layer_phys; il < n_layer; ++il) { + layers[il] = layers[il % n_layer_phys]; + } +} + +std::unique_ptr llama_model_metis16::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +// streams are kept stream-innermost: [n_stream, n_embd, n_tokens] +ggml_tensor * llama_model_metis16::graph::stream_mean(ggml_tensor * x) const { + ggml_tensor * mean = ggml_mean(ctx0, ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1]*x->ne[2])); + return ggml_reshape_2d(ctx0, mean, x->ne[1], x->ne[2]); +} + +// returns the normalized sublayer input +ggml_tensor * llama_model_metis16::graph::hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, + int p, ggml_tensor ** mixed, int il) const { + const int64_t n_s = x->ne[0]; + ggml_tensor * x2 = ggml_reshape_2d(ctx0, x, n_s, x->ne[1]*x->ne[2]); + + ggml_tensor * m = ggml_view_2d(ctx0, mix, n_s, n_s, mix->nb[1], p*mix->nb[2]); + *mixed = ggml_mul_mat(ctx0, m, x2); + + ggml_tensor * r = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, read, n_s, p*read->nb[1]), n_s, 1); + ggml_tensor * src = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, r, x2), x->ne[1], x->ne[2]); + return build_norm(src, norm, NULL, LLM_NORM_RMS, il); +} + +ggml_tensor * llama_model_metis16::graph::hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const { + const int64_t n_s = mixed->ne[0]; + ggml_tensor * w = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, write, n_s, p*write->nb[1]), 1, n_s); + ggml_tensor * u = ggml_reshape_2d(ctx0, update, 1, update->ne[0]*update->ne[1]); + ggml_tensor * out = ggml_add(ctx0, mixed, ggml_mul_mat(ctx0, w, u)); + return ggml_reshape_3d(ctx0, out, n_s, update->ne[0], update->ne[1]); +} + +// blends the retrieved entries into the streams; returns their summary +ggml_tensor * llama_model_metis16::graph::mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const { + const int64_t n_s = (*x)->ne[0]; + const int64_t n_m = model.n_mem; + + ggml_tensor * x_e = ggml_cont(ctx0, ggml_permute(ctx0, *x, 1, 0, 2, 3)); + + ggml_tensor * q = ggml_mul_mat(ctx0, model.depth_mem_query, ggml_reshape_2d(ctx0, x_e, n_embd, n_s*n_tokens)); + q = ggml_reshape_3d(ctx0, ggml_add(ctx0, q, model.depth_mem_query_b), n_m, n_s, n_tokens); + + ggml_tensor * kq = ggml_mul_mat(ctx0, bank_k, q); + kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(n_m)), 0.0f); + + ggml_tensor * ret = ggml_mul_mat(ctx0, bank_vt, kq); + ggml_tensor * proj = ggml_mul_mat(ctx0, model.depth_mem_out, ggml_reshape_2d(ctx0, ret, n_m, n_s*n_tokens)); + proj = ggml_reshape_3d(ctx0, ggml_add(ctx0, proj, model.depth_mem_out_b), n_embd, n_s, n_tokens); + + ggml_tensor * gate = ggml_sum_rows(ctx0, ggml_mul(ctx0, x_e, model.depth_mem_stream_gate)); + gate = ggml_sigmoid(ctx0, ggml_add(ctx0, gate, ggml_reshape_2d(ctx0, model.depth_mem_stream_gate_b, 1, n_s))); + + ggml_tensor * proj_s = ggml_cont(ctx0, ggml_permute(ctx0, proj, 1, 0, 2, 3)); + *x = ggml_add(ctx0, *x, ggml_mul(ctx0, proj_s, ggml_reshape_3d(ctx0, gate, n_s, 1, n_tokens))); + + return stream_mean(proj_s); +} + +void llama_model_metis16::graph::mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary, + ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry) { + const int64_t n_m = model.n_mem; + + if (cont == nullptr) { + cont = ggml_scale(ctx0, confidence, 0.0f); + } + ggml_tensor * meta = ggml_concat(ctx0, ggml_concat(ctx0, summary, entropy, 0), ggml_concat(ctx0, confidence, cont, 0), 0); + + ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_state_write, stream_mean(x)), ggml_mul_mat(ctx0, model.depth_mem_meta_write, meta)); + e = ggml_add(ctx0, e, ggml_view_1d(ctx0, model.depth_mem_entry_bias, n_m, entry*model.depth_mem_entry_bias->nb[1])); + + ggml_tensor * k = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_key, e), model.depth_mem_key_b); + ggml_tensor * v = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_value, e), model.depth_mem_value_b); + k = ggml_reshape_3d(ctx0, k, n_m, 1, n_tokens); + v = ggml_reshape_3d(ctx0, v, 1, n_m, n_tokens); + bank_k = bank_k ? ggml_concat(ctx0, bank_k, k, 1) : k; + bank_vt = bank_vt ? ggml_concat(ctx0, bank_vt, v, 0) : v; +} + +ggml_tensor * llama_model_metis16::graph::route_features(const llama_model_metis16 & model, ggml_tensor * state, + ggml_tensor * summary, ggml_tensor * diff, ggml_tensor * history) const { + ggml_tensor * f = ggml_mul_mat(ctx0, model.depth_mem_route_state, state); + if (summary) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_retr, summary)); + } + if (diff) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_diff, diff)); + } + if (history) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_hist, history)); + } + return ggml_tanh(ctx0, ggml_add(ctx0, f, model.depth_mem_route_b)); +} + +llama_model_metis16::graph::graph(const llama_model_metis16 & model, const llm_graph_params & params) : + llm_build_mamba_base(params) { + const int64_t n_s = model.n_stream; + const int n_p = model.n_pass; + const int n_phys = model.n_layer_phys; + const int64_t n_r = model.n_route; + const int64_t n_hd = hparams.n_embd_head_k(); + const int64_t n_exp = hparams.n_expert; + const int64_t n_used = hparams.n_expert_used(); + + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + auto * inp = build_inp_mem_hybrid(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + ggml_tensor * x = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, inpL, 1, n_embd, n_tokens), n_s, n_embd, n_tokens, 1); + x = ggml_add(ctx0, x, model.stream_embd); + cb(x, "streams_in", -1); + + ggml_tensor * prev_state = stream_mean(x); + ggml_tensor * history = nullptr; // routing history, [n_route, n_tokens] + ggml_tensor * cont = nullptr; // depth head reading of the last pass, [1, n_tokens] + int n_entry = 0; + + for (int p = 0; p < n_p; ++p) { + ggml_tensor * summ = nullptr; + ggml_tensor * ent = nullptr; + ggml_tensor * conf = nullptr; + + for (int il_p = 0; il_p < n_phys; ++il_p) { + const int il = p*n_phys + il_p; + const auto & layer = model.layers[il]; + const bool is_attn = !hparams.is_recr(il); + const bool writes = is_attn || il_p == n_phys - 1; + + ggml_tensor * retrieved = bank_k ? mem_read(model, &x) : nullptr; + ggml_tensor * feats = route_features(model, stream_mean(x), retrieved, nullptr, history); + + ggml_tensor * mixed = nullptr; + ggml_tensor * cur = hc_read(x, layer.hc_attn_mix, layer.hc_attn_read, layer.attn_norm, p, &mixed, il); + if (is_attn) { + ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur); + ggml_tensor * lora = ggml_mul_mat(ctx0, layer.attn_pass_lora_up, ggml_mul_mat(ctx0, layer.attn_pass_lora_down, cur)); + qkv = ggml_add(ctx0, qkv, ggml_mul(ctx0, lora, ggml_view_1d(ctx0, layer.attn_pass_gate, 1, p*layer.attn_pass_gate->nb[0]))); + + const int64_t n_head = hparams.n_head(il); + const int64_t n_head_kv = hparams.n_head_kv(il); + const size_t es = ggml_element_size(qkv); + ggml_tensor * Q = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head, n_tokens, n_hd*es, qkv->nb[1], 0)); + ggml_tensor * K = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*n_head*es)); + ggml_tensor * V = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*(n_head + n_head_kv)*es)); + + cur = build_attn(inp->get_attn(), layer.wo, nullptr, nullptr, Q, K, V, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_hd)), il); + } else { + cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il); + } + cb(cur, "mixer_out", il); + x = hc_write(mixed, layer.hc_attn_write, cur, p); + + cur = hc_read(x, layer.hc_ffn_mix, layer.hc_ffn_read, layer.ffn_norm, p, &mixed, il); + { + ggml_tensor * latent = ggml_mul_mat(ctx0, layer.ffn_latent_down, cur); + ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, ggml_concat(ctx0, latent, feats, 0)); + logits = ggml_add(ctx0, logits, layer.ffn_gate_inp_b); + + // the balance bias picks the experts; the weights come from the unbiased logits + ggml_tensor * selected = ggml_argsort_top_k(ctx0, ggml_add(ctx0, logits, layer.ffn_exp_probs_b), n_used); + + ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr, + n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p, + logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected); + ggml_tensor * shexp = build_ffn(latent, + layer.ffn_up_shexp, NULL, NULL, + layer.ffn_gate_shexp, NULL, NULL, + layer.ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il_p); + cur = ggml_mul_mat(ctx0, layer.ffn_latent_up, ggml_add(ctx0, moe_out, shexp)); + cb(cur, "ffn_out", il); + + // routing summary: the chosen experts' identity vectors blended by their weights + ggml_tensor * logits3 = ggml_reshape_3d(ctx0, logits, 1, n_exp, n_tokens); + ggml_tensor * weights = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, logits3, selected), n_used, n_tokens)); + ggml_tensor * rows = ggml_get_rows(ctx0, layer.ffn_exp_embd, ggml_reshape_1d(ctx0, ggml_cont(ctx0, selected), n_used*n_tokens)); + rows = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, rows, n_r, n_used, n_tokens), 1, 0, 2, 3)); + summ = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, rows, ggml_reshape_3d(ctx0, weights, n_used, 1, n_tokens)), n_r, n_tokens); + + if (writes) { + ggml_tensor * probs = ggml_soft_max(ctx0, logits); + ent = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, ggml_log(ctx0, ggml_clamp(ctx0, probs, 1.0e-9f, 1.0f)))); + ent = ggml_scale(ctx0, ent, -1.0f); + conf = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, probs, 1, n_exp, n_tokens), + ggml_reshape_2d(ctx0, ggml_argmax(ctx0, probs), 1, n_tokens)); + conf = ggml_reshape_2d(ctx0, conf, 1, n_tokens); + } + } + x = hc_write(mixed, layer.hc_ffn_write, cur, p); + cb(x, "streams", il); + + history = history ? ggml_add(ctx0, ggml_scale(ctx0, history, 0.5f), ggml_scale(ctx0, summ, 0.5f)) : ggml_scale(ctx0, summ, 0.5f); + + if (is_attn) { + mem_write(model, x, summ, ent, conf, cont, n_entry++); + } + } + + // end of the pass: one more memory read, the depth head, and an entry recording where the pass left the token + ggml_tensor * retrieved = mem_read(model, &x); + ggml_tensor * state = stream_mean(x); + ggml_tensor * diff = ggml_sub(ctx0, state, prev_state); + ggml_tensor * feats = route_features(model, state, retrieved, diff, history); + + ggml_tensor * h = ggml_mul_mat(ctx0, model.cont_hidden_state, state); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_retr, retrieved)); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_diff, diff)); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_route, feats)); + h = ggml_silu(ctx0, ggml_add(ctx0, h, model.cont_hidden_b)); + + ggml_tensor * logit = ggml_add(ctx0, ggml_mul_mat(ctx0, model.cont_output, h), model.cont_output_b); + ggml_tensor * scale = ggml_clamp(ctx0, ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, state))), 1.0e-6f, INFINITY); + ggml_tensor * innov = ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, diff))); + logit = ggml_add(ctx0, logit, ggml_scale_bias(ctx0, ggml_tanh(ctx0, ggml_div(ctx0, innov, scale)), 1.0f, -0.5f)); + cont = ggml_sigmoid(ctx0, ggml_clamp(ctx0, logit, -model.cont_limit, model.cont_limit)); + cb(cont, "continuation", p); + + mem_write(model, x, summ, ent, conf, cont, n_entry++); + prev_state = state; + } + + ggml_tensor * cur = stream_mean(x); + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + +} diff --git a/src/models/models.h b/src/models/models.h index 1fc7cae1d..dd3b7b1ed 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1799,6 +1799,39 @@ struct llama_model_nemotron_h_moe : public llama_model_nemotron_h { }; +struct llama_model_metis16 : public llama_model_base { + llama_model_metis16(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + uint32_t n_pass = 1; + uint32_t n_stream = 1; + uint32_t n_mem = 0; + uint32_t n_route = 0; + uint32_t n_layer_phys = 0; + float cont_limit = 0.0f; + + struct graph : public llm_build_mamba_base { + graph(const llama_model_metis16 & model, const llm_graph_params & params); + + // memory bank of the current forward: keys [n_mem, n_entry, n_tokens], values [n_entry, n_mem, n_tokens] + ggml_tensor * bank_k = nullptr; + ggml_tensor * bank_vt = nullptr; + + ggml_tensor * stream_mean(ggml_tensor * x) const; + ggml_tensor * hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, int p, ggml_tensor ** mixed, int il) const; + ggml_tensor * hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const; + ggml_tensor * mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const; + void mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary, + ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry); + ggml_tensor * route_features(const llama_model_metis16 & model, ggml_tensor * state, ggml_tensor * summary, + ggml_tensor * diff, ggml_tensor * history) const; + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_exaone : public llama_model_base { llama_model_exaone(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index 254b8a03a..eda5119b5 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -188,6 +188,8 @@ maple ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, mellum ,0 ,any ,MUL+ADD , 4 mellum ,0 ,any ,RMS_NORM+MUL , 9 mellum ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +metis16 ,0 ,any ,MUL+ADD , 8 +metis16 ,0 ,any ,RMS_NORM+MUL , 11 mimo2 ,0 ,any ,MUL+ADD , 4 mimo2 ,0 ,any ,RMS_NORM+MUL , 5 minicpm ,0 ,any ,RMS_NORM+MUL , 5 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 32569dc86..fe949f2c2 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -295,6 +295,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_ff = 192; } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { n_layer = 3; + } else if (arch == LLM_ARCH_METIS16) { + n_layer = 4; // 2 passes over 2 physical layers, one attention and one Mamba-2 } else if (arch == LLM_ARCH_CHAMELEON) { n_vocab = 10240; } else if (arch == LLM_ARCH_QWEN3TTS) { @@ -369,6 +371,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer); } ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer); + } else if (arch == LLM_ARCH_METIS16) { + // the layer types repeat in every pass + std::vector n_head_kv_per_layer; + n_head_kv_per_layer.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + n_head_kv_per_layer.push_back(il % 2 == 0 ? n_head_kv : 0); + } + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_kv_per_layer); } else { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv); @@ -616,6 +627,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f); } + if (arch == LLM_ARCH_METIS16) { + ms.add_kv(LLM_KV_METIS_PASS_COUNT, uint32_t(2)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); + ms.add_kv(LLM_KV_METIS_MEMORY_LENGTH, uint32_t(64)); + ms.add_kv(LLM_KV_METIS_ROUTE_LENGTH, uint32_t(32)); + ms.add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, uint32_t(8)); + ms.add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, 4.0f); + ms.add_kv(LLM_KV_SSM_NORM_EPS, 1e-5f); + ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd/2); + } + if (arch == LLM_ARCH_LIMITE) { ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f); ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f); @@ -972,6 +994,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MAPLE: case LLM_ARCH_MUSE_SPARK: case LLM_ARCH_KOLIBRI1: + case LLM_ARCH_METIS16: return true; default: return false;