1 parent
d1044bc827
commit
9309bf1bcf
20 files changed
+1112
-2
No files matched your search
@@ -197,6 +197,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"MambaLMHeadModel": "mamba",
|
||||
"MapleForCausalLM": "maple",
|
||||
"MellumForCausalLM": "mellum",
|
||||
"Metis16ForCausalLM": "metis",
|
||||
"MiMoV2FlashForCausalLM": "mimo",
|
||||
"MiMoV2ForCausalLM": "mimo",
|
||||
"MiniCPM3ForCausalLM": "minicpm",
|
||||
|
||||
@@ -2116,6 +2116,9 @@ class TextModel(ModelBase):
|
||||
if chkhsh == "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4":
|
||||
# ref: https://huggingface.co/evilfreelancer/ruGPT3XL
|
||||
res = "gpt-2"
|
||||
if chkhsh == "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9":
|
||||
# ref: https://huggingface.co/Lernex/Metis-1.6-Logos
|
||||
res = "gpt-2"
|
||||
if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":
|
||||
# ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B
|
||||
res = "lfm2"
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import LazyTorchTensor, ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("Metis16ForCausalLM")
|
||||
class Metis16Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.METIS16
|
||||
|
||||
# read by generate_extra_tensors() or not used at inference: dropped in modify_tensors()
|
||||
_folded = {
|
||||
"pass_embeddings", "lm_head.weight", "depth_budget.multiplier",
|
||||
"depth_memory.pass_embeddings", "depth_memory.anchor_embeddings",
|
||||
"depth_memory.state_write.bias", "depth_memory.metadata_write.bias",
|
||||
}
|
||||
_folded_layer = {
|
||||
"mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits",
|
||||
"mixer_connection.read_logits", "mixer_connection.write_logits",
|
||||
"moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits",
|
||||
"moe_connection.read_logits", "moe_connection.write_logits",
|
||||
"moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier",
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.hparams
|
||||
self.n_pass = hp["num_passes"]
|
||||
self.attn_layers = list(hp["attention_layer_indices"])
|
||||
self._generated: set[str] = set()
|
||||
# one cache slot per pass and layer, while the tensors exist once per physical layer
|
||||
self.block_count = self.n_pass * hp["num_hidden_layers"]
|
||||
|
||||
def set_vocab(self):
|
||||
self._set_vocab_gpt2()
|
||||
# turns end on the first eos id, so llama.cpp must stop on it too
|
||||
eos = self.hparams.get("eos_token_id")
|
||||
if isinstance(eos, list) and eos:
|
||||
self.gguf_writer.add_eot_token_id(eos[0])
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
super().set_gguf_parameters()
|
||||
|
||||
n_layer = hp["num_hidden_layers"]
|
||||
self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)])
|
||||
|
||||
self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"])
|
||||
self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"])
|
||||
self.gguf_writer.add_expert_shared_count(1)
|
||||
self.gguf_writer.add_moe_latent_size(hp["latent_dim"])
|
||||
|
||||
d_inner = hp["mamba_expand"] * hp["hidden_size"]
|
||||
self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"])
|
||||
self.gguf_writer.add_ssm_inner_size(d_inner)
|
||||
self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"])
|
||||
self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"])
|
||||
self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"])
|
||||
self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"])
|
||||
|
||||
self.gguf_writer.add_hyper_connection_count(hp["n_streams"])
|
||||
self.gguf_writer.add_metis_pass_count(self.n_pass)
|
||||
self.gguf_writer.add_metis_memory_length(hp["memory_dim"])
|
||||
self.gguf_writer.add_metis_route_length(hp["route_feature_dim"])
|
||||
self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"])
|
||||
self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"])
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
# the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it
|
||||
if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")):
|
||||
name += ".bias"
|
||||
elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates"))
|
||||
or name in ("depth_memory.stream_gate", "stream_embeddings")):
|
||||
name += ".weight"
|
||||
elif name == "depth_memory.stream_gate_bias":
|
||||
name = "depth_memory.stream_gate.bias"
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def _load(self, name: str) -> Tensor:
|
||||
return LazyTorchTensor.to_eager(self.model_tensors[name]())
|
||||
|
||||
def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]:
|
||||
self._generated.add(name)
|
||||
return name, data
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
hp = self.hparams
|
||||
r = hp["route_feature_dim"]
|
||||
n_stream = hp["n_streams"]
|
||||
T = gguf.MODEL_TENSOR
|
||||
passes = self._load("pass_embeddings")[: self.n_pass]
|
||||
|
||||
# the mixes depend only on the weights and the pass, so Sinkhorn runs once here
|
||||
# the controller runs in the checkpoint dtype, as the reference does
|
||||
for il in range(hp["num_hidden_layers"]):
|
||||
for src, (mix, read, write) in (
|
||||
("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)),
|
||||
("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)),
|
||||
):
|
||||
prefix = f"layers.{il}.{src}."
|
||||
ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias")
|
||||
mixes, reads, writes = [], [], []
|
||||
for p in range(self.n_pass):
|
||||
delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float()
|
||||
m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream])
|
||||
log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream)
|
||||
for _ in range(hp["mhc_sinkhorn_iterations"]):
|
||||
log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True)
|
||||
log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True)
|
||||
mixes.append(log_m.exp())
|
||||
reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1))
|
||||
writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1))
|
||||
act = ctrl_w.dtype
|
||||
yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float())
|
||||
yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float())
|
||||
yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float())
|
||||
|
||||
# one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width
|
||||
meta_w = self._load("depth_memory.metadata_write.weight").float()
|
||||
mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"]
|
||||
base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float()
|
||||
+ mean_k * meta_w[:, r])
|
||||
pass_e = self._load("depth_memory.pass_embeddings").float()
|
||||
anchor_e = self._load("depth_memory.anchor_embeddings").float()
|
||||
n_anchor = len(self.attn_layers) + 1
|
||||
entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)])
|
||||
yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias)
|
||||
|
||||
def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the graph reads the concatenated features one block at a time; each block is its own tensor
|
||||
for i, part in enumerate(parts):
|
||||
piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:]
|
||||
yield from super().modify_tensors(piece, template.format(part), bid)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name in self._generated:
|
||||
yield name, data_torch
|
||||
return
|
||||
if name in self._folded:
|
||||
return
|
||||
hp = self.hparams
|
||||
d = hp["hidden_size"]
|
||||
|
||||
if bid is None:
|
||||
if name == "stream_embeddings.weight":
|
||||
# the graph keeps the streams innermost: [n_stream, n_embd, n_tokens]
|
||||
yield from super().modify_tensors(data_torch.T.contiguous(), name, bid)
|
||||
return
|
||||
if name == "depth_memory.metadata_write.weight":
|
||||
# metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized
|
||||
r = hp["route_feature_dim"]
|
||||
meta_w = data_torch.float()
|
||||
entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"])
|
||||
data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1)
|
||||
elif name == "depth_memory.stream_gate.weight":
|
||||
data_torch = data_torch.float() / math.sqrt(d)
|
||||
elif name == "depth_memory.route_projection.weight":
|
||||
yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid)
|
||||
return
|
||||
elif name == "continuation.hidden.weight":
|
||||
yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid)
|
||||
return
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
return
|
||||
|
||||
local = name.removeprefix(f"layers.{bid}.")
|
||||
if local in self._folded_layer:
|
||||
return
|
||||
|
||||
if local == "mixer.impl.mixer.A_log":
|
||||
data_torch = -torch.exp(data_torch).unsqueeze(-1)
|
||||
elif local == "mixer.impl.mixer.D":
|
||||
data_torch = data_torch.unsqueeze(-1)
|
||||
elif local == "mixer.impl.mixer.conv1d.weight":
|
||||
data_torch = data_torch.squeeze(1)
|
||||
elif local == "mixer.impl.mixer.norm.weight":
|
||||
data_torch = data_torch.reshape(hp["mamba_n_groups"], -1)
|
||||
elif local == "mixer.pass_gates.weight":
|
||||
# the reference applies tanh(gate) in the checkpoint dtype
|
||||
data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float()
|
||||
elif local == "moe.shared_expert.gate_up.weight":
|
||||
gate, up = data_torch.chunk(2, dim=0)
|
||||
yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid)
|
||||
yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid)
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
||||
# FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32
|
||||
if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate"))
|
||||
or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight",
|
||||
"hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
@@ -198,6 +198,7 @@ pre_computed_hashes = [
|
||||
# jina-v2-de variants
|
||||
{"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
|
||||
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
|
||||
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Lernex/Metis-1.6-Logos", "chkhsh": "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9"},
|
||||
# lfm2 variants
|
||||
{"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},
|
||||
# hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges
|
||||
|
||||
@@ -203,6 +203,7 @@ class Keys:
|
||||
ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank"
|
||||
VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank"
|
||||
GATE_LORA_RANK = "{arch}.attention.gate_lora_rank"
|
||||
PASS_LORA_RANK = "{arch}.attention.pass_lora_rank"
|
||||
REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count"
|
||||
SLIDING_WINDOW = "{arch}.attention.sliding_window"
|
||||
SCALE = "{arch}.attention.scale"
|
||||
@@ -315,6 +316,12 @@ class Keys:
|
||||
L_CYCLES = "{arch}.hrm.l_cycles"
|
||||
PREFIX_LM = "{arch}.hrm.prefix_lm"
|
||||
|
||||
class Metis:
|
||||
PASS_COUNT = "{arch}.metis.pass_count"
|
||||
MEMORY_LENGTH = "{arch}.metis.memory_length"
|
||||
ROUTE_LENGTH = "{arch}.metis.route_length"
|
||||
CONTINUATION_LOGIT_LIMIT = "{arch}.metis.continuation_logit_limit"
|
||||
|
||||
class SSM:
|
||||
CONV_KERNEL = "{arch}.ssm.conv_kernel"
|
||||
INNER_SIZE = "{arch}.ssm.inner_size"
|
||||
@@ -322,6 +329,7 @@ class Keys:
|
||||
TIME_STEP_RANK = "{arch}.ssm.time_step_rank"
|
||||
GROUP_COUNT = "{arch}.ssm.group_count"
|
||||
DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms"
|
||||
NORM_EPSILON = "{arch}.ssm.norm_epsilon"
|
||||
|
||||
class KDA:
|
||||
HEAD_DIM = "{arch}.kda.head_dim"
|
||||
@@ -681,6 +689,7 @@ class MODEL_ARCH(IntEnum):
|
||||
NEMOTRON = auto()
|
||||
NEMOTRON_H = auto()
|
||||
NEMOTRON_H_MOE = auto()
|
||||
METIS16 = auto()
|
||||
EXAONE = auto()
|
||||
EXAONE4 = auto()
|
||||
EXAONE_MOE = auto()
|
||||
@@ -830,6 +839,36 @@ class MODEL_TENSOR(IntEnum):
|
||||
FFN_GATE_TID2EID = auto()
|
||||
MOE_LATENT_DOWN = auto() # nemotron 3 super
|
||||
MOE_LATENT_UP = auto() # nemotron 3 super
|
||||
STREAM_EMBD = auto()
|
||||
DEPTH_MEM_STATE_WRITE = auto()
|
||||
DEPTH_MEM_META_WRITE = auto()
|
||||
DEPTH_MEM_ENTRY_BIAS = auto()
|
||||
DEPTH_MEM_QUERY = auto()
|
||||
DEPTH_MEM_KEY = auto()
|
||||
DEPTH_MEM_VALUE = auto()
|
||||
DEPTH_MEM_OUT = auto()
|
||||
DEPTH_MEM_ROUTE_STATE = auto()
|
||||
DEPTH_MEM_ROUTE_RETR = auto()
|
||||
DEPTH_MEM_ROUTE_DIFF = auto()
|
||||
DEPTH_MEM_ROUTE_HIST = auto()
|
||||
DEPTH_MEM_ROUTE = auto()
|
||||
DEPTH_MEM_STREAM_GATE = auto()
|
||||
CONT_HIDDEN_STATE = auto()
|
||||
CONT_HIDDEN_RETR = auto()
|
||||
CONT_HIDDEN_DIFF = auto()
|
||||
CONT_HIDDEN_ROUTE = auto()
|
||||
CONT_HIDDEN = auto()
|
||||
CONT_OUTPUT = auto()
|
||||
HC_ATTN_MIX = auto()
|
||||
HC_ATTN_READ = auto()
|
||||
HC_ATTN_WRITE = auto()
|
||||
HC_FFN_MIX = auto()
|
||||
HC_FFN_READ = auto()
|
||||
HC_FFN_WRITE = auto()
|
||||
ATTN_PASS_LORA_DOWN = auto()
|
||||
ATTN_PASS_LORA_UP = auto()
|
||||
ATTN_PASS_GATE = auto()
|
||||
FFN_EXP_EMBD = auto()
|
||||
FFN_RESID_ALPHA = auto()
|
||||
FFN_RESID_BETA = auto()
|
||||
FFN_PRE_EXP_NORM = auto()
|
||||
@@ -1558,6 +1597,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
|
||||
MODEL_ARCH.NEMOTRON: "nemotron",
|
||||
MODEL_ARCH.NEMOTRON_H: "nemotron_h",
|
||||
MODEL_ARCH.NEMOTRON_H_MOE: "nemotron_h_moe",
|
||||
MODEL_ARCH.METIS16: "metis16",
|
||||
MODEL_ARCH.EXAONE: "exaone",
|
||||
MODEL_ARCH.EXAONE4: "exaone4",
|
||||
MODEL_ARCH.EXAONE_MOE: "exaone-moe",
|
||||
@@ -1708,6 +1748,36 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid",
|
||||
MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super
|
||||
MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super
|
||||
MODEL_TENSOR.STREAM_EMBD: "stream_embd",
|
||||
MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: "depth_mem_state_write",
|
||||
MODEL_TENSOR.DEPTH_MEM_META_WRITE: "depth_mem_meta_write",
|
||||
MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS: "depth_mem_entry_bias",
|
||||
MODEL_TENSOR.DEPTH_MEM_QUERY: "depth_mem_query",
|
||||
MODEL_TENSOR.DEPTH_MEM_KEY: "depth_mem_key",
|
||||
MODEL_TENSOR.DEPTH_MEM_VALUE: "depth_mem_value",
|
||||
MODEL_TENSOR.DEPTH_MEM_OUT: "depth_mem_out",
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: "depth_mem_route_state",
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: "depth_mem_route_retr",
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: "depth_mem_route_diff",
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: "depth_mem_route_hist",
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE: "depth_mem_route",
|
||||
MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: "depth_mem_stream_gate",
|
||||
MODEL_TENSOR.CONT_HIDDEN_STATE: "cont_hidden_state",
|
||||
MODEL_TENSOR.CONT_HIDDEN_RETR: "cont_hidden_retr",
|
||||
MODEL_TENSOR.CONT_HIDDEN_DIFF: "cont_hidden_diff",
|
||||
MODEL_TENSOR.CONT_HIDDEN_ROUTE: "cont_hidden_route",
|
||||
MODEL_TENSOR.CONT_HIDDEN: "cont_hidden",
|
||||
MODEL_TENSOR.CONT_OUTPUT: "cont_output",
|
||||
MODEL_TENSOR.HC_ATTN_MIX: "blk.{bid}.hc_attn_mix",
|
||||
MODEL_TENSOR.HC_ATTN_READ: "blk.{bid}.hc_attn_read",
|
||||
MODEL_TENSOR.HC_ATTN_WRITE: "blk.{bid}.hc_attn_write",
|
||||
MODEL_TENSOR.HC_FFN_MIX: "blk.{bid}.hc_ffn_mix",
|
||||
MODEL_TENSOR.HC_FFN_READ: "blk.{bid}.hc_ffn_read",
|
||||
MODEL_TENSOR.HC_FFN_WRITE: "blk.{bid}.hc_ffn_write",
|
||||
MODEL_TENSOR.ATTN_PASS_LORA_DOWN: "blk.{bid}.attn_pass_lora_down",
|
||||
MODEL_TENSOR.ATTN_PASS_LORA_UP: "blk.{bid}.attn_pass_lora_up",
|
||||
MODEL_TENSOR.ATTN_PASS_GATE: "blk.{bid}.attn_pass_gate",
|
||||
MODEL_TENSOR.FFN_EXP_EMBD: "blk.{bid}.ffn_exp_embd",
|
||||
MODEL_TENSOR.FFN_RESID_ALPHA: "blk.{bid}.ffn_resid_alpha",
|
||||
MODEL_TENSOR.FFN_RESID_BETA: "blk.{bid}.ffn_resid_beta",
|
||||
MODEL_TENSOR.FFN_PRE_EXP_NORM: "blk.{bid}.ffn_pre_exp_norm",
|
||||
@@ -4917,6 +4987,61 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
],
|
||||
MODEL_ARCH.METIS16: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
MODEL_TENSOR.OUTPUT,
|
||||
MODEL_TENSOR.ATTN_NORM,
|
||||
MODEL_TENSOR.FFN_NORM,
|
||||
MODEL_TENSOR.STREAM_EMBD,
|
||||
MODEL_TENSOR.DEPTH_MEM_STATE_WRITE,
|
||||
MODEL_TENSOR.DEPTH_MEM_META_WRITE,
|
||||
MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS,
|
||||
MODEL_TENSOR.DEPTH_MEM_QUERY,
|
||||
MODEL_TENSOR.DEPTH_MEM_KEY,
|
||||
MODEL_TENSOR.DEPTH_MEM_VALUE,
|
||||
MODEL_TENSOR.DEPTH_MEM_OUT,
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE,
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR,
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF,
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST,
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE,
|
||||
MODEL_TENSOR.DEPTH_MEM_STREAM_GATE,
|
||||
MODEL_TENSOR.CONT_HIDDEN_STATE,
|
||||
MODEL_TENSOR.CONT_HIDDEN_RETR,
|
||||
MODEL_TENSOR.CONT_HIDDEN_DIFF,
|
||||
MODEL_TENSOR.CONT_HIDDEN_ROUTE,
|
||||
MODEL_TENSOR.CONT_HIDDEN,
|
||||
MODEL_TENSOR.CONT_OUTPUT,
|
||||
MODEL_TENSOR.HC_ATTN_MIX,
|
||||
MODEL_TENSOR.HC_ATTN_READ,
|
||||
MODEL_TENSOR.HC_ATTN_WRITE,
|
||||
MODEL_TENSOR.HC_FFN_MIX,
|
||||
MODEL_TENSOR.HC_FFN_READ,
|
||||
MODEL_TENSOR.HC_FFN_WRITE,
|
||||
MODEL_TENSOR.ATTN_PASS_LORA_DOWN,
|
||||
MODEL_TENSOR.ATTN_PASS_LORA_UP,
|
||||
MODEL_TENSOR.ATTN_PASS_GATE,
|
||||
MODEL_TENSOR.FFN_EXP_EMBD,
|
||||
MODEL_TENSOR.SSM_IN,
|
||||
MODEL_TENSOR.SSM_CONV1D,
|
||||
MODEL_TENSOR.SSM_DT,
|
||||
MODEL_TENSOR.SSM_A,
|
||||
MODEL_TENSOR.SSM_D,
|
||||
MODEL_TENSOR.SSM_NORM,
|
||||
MODEL_TENSOR.SSM_OUT,
|
||||
MODEL_TENSOR.ATTN_QKV,
|
||||
MODEL_TENSOR.ATTN_OUT,
|
||||
MODEL_TENSOR.MOE_LATENT_DOWN,
|
||||
MODEL_TENSOR.MOE_LATENT_UP,
|
||||
MODEL_TENSOR.FFN_GATE_INP,
|
||||
MODEL_TENSOR.FFN_EXP_PROBS_B,
|
||||
MODEL_TENSOR.FFN_GATE_UP_EXP,
|
||||
MODEL_TENSOR.FFN_DOWN_EXP,
|
||||
MODEL_TENSOR.FFN_GATE_SHEXP,
|
||||
MODEL_TENSOR.FFN_UP_SHEXP,
|
||||
MODEL_TENSOR.FFN_DOWN_SHEXP,
|
||||
],
|
||||
MODEL_ARCH.NEMOTRON_H: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
|
||||
@@ -975,6 +975,18 @@ class GGUFWriter:
|
||||
def add_hrm_prefix_lm(self, value: bool) -> None:
|
||||
self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value)
|
||||
|
||||
def add_metis_pass_count(self, value: int) -> None:
|
||||
self.add_uint32(Keys.Metis.PASS_COUNT.format(arch=self.arch), value)
|
||||
|
||||
def add_metis_memory_length(self, value: int) -> None:
|
||||
self.add_uint32(Keys.Metis.MEMORY_LENGTH.format(arch=self.arch), value)
|
||||
|
||||
def add_metis_route_length(self, value: int) -> None:
|
||||
self.add_uint32(Keys.Metis.ROUTE_LENGTH.format(arch=self.arch), value)
|
||||
|
||||
def add_metis_continuation_logit_limit(self, value: float) -> None:
|
||||
self.add_float32(Keys.Metis.CONTINUATION_LOGIT_LIMIT.format(arch=self.arch), value)
|
||||
|
||||
def add_adapter_count(self, count: int) -> None:
|
||||
self.add_uint32(Keys.Adapters.COUNT.format(arch=self.arch), count)
|
||||
|
||||
@@ -1086,6 +1098,9 @@ class GGUFWriter:
|
||||
def add_attention_output_lora_rank(self, length: int) -> None:
|
||||
self.add_uint32(Keys.Attention.OUTPUT_LORA_RANK.format(arch=self.arch), length)
|
||||
|
||||
def add_attention_pass_lora_rank(self, length: int) -> None:
|
||||
self.add_uint32(Keys.Attention.PASS_LORA_RANK.format(arch=self.arch), length)
|
||||
|
||||
def add_attention_compress_ratios(self, values: Sequence[int]) -> None:
|
||||
self.add_array(Keys.Attention.COMPRESS_RATIOS.format(arch=self.arch), values)
|
||||
|
||||
@@ -1267,6 +1282,9 @@ class GGUFWriter:
|
||||
def add_ssm_dt_b_c_rms(self, value: bool) -> None:
|
||||
self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value)
|
||||
|
||||
def add_ssm_norm_epsilon(self, value: float) -> None:
|
||||
self.add_float32(Keys.SSM.NORM_EPSILON.format(arch=self.arch), value)
|
||||
|
||||
def add_expert_latent_length(self, value: int) -> None:
|
||||
self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value)
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ class TensorNameMap:
|
||||
"model.transformer.wte", # llada
|
||||
"embed_tokens", # qwen3-embedding
|
||||
"model.embed", # talkie
|
||||
"embedding", # metis16
|
||||
),
|
||||
|
||||
# Masked embeddings
|
||||
@@ -224,6 +225,7 @@ class TensorNameMap:
|
||||
block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {
|
||||
# Attention norm
|
||||
MODEL_TENSOR.ATTN_NORM: (
|
||||
"layers.{bid}.mixer_connection.norm", # metis16
|
||||
"model.lilicorr.layers.{bid}.attn_norm", # lilicorr
|
||||
"gpt_neox.layers.{bid}.input_layernorm", # gptneox
|
||||
"transformer.h.{bid}.ln_1", # gpt2 gpt-j refact qwen jais exaone
|
||||
@@ -294,6 +296,7 @@ class TensorNameMap:
|
||||
"model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5
|
||||
"head.layers.{bid}.self_attn.in_proj", # laya
|
||||
"layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 text
|
||||
"layers.{bid}.mixer.qkv", # metis16
|
||||
),
|
||||
|
||||
# Attention query
|
||||
@@ -401,6 +404,7 @@ class TensorNameMap:
|
||||
"model.layers.{bid}.self_attn.language_expert_dense", # cogvlm
|
||||
"model.blocks.{bid}.attn.attn_resid", # talkie
|
||||
"head.layers.{bid}.self_attn.out_proj", # laya
|
||||
"layers.{bid}.mixer.out", # metis16
|
||||
),
|
||||
|
||||
# Attention output norm
|
||||
@@ -449,6 +453,7 @@ class TensorNameMap:
|
||||
|
||||
# Feed-forward norm
|
||||
MODEL_TENSOR.FFN_NORM: (
|
||||
"layers.{bid}.moe_connection.norm", # metis16
|
||||
"model.lilicorr.layers.{bid}.mlp_norm", # lilicorr
|
||||
"model.lilicorr.feature_mlp.{bid}.norm", # lilicorr
|
||||
"gpt_neox.layers.{bid}.post_attention_layernorm", # gptneox
|
||||
@@ -536,6 +541,7 @@ class TensorNameMap:
|
||||
"backbone.layers.{bid}.mixer.gate", # nemotron-h-moe
|
||||
"model.layers.{bid}.moe.gate", # step3.5
|
||||
"model.layers.{bid}.router.proj", # gemma4
|
||||
"layers.{bid}.moe.expert_router", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.FFN_GATE_INP_SHEXP: (
|
||||
@@ -543,6 +549,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.FFN_EXP_PROBS_B: (
|
||||
"layers.{bid}.moe.selection_bias", # metis16
|
||||
"model.layers.{bid}.mlp.gate.e_score_correction", # deepseek-v3 dots1
|
||||
"model.layers.{bid}.mlp.gate.e_score_correction_bias", # muse-spark
|
||||
"model.layers.{bid}.mlp.moe_statics.e_score_correction", # ernie4.5-moe
|
||||
@@ -617,6 +624,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.FFN_UP_SHEXP: (
|
||||
"layers.{bid}.moe.shared_expert.up", # metis16
|
||||
"model.layers.{bid}.mlp.shared_expert.up_proj", # qwen2moe
|
||||
"model.layers.{bid}.mlp.shared_experts.up_proj", # deepseek deepseek2
|
||||
"model.layers.{bid}.feed_forward.shared_expert.up_proj", # llama4
|
||||
@@ -670,6 +678,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.FFN_GATE_SHEXP: (
|
||||
"layers.{bid}.moe.shared_expert.gate", # metis16
|
||||
"model.layers.{bid}.mlp.shared_expert.gate_proj", # qwen2moe
|
||||
"model.layers.{bid}.mlp.shared_experts.gate_proj", # deepseek deepseek2
|
||||
"model.layers.{bid}.feed_forward.shared_expert.gate_proj", # llama4
|
||||
@@ -686,18 +695,94 @@ class TensorNameMap:
|
||||
MODEL_TENSOR.FFN_GATE_UP_EXP: (
|
||||
"model.layers.{bid}.mlp.experts.gate_up_proj",
|
||||
"model.layers.{bid}.experts.gate_up_proj", # gemma4
|
||||
"layers.{bid}.moe.local_experts.gate_up", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.MOE_LATENT_DOWN: (
|
||||
"backbone.layers.{bid}.mixer.fc1_latent_proj", # nemotron 3 super
|
||||
"model.layers.{bid}.mlp.pre_expert_proj", # muse-spark
|
||||
"layers.{bid}.moe.latent_down", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.MOE_LATENT_UP: (
|
||||
"backbone.layers.{bid}.mixer.fc2_latent_proj", # nemotron 3 super
|
||||
"model.layers.{bid}.mlp.post_expert_proj", # muse-spark
|
||||
"layers.{bid}.moe.latent_up", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.STREAM_EMBD: (
|
||||
"stream_embeddings", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: (
|
||||
"depth_memory.state_write", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_META_WRITE: (
|
||||
"depth_memory.metadata_write", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_QUERY: (
|
||||
"depth_memory.query", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_KEY: (
|
||||
"depth_memory.key", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_VALUE: (
|
||||
"depth_memory.value", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_OUT: (
|
||||
"depth_memory.output", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: (
|
||||
"depth_memory.route_state", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: (
|
||||
"depth_memory.route_retr", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: (
|
||||
"depth_memory.route_diff", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: (
|
||||
"depth_memory.route_hist", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_ROUTE: (
|
||||
"depth_memory.route_projection", # metis16
|
||||
),
|
||||
MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: (
|
||||
"depth_memory.stream_gate", # metis16
|
||||
),
|
||||
MODEL_TENSOR.CONT_HIDDEN_STATE: (
|
||||
"continuation.hidden_state", # metis16
|
||||
),
|
||||
MODEL_TENSOR.CONT_HIDDEN_RETR: (
|
||||
"continuation.hidden_retr", # metis16
|
||||
),
|
||||
MODEL_TENSOR.CONT_HIDDEN_DIFF: (
|
||||
"continuation.hidden_diff", # metis16
|
||||
),
|
||||
MODEL_TENSOR.CONT_HIDDEN_ROUTE: (
|
||||
"continuation.hidden_route", # metis16
|
||||
),
|
||||
MODEL_TENSOR.CONT_HIDDEN: (
|
||||
"continuation.hidden", # metis16
|
||||
),
|
||||
MODEL_TENSOR.CONT_OUTPUT: (
|
||||
"continuation.output", # metis16
|
||||
),
|
||||
MODEL_TENSOR.ATTN_PASS_LORA_DOWN: (
|
||||
"layers.{bid}.mixer.pass_lora_down", # metis16
|
||||
),
|
||||
MODEL_TENSOR.ATTN_PASS_LORA_UP: (
|
||||
"layers.{bid}.mixer.pass_lora_up", # metis16
|
||||
),
|
||||
MODEL_TENSOR.ATTN_PASS_GATE: (
|
||||
"layers.{bid}.mixer.pass_gates", # metis16
|
||||
),
|
||||
MODEL_TENSOR.FFN_EXP_EMBD: (
|
||||
"layers.{bid}.moe.expert_embeddings", # metis16
|
||||
),
|
||||
|
||||
|
||||
|
||||
|
||||
MODEL_TENSOR.FFN_PRE_EXP_NORM: (
|
||||
"model.layers.{bid}.mlp.pre_expert_norm", # muse-spark
|
||||
),
|
||||
@@ -759,6 +844,7 @@ class TensorNameMap:
|
||||
"model.layers.{bid}.block_sparse_moe.experts.down", # smallthinker
|
||||
"model.layers.{bid}.moe.down_proj", # step3.5
|
||||
"model.layers.{bid}.experts.down_proj", # gemma4
|
||||
"layers.{bid}.moe.local_experts.down", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.FFN_DOWN_SHEXP: (
|
||||
@@ -771,6 +857,7 @@ class TensorNameMap:
|
||||
"backbone.layers.{bid}.mixer.shared_experts.down_proj", # nemotron-h-moe
|
||||
"model.layers.{bid}.block_sparse_moe.shared_experts.down_proj", # kimi
|
||||
"model.layers.{bid}.share_expert.down_proj", # step3.5
|
||||
"layers.{bid}.moe.shared_expert.down", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.FFN_DOWN_CHEXP: (
|
||||
@@ -906,6 +993,7 @@ class TensorNameMap:
|
||||
"model.layers.{bid}.mamba.in_proj", # jamba falcon-h1 granite-hybrid
|
||||
"model.layers.layers.{bid}.mixer.in_proj", # plamo2
|
||||
"model.layers.{bid}.linear_attn.in_proj_qkvz", # qwen3next
|
||||
"layers.{bid}.mixer.impl.mixer.in_proj", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_CONV1D: (
|
||||
@@ -915,6 +1003,7 @@ class TensorNameMap:
|
||||
"model.layers.layers.{bid}.mixer.conv1d", # plamo2
|
||||
"model.layers.{bid}.linear_attn.conv1d", # qwen3next
|
||||
"layers.{bid}.linear_attn.conv1d", # qwen3.5 text
|
||||
"layers.{bid}.mixer.impl.mixer.conv1d", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_X: (
|
||||
@@ -925,6 +1014,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_DT: (
|
||||
"layers.{bid}.mixer.impl.mixer.dt_bias", # metis16
|
||||
"model.layers.{bid}.dt_proj", # mamba-hf
|
||||
"backbone.layers.{bid}.mixer.dt_proj", # mamba
|
||||
"model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid
|
||||
@@ -942,6 +1032,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_A: (
|
||||
"layers.{bid}.mixer.impl.mixer.A_log", # metis16
|
||||
"model.layers.{bid}.A_log", # mamba-hf
|
||||
"backbone.layers.{bid}.mixer.A_log", # mamba
|
||||
"model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid
|
||||
@@ -965,6 +1056,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_D: (
|
||||
"layers.{bid}.mixer.impl.mixer.D", # metis16
|
||||
"model.layers.{bid}.D", # mamba-hf
|
||||
"backbone.layers.{bid}.mixer.D", # mamba
|
||||
"model.layers.{bid}.mamba.D", # jamba falcon-h1 granite-hybrid
|
||||
@@ -972,6 +1064,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_NORM: (
|
||||
"layers.{bid}.mixer.impl.mixer.norm", # metis16
|
||||
"model.layers.{bid}.mamba.norm", # falcon-h1 granite-hybrid
|
||||
"model.layers.{bid}.linear_attn.norm", # qwen3next
|
||||
"layers.{bid}.linear_attn.norm", # qwen3.5 text
|
||||
@@ -987,6 +1080,7 @@ class TensorNameMap:
|
||||
"model.layers.{bid}.linear_attn.out_proj", # qwen3next
|
||||
"layers.{bid}.linear_attn.out_proj", # qwen3.5 text
|
||||
"model.layers.layers.{bid}.mixer.out_proj", # plamo2
|
||||
"layers.{bid}.mixer.impl.mixer.out_proj", # metis16
|
||||
),
|
||||
|
||||
MODEL_TENSOR.SSM_ALPHA: (
|
||||
|
||||
@@ -102,6 +102,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
||||
{ LLM_ARCH_NEMOTRON, "nemotron" },
|
||||
{ LLM_ARCH_NEMOTRON_H, "nemotron_h" },
|
||||
{ LLM_ARCH_NEMOTRON_H_MOE, "nemotron_h_moe" },
|
||||
{ LLM_ARCH_METIS16, "metis16" },
|
||||
{ LLM_ARCH_EXAONE, "exaone" },
|
||||
{ LLM_ARCH_EXAONE4, "exaone4" },
|
||||
{ LLM_ARCH_EXAONE_MOE, "exaone-moe" },
|
||||
@@ -272,6 +273,10 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" },
|
||||
{ LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" },
|
||||
{ LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" },
|
||||
{ LLM_KV_METIS_PASS_COUNT, "%s.metis.pass_count" },
|
||||
{ LLM_KV_METIS_MEMORY_LENGTH, "%s.metis.memory_length" },
|
||||
{ LLM_KV_METIS_ROUTE_LENGTH, "%s.metis.route_length" },
|
||||
{ LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, "%s.metis.continuation_logit_limit" },
|
||||
{ LLM_KV_MUDD_FEED_FORWARD_LENGTH, "%s.mudd.feed_forward_length" },
|
||||
{ LLM_KV_MUDD_TAP_COUNT, "%s.mudd.tap_count" },
|
||||
{ LLM_KV_MUDD_TAP_INDICES, "%s.mudd.tap_indices" },
|
||||
@@ -329,6 +334,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_ATTENTION_KV_SOURCE_LAYERS, "%s.attention.kv_source_layers" },
|
||||
{ LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" },
|
||||
{ LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" },
|
||||
{ LLM_KV_ATTENTION_PASS_LORA_RANK, "%s.attention.pass_lora_rank" },
|
||||
{ LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" },
|
||||
{ LLM_KV_ATTENTION_COMPRESS_RATIOS, "%s.attention.compress_ratios" },
|
||||
{ LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" },
|
||||
@@ -387,6 +393,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },
|
||||
{ LLM_KV_SSM_GROUP_COUNT, "%s.ssm.group_count" },
|
||||
{ LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" },
|
||||
{ LLM_KV_SSM_NORM_EPS, "%s.ssm.norm_epsilon" },
|
||||
|
||||
{ LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" },
|
||||
{ LLM_KV_KDA_SAFE_GATE, "%s.kda.safe_gate" },
|
||||
@@ -548,6 +555,36 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
|
||||
{ LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" },
|
||||
{ LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" },
|
||||
{ LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" },
|
||||
{ LLM_TENSOR_STREAM_EMBD, "stream_embd" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "depth_mem_state_write" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_META_WRITE, "depth_mem_meta_write" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "depth_mem_entry_bias" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_QUERY, "depth_mem_query" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_KEY, "depth_mem_key" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_VALUE, "depth_mem_value" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_OUT, "depth_mem_out" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "depth_mem_route_state" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "depth_mem_route_retr" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "depth_mem_route_diff" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "depth_mem_route_hist" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_ROUTE, "depth_mem_route" },
|
||||
{ LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "depth_mem_stream_gate" },
|
||||
{ LLM_TENSOR_CONT_HIDDEN_STATE, "cont_hidden_state" },
|
||||
{ LLM_TENSOR_CONT_HIDDEN_RETR, "cont_hidden_retr" },
|
||||
{ LLM_TENSOR_CONT_HIDDEN_DIFF, "cont_hidden_diff" },
|
||||
{ LLM_TENSOR_CONT_HIDDEN_ROUTE, "cont_hidden_route" },
|
||||
{ LLM_TENSOR_CONT_HIDDEN, "cont_hidden" },
|
||||
{ LLM_TENSOR_CONT_OUTPUT, "cont_output" },
|
||||
{ LLM_TENSOR_HC_ATTN_MIX, "blk.%d.hc_attn_mix" },
|
||||
{ LLM_TENSOR_HC_ATTN_READ, "blk.%d.hc_attn_read" },
|
||||
{ LLM_TENSOR_HC_ATTN_WRITE, "blk.%d.hc_attn_write" },
|
||||
{ LLM_TENSOR_HC_FFN_MIX, "blk.%d.hc_ffn_mix" },
|
||||
{ LLM_TENSOR_HC_FFN_READ, "blk.%d.hc_ffn_read" },
|
||||
{ LLM_TENSOR_HC_FFN_WRITE, "blk.%d.hc_ffn_write" },
|
||||
{ LLM_TENSOR_ATTN_PASS_LORA_DOWN, "blk.%d.attn_pass_lora_down" },
|
||||
{ LLM_TENSOR_ATTN_PASS_LORA_UP, "blk.%d.attn_pass_lora_up" },
|
||||
{ LLM_TENSOR_ATTN_PASS_GATE, "blk.%d.attn_pass_gate" },
|
||||
{ LLM_TENSOR_FFN_EXP_EMBD, "blk.%d.ffn_exp_embd" },
|
||||
{ LLM_TENSOR_FFN_PRE_EXP_NORM, "blk.%d.ffn_pre_exp_norm" },
|
||||
{ LLM_TENSOR_FFN_POST_EXP_NORM, "blk.%d.ffn_post_exp_norm" },
|
||||
{ LLM_TENSOR_FFN_RESID_ALPHA, "blk.%d.ffn_resid_alpha" },
|
||||
@@ -1158,6 +1195,36 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
|
||||
// latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU
|
||||
{LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_FFN_LATENT_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_STREAM_EMBD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}},
|
||||
{LLM_TENSOR_DEPTH_MEM_STATE_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_META_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_KEY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_VALUE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_DEPTH_MEM_STREAM_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_CONT_HIDDEN_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_CONT_HIDDEN_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_CONT_HIDDEN_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_CONT_HIDDEN_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_CONT_HIDDEN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_CONT_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_ATTN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_ATTN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_ATTN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_FFN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_FFN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_HC_FFN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_ATTN_PASS_LORA_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_ATTN_PASS_LORA_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_ATTN_PASS_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_FFN_EXP_EMBD, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},
|
||||
{LLM_TENSOR_FFN_PRE_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_FFN_POST_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
{LLM_TENSOR_FFN_RESID_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
|
||||
@@ -1286,6 +1353,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
|
||||
case LLM_ARCH_LFM3:
|
||||
case LLM_ARCH_NEMOTRON_H:
|
||||
case LLM_ARCH_NEMOTRON_H_MOE:
|
||||
case LLM_ARCH_METIS16:
|
||||
case LLM_ARCH_QWEN3NEXT:
|
||||
case LLM_ARCH_KIMI_LINEAR:
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
|
||||
@@ -106,6 +106,7 @@ enum llm_arch {
|
||||
LLM_ARCH_NEMOTRON,
|
||||
LLM_ARCH_NEMOTRON_H,
|
||||
LLM_ARCH_NEMOTRON_H_MOE,
|
||||
LLM_ARCH_METIS16,
|
||||
LLM_ARCH_EXAONE,
|
||||
LLM_ARCH_EXAONE4,
|
||||
LLM_ARCH_EXAONE_MOE,
|
||||
@@ -276,6 +277,10 @@ enum llm_kv {
|
||||
LLM_KV_HRM_H_CYCLES,
|
||||
LLM_KV_HRM_L_CYCLES,
|
||||
LLM_KV_HRM_PREFIX_LM,
|
||||
LLM_KV_METIS_PASS_COUNT,
|
||||
LLM_KV_METIS_MEMORY_LENGTH,
|
||||
LLM_KV_METIS_ROUTE_LENGTH,
|
||||
LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT,
|
||||
LLM_KV_REALTIME_DELAY_TOKENS,
|
||||
LLM_KV_MUDD_FEED_FORWARD_LENGTH,
|
||||
LLM_KV_MUDD_TAP_COUNT,
|
||||
@@ -333,6 +338,7 @@ enum llm_kv {
|
||||
LLM_KV_ATTENTION_KV_SOURCE_LAYERS,
|
||||
LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,
|
||||
LLM_KV_ATTENTION_OUTPUT_LORA_RANK,
|
||||
LLM_KV_ATTENTION_PASS_LORA_RANK,
|
||||
LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE,
|
||||
LLM_KV_ATTENTION_COMPRESS_RATIOS,
|
||||
LLM_KV_ATTENTION_SHARED_KV_LAYERS,
|
||||
@@ -391,6 +397,7 @@ enum llm_kv {
|
||||
LLM_KV_SSM_TIME_STEP_RANK,
|
||||
LLM_KV_SSM_GROUP_COUNT,
|
||||
LLM_KV_SSM_DT_B_C_RMS,
|
||||
LLM_KV_SSM_NORM_EPS,
|
||||
|
||||
LLM_KV_KDA_HEAD_DIM,
|
||||
LLM_KV_KDA_SAFE_GATE,
|
||||
@@ -566,6 +573,36 @@ enum llm_tensor {
|
||||
LLM_TENSOR_FFN_EXP_PROBS_B_VL,
|
||||
LLM_TENSOR_FFN_LATENT_DOWN,
|
||||
LLM_TENSOR_FFN_LATENT_UP,
|
||||
LLM_TENSOR_STREAM_EMBD,
|
||||
LLM_TENSOR_DEPTH_MEM_STATE_WRITE,
|
||||
LLM_TENSOR_DEPTH_MEM_META_WRITE,
|
||||
LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS,
|
||||
LLM_TENSOR_DEPTH_MEM_QUERY,
|
||||
LLM_TENSOR_DEPTH_MEM_KEY,
|
||||
LLM_TENSOR_DEPTH_MEM_VALUE,
|
||||
LLM_TENSOR_DEPTH_MEM_OUT,
|
||||
LLM_TENSOR_DEPTH_MEM_ROUTE_STATE,
|
||||
LLM_TENSOR_DEPTH_MEM_ROUTE_RETR,
|
||||
LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF,
|
||||
LLM_TENSOR_DEPTH_MEM_ROUTE_HIST,
|
||||
LLM_TENSOR_DEPTH_MEM_ROUTE,
|
||||
LLM_TENSOR_DEPTH_MEM_STREAM_GATE,
|
||||
LLM_TENSOR_CONT_HIDDEN_STATE,
|
||||
LLM_TENSOR_CONT_HIDDEN_RETR,
|
||||
LLM_TENSOR_CONT_HIDDEN_DIFF,
|
||||
LLM_TENSOR_CONT_HIDDEN_ROUTE,
|
||||
LLM_TENSOR_CONT_HIDDEN,
|
||||
LLM_TENSOR_CONT_OUTPUT,
|
||||
LLM_TENSOR_HC_ATTN_MIX,
|
||||
LLM_TENSOR_HC_ATTN_READ,
|
||||
LLM_TENSOR_HC_ATTN_WRITE,
|
||||
LLM_TENSOR_HC_FFN_MIX,
|
||||
LLM_TENSOR_HC_FFN_READ,
|
||||
LLM_TENSOR_HC_FFN_WRITE,
|
||||
LLM_TENSOR_ATTN_PASS_LORA_DOWN,
|
||||
LLM_TENSOR_ATTN_PASS_LORA_UP,
|
||||
LLM_TENSOR_ATTN_PASS_GATE,
|
||||
LLM_TENSOR_FFN_EXP_EMBD,
|
||||
LLM_TENSOR_FFN_PRE_EXP_NORM,
|
||||
LLM_TENSOR_FFN_POST_EXP_NORM,
|
||||
LLM_TENSOR_FFN_RESID_ALPHA,
|
||||
|
||||
@@ -2703,6 +2703,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
|
||||
res = std::max<uint32_t>(n_tokens * 80, 64u * model.n_tensors());
|
||||
} else if (model.arch == LLM_ARCH_QWEN3NEXT ||
|
||||
model.arch == LLM_ARCH_KIMI_LINEAR ||
|
||||
model.arch == LLM_ARCH_METIS16 ||
|
||||
model.arch == LLM_ARCH_BAILINGMOE3 ||
|
||||
model.arch == LLM_ARCH_QWEN35 ||
|
||||
model.arch == LLM_ARCH_QWEN35MOE ||
|
||||
|
||||
@@ -209,6 +209,7 @@ struct llama_hparams {
|
||||
uint32_t ssm_d_state = 0;
|
||||
uint32_t ssm_dt_rank = 0;
|
||||
uint32_t ssm_n_group = 0;
|
||||
float ssm_norm_eps = 0.0f; // 0 = use f_norm_rms_eps
|
||||
|
||||
// for MiniMax-Text-01 linear attention
|
||||
uint32_t n_embd_head_la = 0;
|
||||
@@ -230,6 +231,13 @@ struct llama_hparams {
|
||||
uint32_t n_hrm_l_cycles = 0;
|
||||
bool hrm_prefix_lm = false;
|
||||
|
||||
// metis16 (passes over shared layers, depth memory)
|
||||
uint32_t metis_n_pass = 0;
|
||||
uint32_t metis_n_mem = 0;
|
||||
uint32_t metis_n_route = 0;
|
||||
uint32_t metis_pass_lora_rank = 0;
|
||||
float metis_cont_limit = 0.0f;
|
||||
|
||||
// limite
|
||||
float f_xsa_eps = 0.0f;
|
||||
std::array<float, 3> final_logit_sigmoid_capping = {}; // scale, shift, temperature
|
||||
|
||||
@@ -247,6 +247,10 @@ void llama_model_saver::add_kv_from_model() {
|
||||
add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles);
|
||||
add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles);
|
||||
add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm);
|
||||
add_kv(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass);
|
||||
add_kv(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem);
|
||||
add_kv(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route);
|
||||
add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit);
|
||||
if (hparams.final_logit_sigmoid_capping[0] != 0.0f) {
|
||||
add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>(
|
||||
hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end()));
|
||||
@@ -323,6 +327,7 @@ void llama_model_saver::add_kv_from_model() {
|
||||
add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true);
|
||||
add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count);
|
||||
add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank);
|
||||
add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank);
|
||||
add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base);
|
||||
if (model->arch == LLM_ARCH_DEEPSEEK4 || hparams.dsv4_hc_mult > 0) {
|
||||
// the loader requires one compress ratio per layer, including nextn layers
|
||||
@@ -419,6 +424,7 @@ void llama_model_saver::add_kv_from_model() {
|
||||
add_kv(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
add_kv(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms);
|
||||
add_kv(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps);
|
||||
|
||||
add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);
|
||||
add_kv(LLM_KV_KDA_SAFE_GATE, hparams.kda_safe_gate);
|
||||
@@ -521,6 +527,32 @@ void llama_model_saver::add_tensors_from_model() {
|
||||
add_tensor(model->cls_out_b);
|
||||
add_tensor(model->cls_norm);
|
||||
add_tensor(model->hrm_z_l_init);
|
||||
add_tensor(model->stream_embd);
|
||||
add_tensor(model->depth_mem_state_write);
|
||||
add_tensor(model->depth_mem_meta_write);
|
||||
add_tensor(model->depth_mem_entry_bias);
|
||||
add_tensor(model->depth_mem_query);
|
||||
add_tensor(model->depth_mem_query_b);
|
||||
add_tensor(model->depth_mem_key);
|
||||
add_tensor(model->depth_mem_key_b);
|
||||
add_tensor(model->depth_mem_value);
|
||||
add_tensor(model->depth_mem_value_b);
|
||||
add_tensor(model->depth_mem_out);
|
||||
add_tensor(model->depth_mem_out_b);
|
||||
add_tensor(model->depth_mem_route_state);
|
||||
add_tensor(model->depth_mem_route_retr);
|
||||
add_tensor(model->depth_mem_route_diff);
|
||||
add_tensor(model->depth_mem_route_hist);
|
||||
add_tensor(model->depth_mem_route_b);
|
||||
add_tensor(model->depth_mem_stream_gate);
|
||||
add_tensor(model->depth_mem_stream_gate_b);
|
||||
add_tensor(model->cont_hidden_state);
|
||||
add_tensor(model->cont_hidden_retr);
|
||||
add_tensor(model->cont_hidden_diff);
|
||||
add_tensor(model->cont_hidden_route);
|
||||
add_tensor(model->cont_hidden_b);
|
||||
add_tensor(model->cont_output);
|
||||
add_tensor(model->cont_output_b);
|
||||
add_tensor(model->hc_head_fn);
|
||||
add_tensor(model->hc_head_base);
|
||||
add_tensor(model->hc_head_scale);
|
||||
|
||||
+5
-1
@@ -242,6 +242,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
|
||||
return new llama_model_nemotron_h(params);
|
||||
case LLM_ARCH_NEMOTRON_H_MOE:
|
||||
return new llama_model_nemotron_h_moe(params);
|
||||
case LLM_ARCH_METIS16:
|
||||
return new llama_model_metis16(params);
|
||||
case LLM_ARCH_EXAONE:
|
||||
return new llama_model_exaone(params);
|
||||
case LLM_ARCH_EXAONE4:
|
||||
@@ -515,7 +517,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
|
||||
};
|
||||
|
||||
auto get_tensor_config = [&]() -> tensor_config {
|
||||
if (ud->model->arch == LLM_ARCH_HRM_TEXT) {
|
||||
if (ud->model->arch == LLM_ARCH_HRM_TEXT || ud->model->arch == LLM_ARCH_METIS16) {
|
||||
// aliased cache slots cannot satisfy the meta-split invariants, so replicate all tensors
|
||||
return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, tensor, 0, 0};
|
||||
}
|
||||
@@ -1049,6 +1051,7 @@ const char * llm_type_name(llm_type type) {
|
||||
case LLM_TYPE_7B: return "7B";
|
||||
case LLM_TYPE_8B: return "8B";
|
||||
case LLM_TYPE_9B: return "9B";
|
||||
case LLM_TYPE_10B: return "10B";
|
||||
case LLM_TYPE_11B: return "11B";
|
||||
case LLM_TYPE_12B: return "12B";
|
||||
case LLM_TYPE_13B: return "13B";
|
||||
@@ -3618,6 +3621,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
|
||||
case LLM_ARCH_WAVTOKENIZER_DEC:
|
||||
case LLM_ARCH_NEMOTRON_H:
|
||||
case LLM_ARCH_NEMOTRON_H_MOE:
|
||||
case LLM_ARCH_METIS16:
|
||||
case LLM_ARCH_KIMI_LINEAR:
|
||||
case LLM_ARCH_KIMI_K3:
|
||||
case LLM_ARCH_GLM5_NEXT:
|
||||
|
||||
@@ -76,6 +76,7 @@ enum llm_type {
|
||||
LLM_TYPE_7B,
|
||||
LLM_TYPE_8B,
|
||||
LLM_TYPE_9B,
|
||||
LLM_TYPE_10B,
|
||||
LLM_TYPE_11B,
|
||||
LLM_TYPE_12B,
|
||||
LLM_TYPE_13B,
|
||||
@@ -646,6 +647,18 @@ struct llama_layer {
|
||||
// gemma4 layer output scale, reused for talkie embedding skip scale
|
||||
struct ggml_tensor * out_scale = nullptr;
|
||||
|
||||
// metis16 per-pass hyper-connection mixes, attention pass LoRA, expert route embeddings
|
||||
struct ggml_tensor * hc_attn_mix = nullptr;
|
||||
struct ggml_tensor * hc_attn_read = nullptr;
|
||||
struct ggml_tensor * hc_attn_write = nullptr;
|
||||
struct ggml_tensor * hc_ffn_mix = nullptr;
|
||||
struct ggml_tensor * hc_ffn_read = nullptr;
|
||||
struct ggml_tensor * hc_ffn_write = nullptr;
|
||||
struct ggml_tensor * attn_pass_lora_down = nullptr;
|
||||
struct ggml_tensor * attn_pass_lora_up = nullptr;
|
||||
struct ggml_tensor * attn_pass_gate = nullptr;
|
||||
struct ggml_tensor * ffn_exp_embd = nullptr;
|
||||
|
||||
// limite
|
||||
struct ggml_tensor * attn_ve_gate = nullptr;
|
||||
struct ggml_tensor * attn_xsa_alpha = nullptr;
|
||||
@@ -779,6 +792,34 @@ struct llama_model {
|
||||
// hrm-text initial low-cycle state
|
||||
struct ggml_tensor * hrm_z_l_init = nullptr;
|
||||
|
||||
// metis16 hyper-connection stream embeddings, depth memory and continuation head
|
||||
struct ggml_tensor * stream_embd = nullptr;
|
||||
struct ggml_tensor * depth_mem_state_write = nullptr;
|
||||
struct ggml_tensor * depth_mem_meta_write = nullptr;
|
||||
struct ggml_tensor * depth_mem_entry_bias = nullptr;
|
||||
struct ggml_tensor * depth_mem_query = nullptr;
|
||||
struct ggml_tensor * depth_mem_query_b = nullptr;
|
||||
struct ggml_tensor * depth_mem_key = nullptr;
|
||||
struct ggml_tensor * depth_mem_key_b = nullptr;
|
||||
struct ggml_tensor * depth_mem_value = nullptr;
|
||||
struct ggml_tensor * depth_mem_value_b = nullptr;
|
||||
struct ggml_tensor * depth_mem_out = nullptr;
|
||||
struct ggml_tensor * depth_mem_out_b = nullptr;
|
||||
struct ggml_tensor * depth_mem_route_state = nullptr;
|
||||
struct ggml_tensor * depth_mem_route_retr = nullptr;
|
||||
struct ggml_tensor * depth_mem_route_diff = nullptr;
|
||||
struct ggml_tensor * depth_mem_route_hist = nullptr;
|
||||
struct ggml_tensor * depth_mem_route_b = nullptr;
|
||||
struct ggml_tensor * depth_mem_stream_gate = nullptr;
|
||||
struct ggml_tensor * depth_mem_stream_gate_b = nullptr;
|
||||
struct ggml_tensor * cont_hidden_state = nullptr;
|
||||
struct ggml_tensor * cont_hidden_retr = nullptr;
|
||||
struct ggml_tensor * cont_hidden_diff = nullptr;
|
||||
struct ggml_tensor * cont_hidden_route = nullptr;
|
||||
struct ggml_tensor * cont_hidden_b = nullptr;
|
||||
struct ggml_tensor * cont_output = nullptr;
|
||||
struct ggml_tensor * cont_output_b = nullptr;
|
||||
|
||||
// DeepSeek-V4
|
||||
struct ggml_tensor * hc_head_fn = nullptr;
|
||||
struct ggml_tensor * hc_head_base = nullptr;
|
||||
|
||||
@@ -361,6 +361,16 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
|
||||
quantize &= name.find("indexer.k_proj.weight") == std::string::npos;
|
||||
quantize &= name.find("indexer.q_proj.weight") == std::string::npos;
|
||||
|
||||
// metis16: small control tensors that steer every layer; the stream and hc_ tables (ne[0] = 4) would fall back to F16
|
||||
if (arch == LLM_ARCH_METIS16) {
|
||||
quantize &= name.rfind("depth_mem_", 0) == std::string::npos;
|
||||
quantize &= name.rfind("cont_", 0) == std::string::npos;
|
||||
quantize &= name.find("stream_embd") == std::string::npos;
|
||||
quantize &= name.find("attn_pass_lora") == std::string::npos;
|
||||
quantize &= name.find("ffn_exp_embd") == std::string::npos;
|
||||
quantize &= name.find("hc_") == std::string::npos;
|
||||
}
|
||||
|
||||
// glm5-next
|
||||
if (arch == LLM_ARCH_GLM5_NEXT) {
|
||||
quantize &= name.find("hc_") == std::string::npos;
|
||||
|
||||
@@ -290,7 +290,11 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,
|
||||
// grouped RMS norm
|
||||
if (model.layers[il].ssm_norm) {
|
||||
y = ggml_reshape_4d(ctx0, y, d_inner / n_group, n_group, n_seq_tokens, n_seqs);
|
||||
y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il);
|
||||
if (hparams.ssm_norm_eps > 0.0f) {
|
||||
y = ggml_mul(ctx0, ggml_rms_norm(ctx0, y, hparams.ssm_norm_eps), model.layers[il].ssm_norm);
|
||||
} else {
|
||||
y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il);
|
||||
}
|
||||
}
|
||||
|
||||
y = ggml_reshape_2d(ctx0, y, d_inner, n_seq_tokens * n_seqs);
|
||||
|
||||
@@ -0,0 +1,401 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <cmath>
|
||||
|
||||
void llama_model_metis16::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
|
||||
ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
|
||||
ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
|
||||
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
||||
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
||||
ml.get_key(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps);
|
||||
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
|
||||
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
|
||||
ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
|
||||
ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size);
|
||||
|
||||
ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult);
|
||||
ml.get_key(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass);
|
||||
ml.get_key(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem);
|
||||
ml.get_key(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route);
|
||||
ml.get_key(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank);
|
||||
ml.get_key(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit);
|
||||
|
||||
n_pass = hparams.metis_n_pass;
|
||||
n_stream = hparams.dsv4_hc_mult;
|
||||
n_mem = hparams.metis_n_mem;
|
||||
n_route = hparams.metis_n_route;
|
||||
cont_limit = hparams.metis_cont_limit;
|
||||
|
||||
if (n_pass < 1 || hparams.n_layer() % n_pass != 0) {
|
||||
throw std::runtime_error(format("metis16: %u layers do not split into %u passes", hparams.n_layer(), n_pass));
|
||||
}
|
||||
n_layer_phys = hparams.n_layer() / n_pass;
|
||||
|
||||
bool has_attn = false;
|
||||
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
|
||||
if (hparams.n_head_kv(il) != hparams.n_head_kv(il % n_layer_phys)) {
|
||||
throw std::runtime_error(format("metis16: layer %u does not repeat the layout of layer %u", il, il % n_layer_phys));
|
||||
}
|
||||
hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;
|
||||
has_attn = has_attn || !hparams.is_recr_impl[il];
|
||||
}
|
||||
if (!has_attn) {
|
||||
throw std::runtime_error("metis16: at least one attention layer is required");
|
||||
}
|
||||
|
||||
switch (n_layer_phys) {
|
||||
case 12: type = LLM_TYPE_3B; break; // Praxis
|
||||
case 20: type = LLM_TYPE_10B; break; // Logos
|
||||
default: type = LLM_TYPE_UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
void llama_model_metis16::load_arch_tensors(llama_model_loader &) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const int64_t d_conv = hparams.ssm_d_conv;
|
||||
const int64_t d_inner = hparams.ssm_d_inner;
|
||||
const int64_t d_state = hparams.ssm_d_state;
|
||||
const int64_t n_ssm_head = hparams.ssm_dt_rank;
|
||||
const int64_t n_group = hparams.ssm_n_group;
|
||||
const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_ssm_head;
|
||||
const int64_t n_latent = hparams.moe_latent_size;
|
||||
const int64_t n_ff_shexp = hparams.n_ff_shexp;
|
||||
const int64_t n_s = n_stream;
|
||||
const int64_t n_p = n_pass;
|
||||
const int64_t n_m = n_mem;
|
||||
const int64_t n_r = n_route;
|
||||
|
||||
int64_t n_attn = 0;
|
||||
for (uint32_t il = 0; il < n_layer_phys; ++il) {
|
||||
n_attn += !hparams.is_recr(il);
|
||||
}
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
|
||||
if (output == NULL) {
|
||||
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
|
||||
}
|
||||
|
||||
stream_embd = create_tensor(tn(LLM_TENSOR_STREAM_EMBD, "weight"), {n_s, n_embd}, 0);
|
||||
|
||||
depth_mem_state_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "weight"), {n_embd, n_m}, 0);
|
||||
depth_mem_meta_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_META_WRITE, "weight"), {n_r + 3, n_m}, 0);
|
||||
depth_mem_entry_bias = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "weight"), {n_m, n_p*(n_attn + 1)}, 0);
|
||||
depth_mem_query = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "weight"), {n_embd, n_m}, 0);
|
||||
depth_mem_query_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "bias"), {n_m}, 0);
|
||||
depth_mem_key = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "weight"), {n_m, n_m}, 0);
|
||||
depth_mem_key_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "bias"), {n_m}, 0);
|
||||
depth_mem_value = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "weight"), {n_m, n_m}, 0);
|
||||
depth_mem_value_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "bias"), {n_m}, 0);
|
||||
depth_mem_out = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "weight"), {n_m, n_embd}, 0);
|
||||
depth_mem_out_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "bias"), {n_embd}, 0);
|
||||
depth_mem_route_state = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "weight"), {n_embd, n_r}, 0);
|
||||
depth_mem_route_retr = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "weight"), {n_embd, n_r}, 0);
|
||||
depth_mem_route_diff = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "weight"), {n_embd, n_r}, 0);
|
||||
depth_mem_route_hist = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "weight"), {n_r, n_r}, 0);
|
||||
depth_mem_route_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE, "bias"), {n_r}, 0);
|
||||
depth_mem_stream_gate = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "weight"), {n_embd, n_s}, 0);
|
||||
depth_mem_stream_gate_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "bias"), {n_s}, 0);
|
||||
|
||||
cont_hidden_state = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_STATE, "weight"), {n_embd, n_r}, 0);
|
||||
cont_hidden_retr = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_RETR, "weight"), {n_embd, n_r}, 0);
|
||||
cont_hidden_diff = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_DIFF, "weight"), {n_embd, n_r}, 0);
|
||||
cont_hidden_route = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_ROUTE, "weight"), {n_r, n_r}, 0);
|
||||
cont_hidden_b = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN, "bias"), {n_r}, 0);
|
||||
cont_output = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "weight"), {n_r, 1}, 0);
|
||||
cont_output_b = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "bias"), {1}, 0);
|
||||
|
||||
for (int il = 0; il < (int) n_layer_phys; ++il) {
|
||||
auto & layer = layers[il];
|
||||
|
||||
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), {n_embd}, 0);
|
||||
layer.hc_attn_mix = create_tensor(tn(LLM_TENSOR_HC_ATTN_MIX, "weight", il), {n_s, n_s, n_p}, 0);
|
||||
layer.hc_attn_read = create_tensor(tn(LLM_TENSOR_HC_ATTN_READ, "weight", il), {n_s, n_p}, 0);
|
||||
layer.hc_attn_write = create_tensor(tn(LLM_TENSOR_HC_ATTN_WRITE, "weight", il), {n_s, n_p}, 0);
|
||||
layer.hc_ffn_mix = create_tensor(tn(LLM_TENSOR_HC_FFN_MIX, "weight", il), {n_s, n_s, n_p}, 0);
|
||||
layer.hc_ffn_read = create_tensor(tn(LLM_TENSOR_HC_FFN_READ, "weight", il), {n_s, n_p}, 0);
|
||||
layer.hc_ffn_write = create_tensor(tn(LLM_TENSOR_HC_FFN_WRITE, "weight", il), {n_s, n_p}, 0);
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", il), {n_embd}, 0);
|
||||
|
||||
if (hparams.is_recr(il)) {
|
||||
layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", il), {n_embd, d_in_proj}, 0);
|
||||
layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), {d_conv, d_inner + 2*n_group*d_state}, 0);
|
||||
layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", il), {d_inner + 2*n_group*d_state}, 0);
|
||||
layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), {n_ssm_head}, 0);
|
||||
layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), {d_inner / n_group, n_group}, 0);
|
||||
layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), {d_inner, n_embd}, 0);
|
||||
|
||||
// no "weight" suffix for these
|
||||
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, il), {1, n_ssm_head}, 0);
|
||||
layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, il), {1, n_ssm_head}, 0);
|
||||
} else {
|
||||
const int64_t n_qkv = n_embd_head_k * (hparams.n_head(il) + 2*hparams.n_head_kv(il));
|
||||
const int64_t rank = hparams.metis_pass_lora_rank;
|
||||
|
||||
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), {n_embd, n_qkv}, 0);
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), {n_embd_head_k * hparams.n_head(il), n_embd}, 0);
|
||||
layer.attn_pass_lora_down = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_DOWN, "weight", il), {n_embd, rank}, 0);
|
||||
layer.attn_pass_lora_up = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_UP, "weight", il), {rank, n_qkv}, 0);
|
||||
layer.attn_pass_gate = create_tensor(tn(LLM_TENSOR_ATTN_PASS_GATE, "weight", il), {n_p}, 0);
|
||||
}
|
||||
|
||||
const int64_t n_ff_exp = hparams.n_ff_exp(il);
|
||||
|
||||
layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", il), {n_embd, n_latent}, 0);
|
||||
layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", il), {n_latent, n_embd}, 0);
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), {n_latent + n_r, n_expert}, 0);
|
||||
layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", il), {n_expert}, 0);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), {n_expert}, 0);
|
||||
layer.ffn_exp_embd = create_tensor(tn(LLM_TENSOR_FFN_EXP_EMBD, "weight", il), {n_r, n_expert}, 0);
|
||||
layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", il), {n_latent, 2*n_ff_exp, n_expert}, 0);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), {n_ff_exp, n_latent, n_expert}, 0);
|
||||
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0);
|
||||
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0);
|
||||
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), {n_ff_shexp, n_latent}, 0);
|
||||
}
|
||||
|
||||
// the virtual layers of later passes share the physical layer's weights
|
||||
for (int il = n_layer_phys; il < n_layer; ++il) {
|
||||
layers[il] = layers[il % n_layer_phys];
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_metis16::build_arch_graph(const llm_graph_params & params) const {
|
||||
return std::make_unique<graph>(*this, params);
|
||||
}
|
||||
|
||||
// streams are kept stream-innermost: [n_stream, n_embd, n_tokens]
|
||||
ggml_tensor * llama_model_metis16::graph::stream_mean(ggml_tensor * x) const {
|
||||
ggml_tensor * mean = ggml_mean(ctx0, ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1]*x->ne[2]));
|
||||
return ggml_reshape_2d(ctx0, mean, x->ne[1], x->ne[2]);
|
||||
}
|
||||
|
||||
// returns the normalized sublayer input
|
||||
ggml_tensor * llama_model_metis16::graph::hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm,
|
||||
int p, ggml_tensor ** mixed, int il) const {
|
||||
const int64_t n_s = x->ne[0];
|
||||
ggml_tensor * x2 = ggml_reshape_2d(ctx0, x, n_s, x->ne[1]*x->ne[2]);
|
||||
|
||||
ggml_tensor * m = ggml_view_2d(ctx0, mix, n_s, n_s, mix->nb[1], p*mix->nb[2]);
|
||||
*mixed = ggml_mul_mat(ctx0, m, x2);
|
||||
|
||||
ggml_tensor * r = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, read, n_s, p*read->nb[1]), n_s, 1);
|
||||
ggml_tensor * src = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, r, x2), x->ne[1], x->ne[2]);
|
||||
return build_norm(src, norm, NULL, LLM_NORM_RMS, il);
|
||||
}
|
||||
|
||||
ggml_tensor * llama_model_metis16::graph::hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const {
|
||||
const int64_t n_s = mixed->ne[0];
|
||||
ggml_tensor * w = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, write, n_s, p*write->nb[1]), 1, n_s);
|
||||
ggml_tensor * u = ggml_reshape_2d(ctx0, update, 1, update->ne[0]*update->ne[1]);
|
||||
ggml_tensor * out = ggml_add(ctx0, mixed, ggml_mul_mat(ctx0, w, u));
|
||||
return ggml_reshape_3d(ctx0, out, n_s, update->ne[0], update->ne[1]);
|
||||
}
|
||||
|
||||
// blends the retrieved entries into the streams; returns their summary
|
||||
ggml_tensor * llama_model_metis16::graph::mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const {
|
||||
const int64_t n_s = (*x)->ne[0];
|
||||
const int64_t n_m = model.n_mem;
|
||||
|
||||
ggml_tensor * x_e = ggml_cont(ctx0, ggml_permute(ctx0, *x, 1, 0, 2, 3));
|
||||
|
||||
ggml_tensor * q = ggml_mul_mat(ctx0, model.depth_mem_query, ggml_reshape_2d(ctx0, x_e, n_embd, n_s*n_tokens));
|
||||
q = ggml_reshape_3d(ctx0, ggml_add(ctx0, q, model.depth_mem_query_b), n_m, n_s, n_tokens);
|
||||
|
||||
ggml_tensor * kq = ggml_mul_mat(ctx0, bank_k, q);
|
||||
kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(n_m)), 0.0f);
|
||||
|
||||
ggml_tensor * ret = ggml_mul_mat(ctx0, bank_vt, kq);
|
||||
ggml_tensor * proj = ggml_mul_mat(ctx0, model.depth_mem_out, ggml_reshape_2d(ctx0, ret, n_m, n_s*n_tokens));
|
||||
proj = ggml_reshape_3d(ctx0, ggml_add(ctx0, proj, model.depth_mem_out_b), n_embd, n_s, n_tokens);
|
||||
|
||||
ggml_tensor * gate = ggml_sum_rows(ctx0, ggml_mul(ctx0, x_e, model.depth_mem_stream_gate));
|
||||
gate = ggml_sigmoid(ctx0, ggml_add(ctx0, gate, ggml_reshape_2d(ctx0, model.depth_mem_stream_gate_b, 1, n_s)));
|
||||
|
||||
ggml_tensor * proj_s = ggml_cont(ctx0, ggml_permute(ctx0, proj, 1, 0, 2, 3));
|
||||
*x = ggml_add(ctx0, *x, ggml_mul(ctx0, proj_s, ggml_reshape_3d(ctx0, gate, n_s, 1, n_tokens)));
|
||||
|
||||
return stream_mean(proj_s);
|
||||
}
|
||||
|
||||
void llama_model_metis16::graph::mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary,
|
||||
ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry) {
|
||||
const int64_t n_m = model.n_mem;
|
||||
|
||||
if (cont == nullptr) {
|
||||
cont = ggml_scale(ctx0, confidence, 0.0f);
|
||||
}
|
||||
ggml_tensor * meta = ggml_concat(ctx0, ggml_concat(ctx0, summary, entropy, 0), ggml_concat(ctx0, confidence, cont, 0), 0);
|
||||
|
||||
ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_state_write, stream_mean(x)), ggml_mul_mat(ctx0, model.depth_mem_meta_write, meta));
|
||||
e = ggml_add(ctx0, e, ggml_view_1d(ctx0, model.depth_mem_entry_bias, n_m, entry*model.depth_mem_entry_bias->nb[1]));
|
||||
|
||||
ggml_tensor * k = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_key, e), model.depth_mem_key_b);
|
||||
ggml_tensor * v = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_value, e), model.depth_mem_value_b);
|
||||
k = ggml_reshape_3d(ctx0, k, n_m, 1, n_tokens);
|
||||
v = ggml_reshape_3d(ctx0, v, 1, n_m, n_tokens);
|
||||
bank_k = bank_k ? ggml_concat(ctx0, bank_k, k, 1) : k;
|
||||
bank_vt = bank_vt ? ggml_concat(ctx0, bank_vt, v, 0) : v;
|
||||
}
|
||||
|
||||
ggml_tensor * llama_model_metis16::graph::route_features(const llama_model_metis16 & model, ggml_tensor * state,
|
||||
ggml_tensor * summary, ggml_tensor * diff, ggml_tensor * history) const {
|
||||
ggml_tensor * f = ggml_mul_mat(ctx0, model.depth_mem_route_state, state);
|
||||
if (summary) {
|
||||
f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_retr, summary));
|
||||
}
|
||||
if (diff) {
|
||||
f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_diff, diff));
|
||||
}
|
||||
if (history) {
|
||||
f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_hist, history));
|
||||
}
|
||||
return ggml_tanh(ctx0, ggml_add(ctx0, f, model.depth_mem_route_b));
|
||||
}
|
||||
|
||||
llama_model_metis16::graph::graph(const llama_model_metis16 & model, const llm_graph_params & params) :
|
||||
llm_build_mamba_base(params) {
|
||||
const int64_t n_s = model.n_stream;
|
||||
const int n_p = model.n_pass;
|
||||
const int n_phys = model.n_layer_phys;
|
||||
const int64_t n_r = model.n_route;
|
||||
const int64_t n_hd = hparams.n_embd_head_k();
|
||||
const int64_t n_exp = hparams.n_expert;
|
||||
const int64_t n_used = hparams.n_expert_used();
|
||||
|
||||
ggml_tensor * inpL = build_inp_embd(model.tok_embd);
|
||||
auto * inp = build_inp_mem_hybrid();
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
ggml_tensor * x = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, inpL, 1, n_embd, n_tokens), n_s, n_embd, n_tokens, 1);
|
||||
x = ggml_add(ctx0, x, model.stream_embd);
|
||||
cb(x, "streams_in", -1);
|
||||
|
||||
ggml_tensor * prev_state = stream_mean(x);
|
||||
ggml_tensor * history = nullptr; // routing history, [n_route, n_tokens]
|
||||
ggml_tensor * cont = nullptr; // depth head reading of the last pass, [1, n_tokens]
|
||||
int n_entry = 0;
|
||||
|
||||
for (int p = 0; p < n_p; ++p) {
|
||||
ggml_tensor * summ = nullptr;
|
||||
ggml_tensor * ent = nullptr;
|
||||
ggml_tensor * conf = nullptr;
|
||||
|
||||
for (int il_p = 0; il_p < n_phys; ++il_p) {
|
||||
const int il = p*n_phys + il_p;
|
||||
const auto & layer = model.layers[il];
|
||||
const bool is_attn = !hparams.is_recr(il);
|
||||
const bool writes = is_attn || il_p == n_phys - 1;
|
||||
|
||||
ggml_tensor * retrieved = bank_k ? mem_read(model, &x) : nullptr;
|
||||
ggml_tensor * feats = route_features(model, stream_mean(x), retrieved, nullptr, history);
|
||||
|
||||
ggml_tensor * mixed = nullptr;
|
||||
ggml_tensor * cur = hc_read(x, layer.hc_attn_mix, layer.hc_attn_read, layer.attn_norm, p, &mixed, il);
|
||||
if (is_attn) {
|
||||
ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur);
|
||||
ggml_tensor * lora = ggml_mul_mat(ctx0, layer.attn_pass_lora_up, ggml_mul_mat(ctx0, layer.attn_pass_lora_down, cur));
|
||||
qkv = ggml_add(ctx0, qkv, ggml_mul(ctx0, lora, ggml_view_1d(ctx0, layer.attn_pass_gate, 1, p*layer.attn_pass_gate->nb[0])));
|
||||
|
||||
const int64_t n_head = hparams.n_head(il);
|
||||
const int64_t n_head_kv = hparams.n_head_kv(il);
|
||||
const size_t es = ggml_element_size(qkv);
|
||||
ggml_tensor * Q = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head, n_tokens, n_hd*es, qkv->nb[1], 0));
|
||||
ggml_tensor * K = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*n_head*es));
|
||||
ggml_tensor * V = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*(n_head + n_head_kv)*es));
|
||||
|
||||
cur = build_attn(inp->get_attn(), layer.wo, nullptr, nullptr, Q, K, V, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_hd)), il);
|
||||
} else {
|
||||
cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);
|
||||
}
|
||||
cb(cur, "mixer_out", il);
|
||||
x = hc_write(mixed, layer.hc_attn_write, cur, p);
|
||||
|
||||
cur = hc_read(x, layer.hc_ffn_mix, layer.hc_ffn_read, layer.ffn_norm, p, &mixed, il);
|
||||
{
|
||||
ggml_tensor * latent = ggml_mul_mat(ctx0, layer.ffn_latent_down, cur);
|
||||
ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, ggml_concat(ctx0, latent, feats, 0));
|
||||
logits = ggml_add(ctx0, logits, layer.ffn_gate_inp_b);
|
||||
|
||||
// the balance bias picks the experts; the weights come from the unbiased logits
|
||||
ggml_tensor * selected = ggml_argsort_top_k(ctx0, ggml_add(ctx0, logits, layer.ffn_exp_probs_b), n_used);
|
||||
|
||||
ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr,
|
||||
n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p,
|
||||
logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected);
|
||||
ggml_tensor * shexp = build_ffn(latent,
|
||||
layer.ffn_up_shexp, NULL, NULL,
|
||||
layer.ffn_gate_shexp, NULL, NULL,
|
||||
layer.ffn_down_shexp, NULL, NULL,
|
||||
NULL, LLM_FFN_SILU, LLM_FFN_PAR, il_p);
|
||||
cur = ggml_mul_mat(ctx0, layer.ffn_latent_up, ggml_add(ctx0, moe_out, shexp));
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
// routing summary: the chosen experts' identity vectors blended by their weights
|
||||
ggml_tensor * logits3 = ggml_reshape_3d(ctx0, logits, 1, n_exp, n_tokens);
|
||||
ggml_tensor * weights = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, logits3, selected), n_used, n_tokens));
|
||||
ggml_tensor * rows = ggml_get_rows(ctx0, layer.ffn_exp_embd, ggml_reshape_1d(ctx0, ggml_cont(ctx0, selected), n_used*n_tokens));
|
||||
rows = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, rows, n_r, n_used, n_tokens), 1, 0, 2, 3));
|
||||
summ = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, rows, ggml_reshape_3d(ctx0, weights, n_used, 1, n_tokens)), n_r, n_tokens);
|
||||
|
||||
if (writes) {
|
||||
ggml_tensor * probs = ggml_soft_max(ctx0, logits);
|
||||
ent = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, ggml_log(ctx0, ggml_clamp(ctx0, probs, 1.0e-9f, 1.0f))));
|
||||
ent = ggml_scale(ctx0, ent, -1.0f);
|
||||
conf = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, probs, 1, n_exp, n_tokens),
|
||||
ggml_reshape_2d(ctx0, ggml_argmax(ctx0, probs), 1, n_tokens));
|
||||
conf = ggml_reshape_2d(ctx0, conf, 1, n_tokens);
|
||||
}
|
||||
}
|
||||
x = hc_write(mixed, layer.hc_ffn_write, cur, p);
|
||||
cb(x, "streams", il);
|
||||
|
||||
history = history ? ggml_add(ctx0, ggml_scale(ctx0, history, 0.5f), ggml_scale(ctx0, summ, 0.5f)) : ggml_scale(ctx0, summ, 0.5f);
|
||||
|
||||
if (is_attn) {
|
||||
mem_write(model, x, summ, ent, conf, cont, n_entry++);
|
||||
}
|
||||
}
|
||||
|
||||
// end of the pass: one more memory read, the depth head, and an entry recording where the pass left the token
|
||||
ggml_tensor * retrieved = mem_read(model, &x);
|
||||
ggml_tensor * state = stream_mean(x);
|
||||
ggml_tensor * diff = ggml_sub(ctx0, state, prev_state);
|
||||
ggml_tensor * feats = route_features(model, state, retrieved, diff, history);
|
||||
|
||||
ggml_tensor * h = ggml_mul_mat(ctx0, model.cont_hidden_state, state);
|
||||
h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_retr, retrieved));
|
||||
h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_diff, diff));
|
||||
h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_route, feats));
|
||||
h = ggml_silu(ctx0, ggml_add(ctx0, h, model.cont_hidden_b));
|
||||
|
||||
ggml_tensor * logit = ggml_add(ctx0, ggml_mul_mat(ctx0, model.cont_output, h), model.cont_output_b);
|
||||
ggml_tensor * scale = ggml_clamp(ctx0, ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, state))), 1.0e-6f, INFINITY);
|
||||
ggml_tensor * innov = ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, diff)));
|
||||
logit = ggml_add(ctx0, logit, ggml_scale_bias(ctx0, ggml_tanh(ctx0, ggml_div(ctx0, innov, scale)), 1.0f, -0.5f));
|
||||
cont = ggml_sigmoid(ctx0, ggml_clamp(ctx0, logit, -model.cont_limit, model.cont_limit));
|
||||
cb(cont, "continuation", p);
|
||||
|
||||
mem_write(model, x, summ, ent, conf, cont, n_entry++);
|
||||
prev_state = state;
|
||||
}
|
||||
|
||||
ggml_tensor * cur = stream_mean(x);
|
||||
if (inp_out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
}
|
||||
|
||||
cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
|
||||
cb(cur, "result_norm", -1);
|
||||
res->t_embd = cur;
|
||||
|
||||
cur = build_lora_mm(model.output, cur);
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
|
||||
}
|
||||
@@ -1799,6 +1799,39 @@ struct llama_model_nemotron_h_moe : public llama_model_nemotron_h {
|
||||
};
|
||||
|
||||
|
||||
struct llama_model_metis16 : public llama_model_base {
|
||||
llama_model_metis16(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
void load_arch_tensors(llama_model_loader & ml) override;
|
||||
|
||||
uint32_t n_pass = 1;
|
||||
uint32_t n_stream = 1;
|
||||
uint32_t n_mem = 0;
|
||||
uint32_t n_route = 0;
|
||||
uint32_t n_layer_phys = 0;
|
||||
float cont_limit = 0.0f;
|
||||
|
||||
struct graph : public llm_build_mamba_base {
|
||||
graph(const llama_model_metis16 & model, const llm_graph_params & params);
|
||||
|
||||
// memory bank of the current forward: keys [n_mem, n_entry, n_tokens], values [n_entry, n_mem, n_tokens]
|
||||
ggml_tensor * bank_k = nullptr;
|
||||
ggml_tensor * bank_vt = nullptr;
|
||||
|
||||
ggml_tensor * stream_mean(ggml_tensor * x) const;
|
||||
ggml_tensor * hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, int p, ggml_tensor ** mixed, int il) const;
|
||||
ggml_tensor * hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const;
|
||||
ggml_tensor * mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const;
|
||||
void mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary,
|
||||
ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry);
|
||||
ggml_tensor * route_features(const llama_model_metis16 & model, ggml_tensor * state, ggml_tensor * summary,
|
||||
ggml_tensor * diff, ggml_tensor * history) const;
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
|
||||
struct llama_model_exaone : public llama_model_base {
|
||||
llama_model_exaone(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
|
||||
@@ -188,6 +188,8 @@ maple ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV,
|
||||
mellum ,0 ,any ,MUL+ADD , 4
|
||||
mellum ,0 ,any ,RMS_NORM+MUL , 9
|
||||
mellum ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
metis16 ,0 ,any ,MUL+ADD , 8
|
||||
metis16 ,0 ,any ,RMS_NORM+MUL , 11
|
||||
mimo2 ,0 ,any ,MUL+ADD , 4
|
||||
mimo2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
minicpm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
|
||||
|
@@ -295,6 +295,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
n_ff = 192;
|
||||
} else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
|
||||
n_layer = 3;
|
||||
} else if (arch == LLM_ARCH_METIS16) {
|
||||
n_layer = 4; // 2 passes over 2 physical layers, one attention and one Mamba-2
|
||||
} else if (arch == LLM_ARCH_CHAMELEON) {
|
||||
n_vocab = 10240;
|
||||
} else if (arch == LLM_ARCH_QWEN3TTS) {
|
||||
@@ -369,6 +371,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer);
|
||||
}
|
||||
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);
|
||||
} else if (arch == LLM_ARCH_METIS16) {
|
||||
// the layer types repeat in every pass
|
||||
std::vector<uint32_t> n_head_kv_per_layer;
|
||||
n_head_kv_per_layer.reserve(n_layer);
|
||||
for (uint32_t il = 0; il < n_layer; il++) {
|
||||
n_head_kv_per_layer.push_back(il % 2 == 0 ? n_head_kv : 0);
|
||||
}
|
||||
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
|
||||
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_kv_per_layer);
|
||||
} else {
|
||||
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
|
||||
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv);
|
||||
@@ -616,6 +627,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
|
||||
}
|
||||
|
||||
if (arch == LLM_ARCH_METIS16) {
|
||||
ms.add_kv(LLM_KV_METIS_PASS_COUNT, uint32_t(2));
|
||||
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
|
||||
ms.add_kv(LLM_KV_METIS_MEMORY_LENGTH, uint32_t(64));
|
||||
ms.add_kv(LLM_KV_METIS_ROUTE_LENGTH, uint32_t(32));
|
||||
ms.add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, uint32_t(8));
|
||||
ms.add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, 4.0f);
|
||||
ms.add_kv(LLM_KV_SSM_NORM_EPS, 1e-5f);
|
||||
ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd/2);
|
||||
}
|
||||
|
||||
if (arch == LLM_ARCH_LIMITE) {
|
||||
ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f);
|
||||
ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f);
|
||||
@@ -972,6 +994,7 @@ static bool moe_mandatory(const llm_arch arch) {
|
||||
case LLM_ARCH_MAPLE:
|
||||
case LLM_ARCH_MUSE_SPARK:
|
||||
case LLM_ARCH_KOLIBRI1:
|
||||
case LLM_ARCH_METIS16:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
Reference in new issue
Block a user