model : support for Prism Bonsai 2 27B (#29600)
* Runtime support for Prism Bonsai 2 27B Assisted-by: Claude Code * address prism hadamard runtime feedback * move hadamard tensors into method, define folded weight * load_* hadamard fixes * cont : clean-up * conversion cleanup Assisted-by: Claude Code * prism hadamard key and methods for converter Assisted-by: Claude Code * address review bot feedback Assisted-by: Claude Code --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
1 parent
1cf7767765
commit
a9d27ac693
14 files changed
+817
-7
No files matched your search
@@ -673,6 +673,171 @@ class ModelBase:
|
||||
if algo == "W4A16_NVFP4":
|
||||
self._prec_a4[gguf_name] = False
|
||||
|
||||
def hadamard_folded_names(self) -> set[str]:
|
||||
"""Source-tensor names folded under a Hadamard manifest, or empty."""
|
||||
cached = getattr(self, "_hadamard_folded_names", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
names: set[str] = set()
|
||||
manifest_path = self.dir_model / "hadamard_packing.json"
|
||||
if manifest_path.is_file():
|
||||
with manifest_path.open("r", encoding="utf-8") as f:
|
||||
for record in json.load(f).get("tensors", []):
|
||||
if isinstance(record, dict) and isinstance(record.get("name"), str):
|
||||
names.add(record["name"])
|
||||
self._hadamard_folded_names = names
|
||||
return names
|
||||
|
||||
def add_hadamard_metadata(self) -> None:
|
||||
"""Transfer a packed-checkpoint transform contract into GGUF metadata."""
|
||||
manifest_path = self.dir_model / "hadamard_packing.json"
|
||||
if not manifest_path.is_file():
|
||||
return
|
||||
|
||||
with manifest_path.open("r", encoding="utf-8") as f:
|
||||
manifest = json.load(f)
|
||||
|
||||
schema_version = manifest.get("schema_version")
|
||||
if schema_version not in (1, 2, 3) or manifest.get("kind") != "hadamard-weight-fold":
|
||||
raise ValueError(f"Unsupported Hadamard manifest: {manifest_path}")
|
||||
if manifest.get("status") != "requires-matching-runtime":
|
||||
raise ValueError(f"Unexpected Hadamard manifest status: {manifest.get('status')!r}")
|
||||
|
||||
transform = manifest.get("transform")
|
||||
if not isinstance(transform, dict):
|
||||
raise ValueError("Hadamard manifest is missing transform metadata")
|
||||
block_size = transform.get("block_size")
|
||||
if not isinstance(block_size, int) or block_size <= 0 or block_size & (block_size - 1):
|
||||
raise ValueError(f"Invalid Hadamard block size: {block_size!r}")
|
||||
if transform.get("name") != "normalized-signed-sylvester-walsh-hadamard":
|
||||
raise ValueError(f"Unsupported Hadamard transform: {transform.get('name')!r}")
|
||||
sign_mode = transform.get("sign_mode")
|
||||
if sign_mode not in ("identity", "explicit"):
|
||||
raise ValueError(f"Unsupported Hadamard sign mode: {sign_mode!r}")
|
||||
sign_widths: list[int] = []
|
||||
sign_values: list[int] = []
|
||||
if sign_mode == "explicit":
|
||||
signs = manifest.get("signs")
|
||||
if not isinstance(signs, dict) or not signs:
|
||||
raise ValueError("explicit sign mode requires a signs table")
|
||||
for width_str, vec in sorted(signs.items(), key=lambda kv: int(kv[0])):
|
||||
width = int(width_str)
|
||||
# same width rule as the runtime, so a manifest that converts also loads
|
||||
if width <= 0 or width % block_size != 0:
|
||||
raise ValueError(
|
||||
f"sign width {width} must be positive and a multiple of block size {block_size}"
|
||||
)
|
||||
if len(vec) != width or any(v not in (-1, 1) for v in vec):
|
||||
raise ValueError(f"invalid sign vector for width {width}")
|
||||
sign_widths.append(width)
|
||||
sign_values.extend(int(v) for v in vec)
|
||||
|
||||
tensor_records = manifest.get("tensors")
|
||||
if not isinstance(tensor_records, list) or not tensor_records:
|
||||
raise ValueError("Hadamard manifest has no folded tensors")
|
||||
|
||||
# only build_lora_mm/build_lora_mm_id apply the transform: refuse archs and tensor kinds that can skip them
|
||||
_HADAMARD_ARCHS = {
|
||||
gguf.MODEL_ARCH.LLAMA,
|
||||
gguf.MODEL_ARCH.QWEN3,
|
||||
gguf.MODEL_ARCH.QWEN3MOE,
|
||||
gguf.MODEL_ARCH.QWEN35,
|
||||
gguf.MODEL_ARCH.QWEN35MOE,
|
||||
gguf.MODEL_ARCH.QWEN3NEXT,
|
||||
}
|
||||
if self.model_arch not in _HADAMARD_ARCHS:
|
||||
raise ValueError(
|
||||
f"Hadamard folding is not verified for arch {self.model_arch.name}; "
|
||||
"the runtime would load the GGUF without applying the activation transform"
|
||||
)
|
||||
_HADAMARD_KINDS = re.compile(
|
||||
r"output\.weight|"
|
||||
r"blk\.\d+\.("
|
||||
r"attn_q|attn_k|attn_v|attn_qkv|attn_gate|attn_output"
|
||||
r"|ffn_gate|ffn_up|ffn_down"
|
||||
r"|ffn_gate_exps|ffn_up_exps|ffn_down_exps|ffn_gate_up_exps"
|
||||
r"|ffn_gate_shexp|ffn_up_shexp|ffn_down_shexp"
|
||||
r"|ssm_out"
|
||||
r")\.weight"
|
||||
)
|
||||
weight_names: list[str] = []
|
||||
inverse_weight_names: list[str] = []
|
||||
for record in tensor_records:
|
||||
if not isinstance(record, dict) or not isinstance(record.get("name"), str):
|
||||
raise ValueError("Hadamard manifest has an invalid tensor record")
|
||||
if record.get("axis") != -1:
|
||||
raise ValueError(f"Unsupported Hadamard tensor axis for {record['name']!r}")
|
||||
role = record.get("role", "fold-before-matmul")
|
||||
if role not in ("fold-before-matmul", "inverse-after-lookup"):
|
||||
raise ValueError(f"Unsupported Hadamard tensor role for {record['name']!r}: {role!r}")
|
||||
filtered = self.filter_tensors((record["name"], lambda: torch.empty(0)))
|
||||
if filtered is None:
|
||||
raise ValueError(f"Hadamard tensor is filtered out: {record['name']!r}")
|
||||
mapped = self.map_tensor_name(filtered[0])
|
||||
if role == "inverse-after-lookup":
|
||||
# the runtime applies the inverse only after the token-embedding lookup, other latent tables stay rotated
|
||||
if mapped != "token_embd.weight":
|
||||
raise ValueError(
|
||||
f"Hadamard tensor {record['name']!r} maps to {mapped!r}, which is not a "
|
||||
"verified inverse-after-lookup table"
|
||||
)
|
||||
inverse_weight_names.append(mapped)
|
||||
else:
|
||||
if not _HADAMARD_KINDS.fullmatch(mapped):
|
||||
raise ValueError(
|
||||
f"Hadamard tensor {record['name']!r} maps to {mapped!r}, which is not on a "
|
||||
"verified Hadamard-aware matmul path"
|
||||
)
|
||||
weight_names.append(mapped)
|
||||
|
||||
# --fuse-qkv writes one attn_qkv per layer, so Q, K and V must all be folded and get one name
|
||||
for bid in sorted(self._fusable_qkv_weight_layers):
|
||||
qkv = [
|
||||
self.format_tensor_name(t, bid)
|
||||
for t in (gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V)
|
||||
]
|
||||
n_folded = sum(name in weight_names for name in qkv)
|
||||
if n_folded == 0:
|
||||
continue
|
||||
if n_folded != len(qkv):
|
||||
raise ValueError(f"--fuse-qkv needs all of Q, K and V folded in layer {bid}, or none of them")
|
||||
weight_names = [name for name in weight_names if name not in qkv]
|
||||
weight_names.append(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid))
|
||||
|
||||
tied_output = manifest.get("tied_output", False)
|
||||
if not isinstance(tied_output, bool) or (schema_version == 3) != tied_output:
|
||||
raise ValueError("Hadamard schema 3 requires tied_output=true; older schemas forbid it")
|
||||
if tied_output:
|
||||
if inverse_weight_names != ["token_embd.weight"]:
|
||||
raise ValueError("Tied Hadamard output requires one latent token embedding")
|
||||
if not self.hparams.get("tie_word_embeddings", False):
|
||||
raise ValueError("Tied Hadamard output requires tie_word_embeddings=true")
|
||||
if "output.weight" in weight_names or any(
|
||||
self.tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) == "output.weight"
|
||||
for name in self.model_tensors
|
||||
):
|
||||
raise ValueError("Tied Hadamard output must not carry a separate output head")
|
||||
self.gguf_writer.add_prism_hadamard_tied_output(True)
|
||||
elif "token_embd.weight" in inverse_weight_names and self.hparams.get("tie_word_embeddings", False):
|
||||
raise ValueError("A tied latent embedding requires Hadamard schema 3 and tied_output=true")
|
||||
|
||||
self.gguf_writer.add_prism_hadamard_version(2 if tied_output else 1)
|
||||
self.gguf_writer.add_prism_hadamard_block_size(block_size)
|
||||
self.gguf_writer.add_prism_hadamard_transform("normalized-sylvester-walsh-hadamard")
|
||||
self.gguf_writer.add_prism_hadamard_axis("input-last-dimension")
|
||||
self.gguf_writer.add_prism_hadamard_sign_mode(sign_mode)
|
||||
self.gguf_writer.add_prism_hadamard_weight_names(weight_names)
|
||||
if sign_mode == "explicit":
|
||||
self.gguf_writer.add_prism_hadamard_sign_widths(sign_widths)
|
||||
self.gguf_writer.add_prism_hadamard_sign_values(sign_values)
|
||||
if inverse_weight_names:
|
||||
self.gguf_writer.add_prism_hadamard_inverse_weight_names(inverse_weight_names)
|
||||
if getattr(self, "_hadamard_gdn_v_grouped", False):
|
||||
self.gguf_writer.add_prism_hadamard_gdn_v_grouped(True)
|
||||
logger.info("GGUF Hadamard: linear-attention out_proj kept in grouped V order")
|
||||
logger.info("GGUF Hadamard contract: H%d, sign_mode=%s, %d folded weight(s), %d inverse-lookup",
|
||||
block_size, sign_mode, len(weight_names), len(inverse_weight_names))
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
raise NotImplementedError("set_gguf_parameters() must be implemented in subclasses")
|
||||
|
||||
@@ -1220,6 +1385,8 @@ class ModelBase:
|
||||
logger.info("Set model quantization version")
|
||||
self.gguf_writer.add_quantization_version(gguf.GGML_QUANT_VERSION)
|
||||
|
||||
self.add_hadamard_metadata()
|
||||
|
||||
if self._prec_a4:
|
||||
names = sorted(self._prec_a4.keys())
|
||||
values = [self._prec_a4[n] for n in names]
|
||||
|
||||
+19
-7
@@ -570,11 +570,15 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
|
||||
elif name.endswith((".linear_attn.in_proj_a.weight", ".linear_attn.in_proj_b.weight")):
|
||||
weight, scale = reorder_rows(weight, scale, 1)
|
||||
elif name.endswith(".linear_attn.out_proj.weight"):
|
||||
col_perm = self._reorder_v_heads(
|
||||
torch.arange(num_v_heads * head_v_dim, dtype=torch.long).unsqueeze(0),
|
||||
1, num_k_heads, num_v_per_k, head_v_dim,
|
||||
).squeeze(0)
|
||||
weight, scale = apply_col_perm(weight, scale, col_perm)
|
||||
if self._hadamard_folds_tensor(name):
|
||||
# folded weight: keep the grouped V order, the runtime permutes the activation instead
|
||||
self._hadamard_gdn_v_grouped = True
|
||||
else:
|
||||
col_perm = self._reorder_v_heads(
|
||||
torch.arange(num_v_heads * head_v_dim, dtype=torch.long).unsqueeze(0),
|
||||
1, num_k_heads, num_v_per_k, head_v_dim,
|
||||
).squeeze(0)
|
||||
weight, scale = apply_col_perm(weight, scale, col_perm)
|
||||
|
||||
return weight, scale
|
||||
|
||||
@@ -582,6 +586,10 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
|
||||
weight, scale = self._transform_nvfp4_weight(name, weight, scale)
|
||||
super()._repack_nvfp4(name, weight, scale, scale2, input_scale)
|
||||
|
||||
def _hadamard_folds_tensor(self, name: str) -> bool:
|
||||
# a manifest name and `name` can differ only by leading wrapper prefixes
|
||||
return any(name.endswith(n) or n.endswith(name) for n in self.hadamard_folded_names())
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
num_k_heads = self.hparams.get("linear_num_key_heads", 0)
|
||||
num_v_heads = self.hparams.get("linear_num_value_heads", 0)
|
||||
@@ -628,8 +636,12 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
|
||||
data_torch = torch.cat([qk_part, v_part], dim=0)
|
||||
|
||||
elif ".out_proj." in name:
|
||||
# Out projection weight: reorder columns (input dimension)
|
||||
data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim)
|
||||
if self._hadamard_folds_tensor(name):
|
||||
# folded weight: keep the grouped V order, the runtime permutes the activation instead
|
||||
self._hadamard_gdn_v_grouped = True
|
||||
else:
|
||||
# Out projection weight: reorder columns (input dimension)
|
||||
data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim)
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@@ -495,6 +495,19 @@ class Keys:
|
||||
BETA = "xielu.beta"
|
||||
EPS = "xielu.eps"
|
||||
|
||||
class PrismHadamard:
|
||||
VERSION = "prism.hadamard.version"
|
||||
TIED_OUTPUT = "prism.hadamard.tied_output"
|
||||
BLOCK_SIZE = "prism.hadamard.block_size"
|
||||
TRANSFORM = "prism.hadamard.transform"
|
||||
AXIS = "prism.hadamard.axis"
|
||||
SIGN_MODE = "prism.hadamard.sign_mode"
|
||||
SIGN_WIDTHS = "prism.hadamard.sign_widths"
|
||||
SIGN_VALUES = "prism.hadamard.sign_values"
|
||||
WEIGHT_NAMES = "prism.hadamard.weight_names"
|
||||
INVERSE_WEIGHT_NAMES = "prism.hadamard.inverse_weight_names"
|
||||
GDN_V_GROUPED = "prism.hadamard.gdn_v_grouped"
|
||||
|
||||
|
||||
#
|
||||
# recommended mapping of model tensor names for storage in gguf
|
||||
|
||||
@@ -1634,6 +1634,39 @@ class GGUFWriter:
|
||||
def add_xielu_eps(self, values: Sequence[float]):
|
||||
self.add_array(Keys.xIELU.EPS, values)
|
||||
|
||||
def add_prism_hadamard_version(self, value: int) -> None:
|
||||
self.add_uint32(Keys.PrismHadamard.VERSION, value)
|
||||
|
||||
def add_prism_hadamard_tied_output(self, value: bool) -> None:
|
||||
self.add_bool(Keys.PrismHadamard.TIED_OUTPUT, value)
|
||||
|
||||
def add_prism_hadamard_block_size(self, value: int) -> None:
|
||||
self.add_uint32(Keys.PrismHadamard.BLOCK_SIZE, value)
|
||||
|
||||
def add_prism_hadamard_transform(self, value: str) -> None:
|
||||
self.add_string(Keys.PrismHadamard.TRANSFORM, value)
|
||||
|
||||
def add_prism_hadamard_axis(self, value: str) -> None:
|
||||
self.add_string(Keys.PrismHadamard.AXIS, value)
|
||||
|
||||
def add_prism_hadamard_sign_mode(self, value: str) -> None:
|
||||
self.add_string(Keys.PrismHadamard.SIGN_MODE, value)
|
||||
|
||||
def add_prism_hadamard_sign_widths(self, values: Sequence[int]) -> None:
|
||||
self.add_array(Keys.PrismHadamard.SIGN_WIDTHS, values)
|
||||
|
||||
def add_prism_hadamard_sign_values(self, values: Sequence[int]) -> None:
|
||||
self.add_array(Keys.PrismHadamard.SIGN_VALUES, values)
|
||||
|
||||
def add_prism_hadamard_weight_names(self, names: Sequence[str]) -> None:
|
||||
self.add_array(Keys.PrismHadamard.WEIGHT_NAMES, names)
|
||||
|
||||
def add_prism_hadamard_inverse_weight_names(self, names: Sequence[str]) -> None:
|
||||
self.add_array(Keys.PrismHadamard.INVERSE_WEIGHT_NAMES, names)
|
||||
|
||||
def add_prism_hadamard_gdn_v_grouped(self, value: bool) -> None:
|
||||
self.add_bool(Keys.PrismHadamard.GDN_V_GROUPED, value)
|
||||
|
||||
def add_attention_value_expert_count(self, count: int):
|
||||
self.add_uint32(Keys.Attention.VALUE_EXPERT_COUNT.format(arch=self.arch), count)
|
||||
|
||||
|
||||
@@ -334,6 +334,10 @@ static void llama_adapter_lora_init_impl(llama_model & model, FILE * file, llama
|
||||
if (!model_tensor) {
|
||||
throw std::runtime_error("LoRA tensor '" + name + "' does not exist in base model (hint: maybe wrong base model?)");
|
||||
}
|
||||
// the LoRA delta of a Hadamard-folded weight reads the transformed input, so the result is wrong
|
||||
if (model.hdmd.weight_blocks.count(name) || model.hdmd.inverse_blocks.count(name)) {
|
||||
throw std::runtime_error("LoRA tensor '" + name + "' targets a prism.hadamard folded weight, which is not supported");
|
||||
}
|
||||
|
||||
auto * buft = ggml_backend_buffer_get_type(model_tensor->buffer);
|
||||
|
||||
|
||||
@@ -441,6 +441,18 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
||||
{ LLM_KV_XIELU_BETA, "xielu.beta" },
|
||||
{ LLM_KV_XIELU_EPS, "xielu.eps" },
|
||||
|
||||
{ LLM_KV_PRISM_HADAMARD_VERSION, "prism.hadamard.version" },
|
||||
{ LLM_KV_PRISM_HADAMARD_TIED_OUTPUT, "prism.hadamard.tied_output" },
|
||||
{ LLM_KV_PRISM_HADAMARD_BLOCK_SIZE, "prism.hadamard.block_size" },
|
||||
{ LLM_KV_PRISM_HADAMARD_TRANSFORM, "prism.hadamard.transform" },
|
||||
{ LLM_KV_PRISM_HADAMARD_AXIS, "prism.hadamard.axis" },
|
||||
{ LLM_KV_PRISM_HADAMARD_SIGN_MODE, "prism.hadamard.sign_mode" },
|
||||
{ LLM_KV_PRISM_HADAMARD_SIGN_WIDTHS, "prism.hadamard.sign_widths" },
|
||||
{ LLM_KV_PRISM_HADAMARD_SIGN_VALUES, "prism.hadamard.sign_values" },
|
||||
{ LLM_KV_PRISM_HADAMARD_WEIGHT_NAMES, "prism.hadamard.weight_names" },
|
||||
{ LLM_KV_PRISM_HADAMARD_INVERSE_WEIGHT_NAMES, "prism.hadamard.inverse_weight_names" },
|
||||
{ LLM_KV_PRISM_HADAMARD_GDN_V_GROUPED, "prism.hadamard.gdn_v_grouped" },
|
||||
|
||||
// deprecated
|
||||
{ LLM_KV_TOKENIZER_PREFIX_ID, "tokenizer.ggml.prefix_token_id" },
|
||||
{ LLM_KV_TOKENIZER_SUFFIX_ID, "tokenizer.ggml.suffix_token_id" },
|
||||
|
||||
@@ -440,6 +440,18 @@ enum llm_kv {
|
||||
LLM_KV_XIELU_BETA,
|
||||
LLM_KV_XIELU_EPS,
|
||||
|
||||
LLM_KV_PRISM_HADAMARD_VERSION,
|
||||
LLM_KV_PRISM_HADAMARD_TIED_OUTPUT,
|
||||
LLM_KV_PRISM_HADAMARD_BLOCK_SIZE,
|
||||
LLM_KV_PRISM_HADAMARD_TRANSFORM,
|
||||
LLM_KV_PRISM_HADAMARD_AXIS,
|
||||
LLM_KV_PRISM_HADAMARD_SIGN_MODE,
|
||||
LLM_KV_PRISM_HADAMARD_SIGN_WIDTHS,
|
||||
LLM_KV_PRISM_HADAMARD_SIGN_VALUES,
|
||||
LLM_KV_PRISM_HADAMARD_WEIGHT_NAMES,
|
||||
LLM_KV_PRISM_HADAMARD_INVERSE_WEIGHT_NAMES,
|
||||
LLM_KV_PRISM_HADAMARD_GDN_V_GROUPED,
|
||||
|
||||
// deprecated:
|
||||
LLM_KV_TOKENIZER_PREFIX_ID,
|
||||
LLM_KV_TOKENIZER_SUFFIX_ID,
|
||||
|
||||
@@ -26,6 +26,89 @@
|
||||
// llama_context
|
||||
//
|
||||
|
||||
// check that each folded weight in the graph gets its Hadamard transform, and each latent lookup gets the inverse
|
||||
// without this check, an arch that skips the transform helpers loads and computes wrong results
|
||||
static void llama_verify_hadamard_graph(
|
||||
ggml_cgraph * gf,
|
||||
const llama_hadamard_rotations & rotations,
|
||||
const llama_hadamard_rotations & inverses,
|
||||
const llama_moe_cache * moe_cache) {
|
||||
// the MoE cache gives the matmul a copy of the expert weights, so the copy also needs the transform of its source
|
||||
llama_hadamard_rotations forward = rotations;
|
||||
for (const auto & [w, t] : rotations) {
|
||||
if (const ggml_tensor * cached = moe_cache ? moe_cache->get_experts(w) : nullptr) {
|
||||
forward.emplace(cached, t);
|
||||
}
|
||||
}
|
||||
|
||||
auto unwrap = [](const ggml_tensor * t) {
|
||||
while (t && (t->op == GGML_OP_RESHAPE || t->op == GGML_OP_VIEW)) {
|
||||
t = t->src[0];
|
||||
}
|
||||
return t;
|
||||
};
|
||||
|
||||
std::map<const ggml_tensor *, bool> lookups; // get_rows results of latent tables
|
||||
|
||||
for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) {
|
||||
const ggml_tensor * node = ggml_graph_node(gf, i);
|
||||
|
||||
if (node->op == GGML_OP_GET_ROWS && inverses.count(node->src[0])) {
|
||||
lookups.emplace(node, false);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (node->op != GGML_OP_MUL_MAT && node->op != GGML_OP_MUL_MAT_ID) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (node->op == GGML_OP_MUL_MAT && ((const int32_t *) node->op_params)[1] == GGML_HINT_SRC0_IS_HADAMARD) {
|
||||
const auto lk = lookups.find(unwrap(node->src[1]));
|
||||
if (lk != lookups.end()) {
|
||||
lk->second = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto it = forward.find(node->src[0]);
|
||||
if (it == forward.end()) {
|
||||
if (inverses.count(node->src[0])) {
|
||||
throw std::runtime_error(format(
|
||||
"Hadamard-latent table '%s' is used as a head without a forward transform", node->src[0]->name));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const ggml_tensor * src = unwrap(node->src[1]);
|
||||
const bool transformed = src && src->op == GGML_OP_MUL_MAT &&
|
||||
((const int32_t *) src->op_params)[1] == GGML_HINT_SRC0_IS_HADAMARD &&
|
||||
src->src[0] == it->second.rot;
|
||||
if (!transformed) {
|
||||
throw std::runtime_error(format(
|
||||
"Hadamard-folded weight '%s' is consumed without its activation transform; "
|
||||
"this graph's matmul path does not support prism.hadamard folding",
|
||||
node->src[0]->name));
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto & [node, ok] : lookups) {
|
||||
if (!ok) {
|
||||
for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) {
|
||||
const ggml_tensor * n2 = ggml_graph_node(gf, i);
|
||||
for (int s = 0; s < GGML_MAX_SRC && n2->src[s]; ++s) {
|
||||
if (unwrap(n2->src[s]) == node) {
|
||||
LLAMA_LOG_WARN("%s: latent lookup '%s' consumed by op=%s name='%s' src%d hint=%d\n",
|
||||
__func__, node->name, ggml_op_name(n2->op), n2->name, s,
|
||||
((const int32_t *) n2->op_params)[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw std::runtime_error(format(
|
||||
"Hadamard-latent table '%s' is read without the inverse transform",
|
||||
node->src[0]->name));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) {
|
||||
switch (ctx_type) {
|
||||
case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT;
|
||||
@@ -2562,6 +2645,12 @@ ggml_cgraph * llama_context::graph_reserve(
|
||||
|
||||
auto * gf = model.build_graph(gparams);
|
||||
|
||||
// check the graph before scheduling: cross-backend copies break the producer chain that the check follows
|
||||
if (!hadamard_verified && gf && (!model.hdmd.rot.empty() || !model.hdmd.inv.empty())) {
|
||||
llama_verify_hadamard_graph(gf, model.hdmd.rot, model.hdmd.inv, moe_cache.get());
|
||||
hadamard_verified = true;
|
||||
}
|
||||
|
||||
this->n_input_tensors = llama_graph_n_input_tensors(gf);
|
||||
this->n_outputs = save_n_outputs;
|
||||
|
||||
@@ -2600,6 +2689,7 @@ llm_graph_params llama_context::graph_params(
|
||||
/*.cross =*/ &cross,
|
||||
/*.moe_cache =*/ moe_cache.get(),
|
||||
/*.prec_policy =*/ &model.prec_policy,
|
||||
/*.hdmd =*/ model.hdmd.rot.empty() ? nullptr : &model.hdmd,
|
||||
/*.samplers =*/ sampling.samplers,
|
||||
/*.n_outputs =*/ n_outputs,
|
||||
/*.cb =*/ graph_get_cb(),
|
||||
|
||||
@@ -413,6 +413,9 @@ private:
|
||||
// env: LLAMA_GRAPH_REUSE_DISABLE
|
||||
bool graph_reuse_disable = false;
|
||||
|
||||
// true after the prism.hadamard coverage check passes on a reserved graph
|
||||
bool hadamard_verified = false;
|
||||
|
||||
// perf
|
||||
mutable int64_t t_start_us = 0;
|
||||
mutable int64_t t_load_us = 0;
|
||||
|
||||
@@ -1396,6 +1396,7 @@ void llm_graph_result::reset() {
|
||||
|
||||
inputs.clear();
|
||||
fused_nodes.clear();
|
||||
hdmd_inputs.clear();
|
||||
|
||||
buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));
|
||||
|
||||
@@ -1501,6 +1502,15 @@ void llm_graph_result::add_fused_node(llm_graph_fused_node result) {
|
||||
fused_nodes.push_back(result);
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_result::get_hdmd_input(const ggml_tensor * cur, const ggml_tensor * rot) const {
|
||||
const auto it = hdmd_inputs.find({ cur, rot });
|
||||
return it == hdmd_inputs.end() ? nullptr : it->second;
|
||||
}
|
||||
|
||||
void llm_graph_result::set_hdmd_input(const ggml_tensor * cur, const ggml_tensor * rot, ggml_tensor * res) {
|
||||
hdmd_inputs[{ cur, rot }] = res;
|
||||
}
|
||||
|
||||
void llm_graph_result::set_params(const llm_graph_params & params) {
|
||||
this->params = params;
|
||||
}
|
||||
@@ -1548,6 +1558,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) :
|
||||
cross (params.cross),
|
||||
moe_cache (params.moe_cache),
|
||||
prec_policy (params.prec_policy),
|
||||
hdmd (params.hdmd),
|
||||
samplers (params.samplers),
|
||||
cb_func (params.cb),
|
||||
res (params.res),
|
||||
@@ -1570,10 +1581,42 @@ ggml_tensor * llm_graph_context::build_cvec(
|
||||
return cvec->apply_to(ctx0, cur, il);
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_hadamard_input(
|
||||
ggml_tensor * w,
|
||||
ggml_tensor * cur) const {
|
||||
if (!hdmd) {
|
||||
return cur;
|
||||
}
|
||||
const auto it = hdmd->rot.find(w);
|
||||
if (it == hdmd->rot.end()) {
|
||||
return cur;
|
||||
}
|
||||
const auto & t = it->second;
|
||||
if (ggml_tensor * x = res->get_hdmd_input(cur, t.rot)) {
|
||||
return x;
|
||||
}
|
||||
ggml_tensor * x = cur;
|
||||
if (t.perm_rep > 1) {
|
||||
// tiled [hd, nk, rep] -> grouped [hd, rep, nk] feature order
|
||||
x = ggml_is_contiguous(x) ? x : ggml_cont(ctx0, x);
|
||||
const int64_t ne1 = x->ne[1], ne2 = x->ne[2], ne3 = x->ne[3];
|
||||
x = ggml_reshape_4d(ctx0, x, t.perm_hd, t.perm_nk, t.perm_rep, ne1*ne2*ne3);
|
||||
x = ggml_cont(ctx0, ggml_permute(ctx0, x, 0, 2, 1, 3));
|
||||
x = ggml_reshape_4d(ctx0, x, t.perm_hd*t.perm_nk*t.perm_rep, ne1, ne2, ne3);
|
||||
}
|
||||
if (t.signs) {
|
||||
x = ggml_mul(ctx0, x, t.signs);
|
||||
}
|
||||
x = llama_mul_mat_hadamard(ctx0, x, t.rot);
|
||||
res->set_hdmd_input(cur, t.rot, x);
|
||||
return x;
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_lora_mm(
|
||||
ggml_tensor * w,
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * w_s) const {
|
||||
cur = build_hadamard_input(w, cur);
|
||||
ggml_tensor * res = ggml_mul_mat(ctx0, w, cur);
|
||||
|
||||
if (prec_policy) {
|
||||
@@ -1615,6 +1658,7 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
|
||||
ggml_tensor * ids,
|
||||
ggml_tensor * w_s,
|
||||
ggml_tensor * slots) const {
|
||||
cur = build_hadamard_input(w, cur);
|
||||
// the experts in the MoE cache are selected by their slots
|
||||
ggml_tensor * res = slots == nullptr ?
|
||||
ggml_mul_mat_id(ctx0, w, cur, ids) :
|
||||
@@ -2481,6 +2525,9 @@ ggml_tensor * llm_graph_context::build_moe_cache_slots(
|
||||
ggml_backend_sched_set_tensor_backend(sched, slots, moe_cache->backend(il));
|
||||
cb(slots, "ffn_moe_slots", il);
|
||||
|
||||
// add the lookup now: the transform of a Hadamard-folded expert reads host weights and would start its split first
|
||||
ggml_build_forward_expand(gf, slots);
|
||||
|
||||
return ggml_reshape_2d(ctx0, slots, selected_experts->ne[0], selected_experts->ne[1]); // [n_expert_used, n_tokens]
|
||||
}
|
||||
|
||||
@@ -2544,6 +2591,16 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
|
||||
// TODO: when lora is active, this is likely going to cause issues similar to https://github.com/ggml-org/llama.cpp/pull/30160
|
||||
// need to add lora tests and refactor the logic to make the lora GET_ROWS go at the front of the graph
|
||||
auto build_tok = [&](ggml_tensor * cur, ggml_tensor * ids) {
|
||||
// a Hadamard-latent table stores rotated rows: restore the primal basis, h = s * (H z)
|
||||
if (hdmd) {
|
||||
if (const auto it = hdmd->inv.find(tok_embd); it != hdmd->inv.end()) {
|
||||
cur = llama_mul_mat_hadamard(ctx0, cur, it->second.rot);
|
||||
if (it->second.signs) {
|
||||
cur = ggml_mul(ctx0, cur, it->second.signs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// apply lora for embedding tokens if needed
|
||||
for (const auto & lora : *loras) {
|
||||
llama_adapter_lora_weight * lw = lora.first->get_weight(tok_embd);
|
||||
|
||||
@@ -20,6 +20,7 @@ struct ggml_tensor;
|
||||
struct llama_cparams;
|
||||
struct llama_layer;
|
||||
struct llama_prec_policy;
|
||||
struct llama_hadamard;
|
||||
|
||||
class llama_moe_cache;
|
||||
|
||||
@@ -803,6 +804,8 @@ struct llm_graph_params {
|
||||
|
||||
const llama_prec_policy * prec_policy = nullptr;
|
||||
|
||||
const llama_hadamard * hdmd = nullptr;
|
||||
|
||||
std::map<llama_seq_id, llama_sampler *> samplers;
|
||||
|
||||
static bool samplers_equal(
|
||||
@@ -943,6 +946,10 @@ public:
|
||||
|
||||
void add_fused_node(llm_graph_fused_node result);
|
||||
|
||||
// Hadamard-transformed activations, keyed by (input, rotation): folded weights that read the same activation share one transform
|
||||
ggml_tensor * get_hdmd_input(const ggml_tensor * cur, const ggml_tensor * rot) const;
|
||||
void set_hdmd_input(const ggml_tensor * cur, const ggml_tensor * rot, ggml_tensor * res);
|
||||
|
||||
const std::vector<llm_graph_fused_node> & get_fused_nodes() const { return fused_nodes; }
|
||||
|
||||
void set_params(const llm_graph_params & params);
|
||||
@@ -965,6 +972,8 @@ public:
|
||||
std::vector<llm_graph_input_ptr> inputs;
|
||||
std::vector<llm_graph_fused_node> fused_nodes;
|
||||
|
||||
std::map<std::pair<const ggml_tensor *, const ggml_tensor *>, ggml_tensor *> hdmd_inputs;
|
||||
|
||||
ggml_context_ptr ctx_compute;
|
||||
|
||||
// memory buffers used to evaluate the model
|
||||
@@ -1048,6 +1057,8 @@ struct llm_graph_context {
|
||||
|
||||
const llama_prec_policy * prec_policy;
|
||||
|
||||
const llama_hadamard * hdmd;
|
||||
|
||||
std::map<llama_seq_id, llama_sampler *> samplers;
|
||||
|
||||
const llm_graph_cb & cb_func;
|
||||
@@ -1080,6 +1091,11 @@ struct llm_graph_context {
|
||||
ggml_tensor * cur,
|
||||
int il) const;
|
||||
|
||||
// apply the activation-side transform of a Hadamard-folded weight, if any
|
||||
ggml_tensor * build_hadamard_input(
|
||||
ggml_tensor * w,
|
||||
ggml_tensor * cur) const;
|
||||
|
||||
// do mat_mul, while optionally apply lora and per-tensor scale
|
||||
ggml_tensor * build_lora_mm(
|
||||
ggml_tensor * w,
|
||||
|
||||
@@ -1350,6 +1350,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
|
||||
gguf_kv.emplace(name, value);
|
||||
}
|
||||
|
||||
load_hparams_hadamard(ml);
|
||||
|
||||
// get general kv
|
||||
ml.get_key(LLM_KV_GENERAL_NAME, name, false);
|
||||
|
||||
@@ -2015,9 +2017,342 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
}
|
||||
}
|
||||
|
||||
load_tensors_hadamard();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// read and check the prism.hadamard metadata, and record the folded weights (load_tensors_hadamard makes the tensors)
|
||||
void llama_model_base::load_hparams_hadamard(llama_model_loader & ml) {
|
||||
uint32_t hadamard_version = 0;
|
||||
ml.get_key(LLM_KV_PRISM_HADAMARD_TIED_OUTPUT, hdmd.tied_output, false);
|
||||
if (ml.get_key(LLM_KV_PRISM_HADAMARD_VERSION, hadamard_version, false)) {
|
||||
if (hadamard_version != 1 && hadamard_version != 2) {
|
||||
throw std::runtime_error(format("unsupported prism.hadamard.version: %u", hadamard_version));
|
||||
}
|
||||
|
||||
if ((hadamard_version == 2) != hdmd.tied_output) {
|
||||
throw std::runtime_error("prism.hadamard version 2 requires tied_output=true; version 1 forbids it");
|
||||
}
|
||||
if (hdmd.tied_output && ml.get_weight("output.weight")) {
|
||||
throw std::runtime_error("prism.hadamard.tied_output requires output.weight to be absent");
|
||||
}
|
||||
|
||||
uint32_t block_size = 0;
|
||||
std::string transform;
|
||||
std::string axis;
|
||||
std::string sign_mode;
|
||||
std::vector<std::string> weight_names;
|
||||
|
||||
ml.get_key(LLM_KV_PRISM_HADAMARD_BLOCK_SIZE, block_size);
|
||||
ml.get_key(LLM_KV_PRISM_HADAMARD_TRANSFORM, transform);
|
||||
ml.get_key(LLM_KV_PRISM_HADAMARD_AXIS, axis);
|
||||
ml.get_key(LLM_KV_PRISM_HADAMARD_SIGN_MODE, sign_mode);
|
||||
ml.get_arr(LLM_KV_PRISM_HADAMARD_WEIGHT_NAMES, weight_names);
|
||||
|
||||
if (block_size == 0 || (block_size & (block_size - 1)) != 0) {
|
||||
throw std::runtime_error(format("invalid prism.hadamard.block_size: %u", block_size));
|
||||
}
|
||||
if (transform != "normalized-sylvester-walsh-hadamard") {
|
||||
throw std::runtime_error(format("unsupported prism.hadamard.transform: %s", transform.c_str()));
|
||||
}
|
||||
if (axis != "input-last-dimension") {
|
||||
throw std::runtime_error(format("unsupported prism.hadamard.axis: %s", axis.c_str()));
|
||||
}
|
||||
if (sign_mode != "identity" && sign_mode != "explicit") {
|
||||
throw std::runtime_error(format("unsupported prism.hadamard.sign_mode: %s", sign_mode.c_str()));
|
||||
}
|
||||
if (weight_names.empty()) {
|
||||
throw std::runtime_error("prism.hadamard.weight_names is empty");
|
||||
}
|
||||
|
||||
if (sign_mode == "explicit") {
|
||||
std::vector<int32_t> sign_widths;
|
||||
std::vector<int32_t> sign_values;
|
||||
ml.get_arr(LLM_KV_PRISM_HADAMARD_SIGN_WIDTHS, sign_widths);
|
||||
ml.get_arr(LLM_KV_PRISM_HADAMARD_SIGN_VALUES, sign_values);
|
||||
// explicit mode with no widths gives an empty sign table, which acts as identity and changes the model
|
||||
if (sign_widths.empty()) {
|
||||
throw std::runtime_error("prism.hadamard.sign_mode is explicit but sign_widths is empty");
|
||||
}
|
||||
size_t off = 0;
|
||||
for (const int32_t width : sign_widths) {
|
||||
if (width <= 0 || (uint32_t) width % block_size != 0 || off + width > sign_values.size()) {
|
||||
throw std::runtime_error(format("invalid prism.hadamard sign width: %d", width));
|
||||
}
|
||||
if (hdmd.sign_data.count(width)) {
|
||||
throw std::runtime_error(format("duplicate prism.hadamard sign width: %d", width));
|
||||
}
|
||||
auto & vec = hdmd.sign_data[width];
|
||||
vec.assign(sign_values.begin() + off, sign_values.begin() + off + width);
|
||||
for (const int32_t v : vec) {
|
||||
if (v != 1 && v != -1) {
|
||||
throw std::runtime_error("prism.hadamard sign values must be +/-1");
|
||||
}
|
||||
}
|
||||
off += width;
|
||||
}
|
||||
if (off != sign_values.size()) {
|
||||
throw std::runtime_error("prism.hadamard.sign_values length mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
ml.get_key(LLM_KV_PRISM_HADAMARD_GDN_V_GROUPED, hdmd.gdn_v_grouped, false);
|
||||
|
||||
// only build_lora_mm/build_lora_mm_id apply the transform: refuse archs and tensor kinds that can skip them
|
||||
switch (arch) {
|
||||
case LLM_ARCH_LLAMA:
|
||||
case LLM_ARCH_QWEN3:
|
||||
case LLM_ARCH_QWEN3MOE:
|
||||
case LLM_ARCH_QWEN35:
|
||||
case LLM_ARCH_QWEN35MOE:
|
||||
case LLM_ARCH_QWEN3NEXT:
|
||||
break;
|
||||
default:
|
||||
throw std::runtime_error(format(
|
||||
"prism.hadamard: arch '%s' is not verified to apply the activation transform to all folded weights",
|
||||
llm_arch_name(arch)));
|
||||
}
|
||||
|
||||
// a folded weight W_f = W*D*H is the weight W with the +1/-1 signs D and the normalized block Hadamard H folded in
|
||||
// H*H = I and D*D = I, so W*x = W_f*(H*(D*x)): the graph applies D, then H, to the matmul input
|
||||
const auto is_foldable_weight = [](const std::string & name) {
|
||||
static const char * kinds[] = {
|
||||
"attn_q", "attn_k", "attn_v", "attn_qkv", "attn_gate", "attn_output",
|
||||
"ffn_gate", "ffn_up", "ffn_down",
|
||||
"ffn_gate_exps", "ffn_up_exps", "ffn_down_exps", "ffn_gate_up_exps",
|
||||
"ffn_gate_shexp", "ffn_up_shexp", "ffn_down_shexp",
|
||||
"ssm_out",
|
||||
};
|
||||
if (name == "output.weight") {
|
||||
return true; // the output head is built through build_lora_mm in every arch
|
||||
}
|
||||
if (name.compare(0, 4, "blk.") != 0) {
|
||||
return false;
|
||||
}
|
||||
size_t pos = 4;
|
||||
while (pos < name.size() && isdigit((unsigned char) name[pos])) {
|
||||
pos++;
|
||||
}
|
||||
if (pos == 4 || pos >= name.size() || name[pos] != '.') {
|
||||
return false;
|
||||
}
|
||||
pos++;
|
||||
for (const char * kind : kinds) {
|
||||
const std::string suffix = std::string(kind) + ".weight";
|
||||
if (name.compare(pos, std::string::npos, suffix) == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (const auto & weight_name : weight_names) {
|
||||
if (!is_foldable_weight(weight_name)) {
|
||||
throw std::runtime_error(format(
|
||||
"prism.hadamard: weight '%s' is not on a verified Hadamard-aware matmul path", weight_name.c_str()));
|
||||
}
|
||||
if (!hdmd.weight_blocks.emplace(weight_name, block_size).second) {
|
||||
throw std::runtime_error(format("duplicate prism.hadamard weight: %s", weight_name.c_str()));
|
||||
}
|
||||
}
|
||||
|
||||
// tables read by row lookup store latent rows: the inverse transform goes on the lookup result
|
||||
std::vector<std::string> inverse_names;
|
||||
ml.get_arr(LLM_KV_PRISM_HADAMARD_INVERSE_WEIGHT_NAMES, inverse_names, false);
|
||||
for (const auto & name : inverse_names) {
|
||||
// the graph applies the inverse only after the token-embedding lookup, other latent tables stay rotated
|
||||
if (name != "token_embd.weight") {
|
||||
throw std::runtime_error(format(
|
||||
"prism.hadamard: weight '%s' is not a verified inverse-after-lookup table", name.c_str()));
|
||||
}
|
||||
if (hdmd.weight_blocks.count(name) || !hdmd.inverse_blocks.emplace(name, block_size).second) {
|
||||
throw std::runtime_error(format("duplicate prism.hadamard inverse weight: %s", name.c_str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hdmd.tied_output) {
|
||||
if (hadamard_version != 2) {
|
||||
throw std::runtime_error("prism.hadamard.tied_output requires version 2");
|
||||
}
|
||||
const auto it = hdmd.inverse_blocks.find("token_embd.weight");
|
||||
if (it == hdmd.inverse_blocks.end()) {
|
||||
throw std::runtime_error("prism.hadamard.tied_output requires a latent token embedding");
|
||||
}
|
||||
hdmd.weight_blocks.emplace("token_embd.weight", it->second);
|
||||
} else if (hdmd.inverse_blocks.count("token_embd.weight") && !ml.get_weight("output.weight")) {
|
||||
throw std::runtime_error("a tied Hadamard output requires version 2 and tied_output=true");
|
||||
}
|
||||
}
|
||||
|
||||
// make one Hadamard matrix per block size and one sign vector per width (the GGUF does not contain them)
|
||||
// each tensor goes on the buffer type of its weights (never a CPU extra type), and each transform goes in hdmd.rot or hdmd.inv
|
||||
void llama_model_base::load_tensors_hadamard() {
|
||||
if (hdmd.weight_blocks.empty() && hdmd.inverse_blocks.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct hadamard_rotation {
|
||||
uint32_t block_size;
|
||||
ggml_backend_buffer_type_t buft;
|
||||
ggml_tensor * tensor;
|
||||
};
|
||||
|
||||
std::vector<hadamard_rotation> rotations;
|
||||
std::map<std::pair<uint32_t, ggml_backend_buffer_type_t>, ggml_tensor *> sign_tensors;
|
||||
|
||||
const std::pair<const std::unordered_map<std::string, uint32_t> *, llama_hadamard_rotations *> groups[] = {
|
||||
{ &hdmd.weight_blocks, &hdmd.rot },
|
||||
{ &hdmd.inverse_blocks, &hdmd.inv },
|
||||
};
|
||||
// inverse transforms use the buffer type of the forward rotations, not the host type of a CPU-mapped table (no PCIe round trip per token)
|
||||
ggml_backend_buffer_type_t preferred_buft = nullptr;
|
||||
|
||||
for (const auto & [blocks, target] : groups) {
|
||||
for (const auto & entry : *blocks) {
|
||||
const std::string & weight_name = entry.first;
|
||||
const uint32_t block_size = entry.second;
|
||||
const ggml_tensor * weight = get_tensor(weight_name.c_str());
|
||||
if (hdmd.tied_output && weight_name == "token_embd.weight") {
|
||||
weight = target == &hdmd.rot ? output : tok_embd;
|
||||
if (!weight || strcmp(weight->name, "token_embd.weight") != 0) {
|
||||
throw std::runtime_error("prism.hadamard.tied_output is not bound to the token embedding");
|
||||
}
|
||||
}
|
||||
if (weight == nullptr) {
|
||||
throw std::runtime_error(format("prism.hadamard weight not found: %s", weight_name.c_str()));
|
||||
}
|
||||
if (weight->ne[0] % block_size != 0) {
|
||||
throw std::runtime_error(format(
|
||||
"prism.hadamard block size %u does not divide input dimension %lld for %s",
|
||||
block_size, (long long) weight->ne[0], weight_name.c_str()));
|
||||
}
|
||||
if (weight->buffer == nullptr) {
|
||||
throw std::runtime_error(format("prism.hadamard weight has no buffer: %s", weight_name.c_str()));
|
||||
}
|
||||
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(weight->buffer);
|
||||
// CPU extra buffer types (e.g. CPU_REPACK) only accept tensors they can repack
|
||||
if (ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft)) {
|
||||
if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
|
||||
buft = ggml_backend_dev_buffer_type(dev);
|
||||
}
|
||||
}
|
||||
if (target == &hdmd.rot) {
|
||||
preferred_buft = buft;
|
||||
} else if (preferred_buft) {
|
||||
buft = preferred_buft;
|
||||
}
|
||||
auto it = std::find_if(rotations.begin(), rotations.end(),
|
||||
[block_size, buft](const hadamard_rotation & rotation) {
|
||||
return rotation.block_size == block_size && rotation.buft == buft;
|
||||
});
|
||||
|
||||
if (it == rotations.end()) {
|
||||
ggml_init_params params = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead(),
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ggml_context_ptr ctx { ggml_init(params) };
|
||||
if (!ctx) {
|
||||
throw std::runtime_error("failed to create Hadamard rotation context");
|
||||
}
|
||||
|
||||
ggml_tensor * rotation = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, block_size, block_size);
|
||||
char rotation_name[GGML_MAX_NAME];
|
||||
snprintf(rotation_name, sizeof(rotation_name), "prism.hadamard.%u", block_size);
|
||||
ggml_set_name(rotation, rotation_name);
|
||||
|
||||
ggml_backend_buffer_ptr buffer { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) };
|
||||
if (!buffer) {
|
||||
throw std::runtime_error(format("unable to allocate %s Hadamard rotation buffer", ggml_backend_buft_name(buft)));
|
||||
}
|
||||
ggml_backend_buffer_set_usage(buffer.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
|
||||
std::vector<float> data((size_t) block_size * block_size);
|
||||
const float scale = 1.0f / sqrtf((float) block_size);
|
||||
for (uint32_t row = 0; row < block_size; ++row) {
|
||||
for (uint32_t col = 0; col < block_size; ++col) {
|
||||
uint32_t parity = row & col;
|
||||
parity ^= parity >> 16;
|
||||
parity ^= parity >> 8;
|
||||
parity ^= parity >> 4;
|
||||
parity ^= parity >> 2;
|
||||
parity ^= parity >> 1;
|
||||
data[(size_t) row * block_size + col] = (parity & 1) ? -scale : scale;
|
||||
}
|
||||
}
|
||||
ggml_backend_tensor_set(rotation, data.data(), 0, data.size() * sizeof(float));
|
||||
|
||||
std::vector<ggml_backend_buffer_ptr> buffers;
|
||||
buffers.emplace_back(std::move(buffer));
|
||||
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
|
||||
rotations.push_back({ block_size, buft, rotation });
|
||||
it = std::prev(rotations.end());
|
||||
}
|
||||
|
||||
ggml_tensor * sign_tensor = nullptr;
|
||||
if (!hdmd.sign_data.empty()) {
|
||||
const uint32_t width = (uint32_t) weight->ne[0];
|
||||
const auto sd = hdmd.sign_data.find(width);
|
||||
if (sd == hdmd.sign_data.end()) {
|
||||
throw std::runtime_error(format(
|
||||
"prism.hadamard has no sign vector for width %u (%s)", width, weight_name.c_str()));
|
||||
}
|
||||
const auto key = std::make_pair(width, buft);
|
||||
auto st = sign_tensors.find(key);
|
||||
if (st == sign_tensors.end()) {
|
||||
ggml_init_params params = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead(),
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ggml_context_ptr ctx { ggml_init(params) };
|
||||
if (!ctx) {
|
||||
throw std::runtime_error("failed to create Hadamard sign context");
|
||||
}
|
||||
|
||||
ggml_tensor * signs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, width);
|
||||
char sign_name[GGML_MAX_NAME];
|
||||
snprintf(sign_name, sizeof(sign_name), "prism.hadamard.signs.%u", width);
|
||||
ggml_set_name(signs, sign_name);
|
||||
|
||||
ggml_backend_buffer_ptr buffer { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) };
|
||||
if (!buffer) {
|
||||
throw std::runtime_error(format("unable to allocate %s Hadamard sign buffer", ggml_backend_buft_name(buft)));
|
||||
}
|
||||
ggml_backend_buffer_set_usage(buffer.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
|
||||
std::vector<float> data(width);
|
||||
for (uint32_t i = 0; i < width; ++i) {
|
||||
data[i] = (float) sd->second[i];
|
||||
}
|
||||
ggml_backend_tensor_set(signs, data.data(), 0, data.size() * sizeof(float));
|
||||
|
||||
std::vector<ggml_backend_buffer_ptr> buffers;
|
||||
buffers.emplace_back(std::move(buffer));
|
||||
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
|
||||
st = sign_tensors.emplace(key, signs).first;
|
||||
}
|
||||
sign_tensor = st->second;
|
||||
}
|
||||
|
||||
llama_hadamard_transform transform { it->tensor, sign_tensor };
|
||||
// the GDN output projection reads its value heads in tiled order, see set_gdn_v_perm
|
||||
if (hdmd.gdn_v_grouped && weight_name.find(".ssm_out.") != std::string::npos &&
|
||||
!transform.set_gdn_v_perm(weight->ne[0], hparams.ssm_dt_rank, hparams.ssm_n_group)) {
|
||||
throw std::runtime_error(format("prism.hadamard: bad GDN head geometry for %s", weight_name.c_str()));
|
||||
}
|
||||
target->emplace(weight, transform);
|
||||
}
|
||||
}
|
||||
|
||||
LLAMA_LOG_INFO("%s: loaded %zu Hadamard-folded weight(s) (%zu inverse-lookup) using %zu rotation(s) and %zu sign vector(s)\n",
|
||||
__func__, hdmd.rot.size() + hdmd.inv.size(), hdmd.inv.size(), rotations.size(), sign_tensors.size());
|
||||
}
|
||||
|
||||
ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
|
||||
const buft_list_t * buft_list_layer = nullptr;
|
||||
if (tn.bid != -1) {
|
||||
|
||||
@@ -638,6 +638,44 @@ struct llama_prec_policy {
|
||||
void load(llama_model_loader & ml, const llama_model & model);
|
||||
};
|
||||
|
||||
// transform of a folded weight, applied to the matmul input: optional sign flip, then the normalized block Hadamard rotation
|
||||
struct llama_hadamard_transform {
|
||||
ggml_tensor * rot;
|
||||
ggml_tensor * signs; // nullptr for identity sign mode
|
||||
|
||||
// if perm_rep > 1, permute the input from tiled head order [hd, nk, rep] to grouped order [hd, rep, nk] before signs and rotation
|
||||
int64_t perm_hd = 0;
|
||||
int64_t perm_nk = 0;
|
||||
int64_t perm_rep = 0;
|
||||
|
||||
// ssm_out of a gated delta net gets its value heads in tiled order, but the fold used grouped order
|
||||
// record the head geometry for that permutation, return false if it does not match the input width
|
||||
bool set_gdn_v_perm(int64_t n_in, int64_t n_v, int64_t n_k) {
|
||||
if (n_k <= 0 || n_v <= 0 || n_v % n_k != 0 || n_in % n_v != 0) {
|
||||
return false;
|
||||
}
|
||||
perm_hd = n_in / n_v;
|
||||
perm_nk = n_k;
|
||||
perm_rep = n_v / n_k;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
using llama_hadamard_rotations = std::unordered_map<const ggml_tensor *, llama_hadamard_transform>;
|
||||
|
||||
struct llama_hadamard {
|
||||
// names and sign data come from the GGUF metadata in load_hparams, the transforms are made in load_tensors
|
||||
std::unordered_map<std::string, uint32_t> weight_blocks;
|
||||
std::unordered_map<std::string, uint32_t> inverse_blocks;
|
||||
|
||||
std::map<uint32_t, std::vector<int32_t>> sign_data;
|
||||
|
||||
bool gdn_v_grouped = false;
|
||||
bool tied_output = false;
|
||||
|
||||
llama_hadamard_rotations rot; // folded weight -> activation transform
|
||||
llama_hadamard_rotations inv; // latent lookup table -> inverse transform
|
||||
};
|
||||
|
||||
struct llama_model {
|
||||
llm_type type = LLM_TYPE_UNKNOWN;
|
||||
llm_arch arch = LLM_ARCH_UNKNOWN;
|
||||
@@ -650,6 +688,8 @@ struct llama_model {
|
||||
// per-tensor activation precision policy
|
||||
llama_prec_policy prec_policy;
|
||||
|
||||
llama_hadamard hdmd;
|
||||
|
||||
// for classifier models
|
||||
std::vector<std::string> classifier_labels;
|
||||
|
||||
@@ -857,6 +897,12 @@ struct llama_model_base : public llama_model {
|
||||
};
|
||||
nextn_flags_t nextn_flags(llama_model_loader & ml, llm_tensor trunk_probe = LLM_TENSOR_ATTN_NORM) const;
|
||||
|
||||
// helper: read the prism.hadamard metadata and record which weights are folded
|
||||
void load_hparams_hadamard(llama_model_loader & ml);
|
||||
|
||||
// helper: make the prism.hadamard rotation and sign tensors for the folded weights
|
||||
void load_tensors_hadamard();
|
||||
|
||||
// helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern
|
||||
void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false);
|
||||
|
||||
|
||||
@@ -534,6 +534,16 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr
|
||||
ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
|
||||
|
||||
tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
|
||||
|
||||
// a Hadamard-latent table stores rotated rows; restore the primal basis
|
||||
if (hdmd) {
|
||||
if (const auto it = hdmd->inv.find(tok_embd_w); it != hdmd->inv.end()) {
|
||||
tok_embd = llama_mul_mat_hadamard(ctx0, tok_embd, it->second.rot);
|
||||
if (it->second.signs) {
|
||||
tok_embd = ggml_mul(ctx0, tok_embd, it->second.signs);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tok_embd = inp->embd;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user