diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index 3f91364c5..af84ff323 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -47,6 +47,8 @@ common_chat_params peg_generator::generate_parser(const common_chat_template & data.generation_prompt = common_chat_template_generation_prompt(tmpl, inputs); data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.preserved_tokens = autoparser.preserved_tokens; + data.additional_stops.insert(data.additional_stops.end(), + autoparser.additional_stops.begin(), autoparser.additional_stops.end()); std::string parser_generation_prompt = data.generation_prompt; @@ -286,7 +288,13 @@ common_peg_parser analyze_tools::build_func_parser(common_chat_peg_builder & p, // we only emit tool_close when we can actually see the closing marker. This prevents // premature closing during partial parsing when we've seen e.g. "" (end) or "" prefix that failed to match. - func_parser = func_parser + p.tool_close(p.peek(p.literal(format.per_call_end))); + // Laguna (v4): the model may emit whitespace between the last and + // even though the template renders them tight. Tolerate optional + // leading space in the close lookahead so the tool call still closes. + auto close_peek = arguments.tolerate_intertag_whitespace + ? p.peek(p.space() + p.literal(format.per_call_end)) + : p.peek(p.literal(format.per_call_end)); + func_parser = func_parser + p.tool_close(close_peek); } else { func_parser = func_parser + p.tool_close(p.space()); // force this to process tool closing callbacks in mapper } diff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.h index d47b09dcf..074216b11 100644 --- a/common/chat-auto-parser.h +++ b/common/chat-auto-parser.h @@ -206,6 +206,7 @@ struct tool_arguments_analysis { std::string value_prefix; // e.g., "", "", "" std::string value_suffix; // e.g., "", "", "" std::string separator; // e.g., "", "\n", "," + bool tolerate_intertag_whitespace = false; // Laguna: accept optional whitespace between arg tags }; struct tool_id_analysis { @@ -388,6 +389,7 @@ struct autoparser { // Preserved tokens for tokenizer (union of all non-empty markers) std::vector preserved_tokens; + std::vector additional_stops; // literal stop strings (e.g. Laguna ) caught however tokenized autoparser() = default; diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cpp index 127278dfb..7db1dcb0f 100644 --- a/common/chat-diff-analyzer.cpp +++ b/common/chat-diff-analyzer.cpp @@ -173,6 +173,26 @@ static std::vector\n", "\n") that + // the model does not emit, so the inferred delimiters carry a spurious + // newline and never match the model output. Trim to the bare tag. (v8 + // renders without the whitespace, so this is a no-op there.) + [](const common_chat_template & tmpl, autoparser & analysis) -> void { + if (tmpl.src.find("laguna_glm_thinking") != std::string::npos) { + analysis.reasoning.start = trim_whitespace(analysis.reasoning.start); + analysis.reasoning.end = trim_whitespace(analysis.reasoning.end); + analysis.tools.arguments.value_prefix = trim_whitespace(analysis.tools.arguments.value_prefix); + analysis.tools.arguments.value_suffix = trim_whitespace(analysis.tools.arguments.value_suffix); + analysis.tools.arguments.separator = trim_whitespace(analysis.tools.arguments.separator); + analysis.tools.arguments.tolerate_intertag_whitespace = true; + // The CONTROL/eot token only halts generation when emitted as the + // single token; after tool calls the model can spell it out as text tokens. + // A literal stop string catches it either way. + analysis.additional_stops.push_back(""); + LOG_DBG(ANSI_ORANGE "[Patch: Laguna]\n" ANSI_RESET); + } + }, }); diff --git a/conversion/__init__.py b/conversion/__init__.py index 465c010f6..7936f1159 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -18,6 +18,7 @@ __all__ = [ TEXT_MODEL_MAP: dict[str, str] = { "AfmoeForCausalLM": "afmoe", + "LagunaForCausalLM": "laguna", "ApertusForCausalLM": "llama", "ArceeForCausalLM": "llama", "ArcticForCausalLM": "arctic", diff --git a/conversion/base.py b/conversion/base.py index 1b85ef0a1..051b8b4e5 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1682,6 +1682,9 @@ class TextModel(ModelBase): if chkhsh == "9dcf830ee9990cdbf78cc523a5f7bd9ad8f3f9890c2d3581d2785ad10f07049d": # ref: https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base res = "mellum2" + if chkhsh == "972da7b59cec44d1f0a490a86c96df53859e486e481563e5dddac155013d87ac": + # ref: https://huggingface.co/poolside/Laguna-XS.2 + res = "laguna" if res is None: logger.warning("\n") diff --git a/conversion/laguna.py b/conversion/laguna.py new file mode 100644 index 000000000..a90f355ca --- /dev/null +++ b/conversion/laguna.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import re +from collections.abc import Iterable +from typing import TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf, logger + + +@ModelBase.register("LagunaForCausalLM") +class LagunaModel(TextModel): + model_arch = gguf.MODEL_ARCH.LAGUNA + _experts: list[dict] | None = None + _gate_types: list[str] | None = None + + # --- vocab --------------------------------------------------------------- + + def set_vocab(self) -> None: + self._set_vocab_gpt2() + + # Some Laguna releases wrap the chat template in tokenizer_config.json as + # "{% include 'chat_template.jinja' %}", which SpecialVocab embeds verbatim + # and llama.cpp's jinja engine cannot process. Prefer the resolved template + # from the chat_template.jinja file so the GGUF is self-contained. + tmpl_file = self.dir_model / "chat_template.jinja" + if tmpl_file.is_file(): + self.gguf_writer.add_chat_template(tmpl_file.read_text(encoding="utf-8")) + logger.info("gguf: embedded resolved chat_template.jinja (overriding include directive)") + + # eos_token_id is a list [2, 24]: token 2 (EOS, also BOS) and token 24 + # (, the turn-end). _set_vocab_gpt2 only records the scalar + # eos, so register the extra id as eot; llama.cpp folds eot into its EOG + # set, so the model halts on natively. + eos_ids = self.hparams.get("eos_token_id") + if isinstance(eos_ids, list): + bos_id = self.hparams.get("bos_token_id") + extra = [e for e in eos_ids if e != bos_id] + if extra: + self.gguf_writer.add_eot_token_id(extra[0]) + logger.info(f"gguf: registered eot_token_id={extra[0]} from eos list {eos_ids}") + + def get_vocab_base(self) -> tuple[list[str], list[int], str]: + # is the assistant turn-end (registered as eot below). The + # HF tokenizer flags it special=false, so the base classifies it as + # USER_DEFINED and llama.cpp renders its text into generated content, + # leaking "" and breaking response parsing. It is a control + # marker, so promote it to CONTROL: llama.cpp then treats it as + # end-of-generation and suppresses its text. + tokens, toktypes, tokpre = super().get_vocab_base() + for i, tok in enumerate(tokens): + if tok == "": + toktypes[i] = gguf.TokenType.CONTROL + logger.info(f"gguf: marked (id {i}) as CONTROL token") + return tokens, toktypes, tokpre + + # --- hparams ------------------------------------------------------------- + + def set_gguf_parameters(self) -> None: + super().set_gguf_parameters() + hparams = self.hparams + + # super() does not emit vocab_size for the gpt2 vocab path; head_count is + # overridden with a per-layer array (XS.2 varies heads per layer via + # num_attention_heads_per_layer; M.1 is uniform and omits it). + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + + per_layer_heads = hparams.get("num_attention_heads_per_layer") + if not per_layer_heads: + per_layer_heads = [hparams["num_attention_heads"]] * hparams["num_hidden_layers"] + assert len(per_layer_heads) == hparams["num_hidden_layers"], ( + f"num_attention_heads_per_layer length {len(per_layer_heads)} != " + f"num_hidden_layers {hparams['num_hidden_layers']}" + ) + self.gguf_writer.add_head_count(per_layer_heads) + + # Resolve + validate the attention gate type now so an inconsistent + # `gating` field fails at conversion time. See _attn_gate_types. + self._attn_gate_types() + + # SWA window size (M.1 has none -> key omitted, swa_type stays NONE). + sliding_window = hparams.get("sliding_window") or 0 + if sliding_window > 0: + self.gguf_writer.add_sliding_window(sliding_window) + + # MoE (expert_count / expert_used_count come from super().set_gguf_parameters()) + self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) + self.gguf_writer.add_expert_shared_feed_forward_length(hparams["shared_expert_intermediate_size"]) + self.gguf_writer.add_expert_weights_norm(True) # HF reference always sum-normalises after top-k + self.gguf_writer.add_expert_weights_scale(float(hparams["moe_routed_scaling_factor"])) + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) + + # Leading dense layers (XS.2 has 1, M.1 has 3) before the MoE layers. + mlp_layer_types: list[str] = hparams["mlp_layer_types"] + leading_dense = 0 + for t in mlp_layer_types: + if t == "dense": + leading_dense += 1 + else: + break + self.gguf_writer.add_leading_dense_block_count(leading_dense) + + # Per-layer-type RoPE dimension count (partial rotary). base emits + # rope_freq_base(_swa) and the YaRN params from self.rope_parameters. + head_dim = hparams["head_dim"] + full_rope = self.rope_parameters["full_attention"] + self.gguf_writer.add_rope_dimension_count( + int(head_dim * float(full_rope.get("partial_rotary_factor", 1.0)))) + swa_rope = self.rope_parameters.get("sliding_attention") + if swa_rope is not None: + self.gguf_writer.add_rope_dimension_count_swa( + int(head_dim * float(swa_rope.get("partial_rotary_factor", 1.0)))) + + def _attn_gate_types(self) -> list[str]: + """Per-layer attention output gate type: "per_head" or "per_element". + + `gating_types` (per layer) is authoritative when present; otherwise the + scalar `gating` field is used (the "per-element"/"per-head" string, or + the legacy boolean True == per-head, as in Laguna-XS.2). + + Fails loudly when the model is per-element but the `gating` field does + not declare that as a string: runtimes that key off `gating` (vLLM, + transformers) ignore gating_types and read a bare boolean True as + per-head, silently corrupting the model. Surfacing it here keeps a + broken checkpoint from being packaged as if it were fine. + """ + if self._gate_types is not None: + return self._gate_types + hparams = self.hparams + n_layer = hparams["num_hidden_layers"] + gating = hparams.get("gating") + gating_types = hparams.get("gating_types") + + def _norm(t: object) -> str: + sval = str(t).replace("-", "_") + if sval in ("per_element", "per_head"): + return sval + raise ValueError(f"Laguna: unrecognised attention gate type {t!r}") + + if gating_types: + assert len(gating_types) == n_layer, ( + f"gating_types length {len(gating_types)} != num_hidden_layers {n_layer}") + types = [_norm(t) for t in gating_types] + elif isinstance(gating, str): + types = [_norm(gating)] * n_layer + elif gating is True: + types = ["per_head"] * n_layer + else: + raise ValueError( + f"Laguna: cannot determine attention gate type " + f"(gating={gating!r}, gating_types={gating_types!r})") + + if any(t == "per_element" for t in types) and not ( + isinstance(gating, str) and _norm(gating) == "per_element"): + raise ValueError( + f"Laguna config declares a per-element attention gate but " + f"`gating`={gating!r} is not the string \"per-element\". Runtimes that " + f"read `gating` (vLLM, transformers) will mis-handle this checkpoint as " + f"per-head. Set gating=\"per-element\" in the source config.") + + self._gate_types = types + return types + + # --- tensor handling ----------------------------------------------------- + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # Per-expert MoE weights: model.layers.{bid}.mlp.experts.{xid}.{w}.weight. + # Only the NUMBERED per-expert weights are stacked; the router bias + # (mlp.experts.e_score_correction_bias) takes the normal mapping path. + if re.search(r"mlp\.experts\.\d+\.", name): + n_experts = self.find_hparam(["num_local_experts", "num_experts"]) + assert bid is not None + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + self._experts[bid][name] = data_torch + needed = [f"model.layers.{bid}.mlp.experts.{x}.{w}.weight" + for x in range(n_experts) for w in ("gate_proj", "up_proj", "down_proj")] + if all(e in self._experts[bid] for e in needed): + for w_name in ["gate_proj", "up_proj", "down_proj"]: + datas = [self._experts[bid][f"model.layers.{bid}.mlp.experts.{x}.{w_name}.weight"] + for x in range(n_experts)] + stacked = torch.stack(datas, dim=0) + merged = f"model.layers.{bid}.mlp.experts.{w_name}.weight" + yield from TextModel.modify_tensors(self, stacked, merged, bid) + self._experts[bid].clear() + return + return + # Cross-check the gate projection width against the declared gate type; + # a mismatch means the weights and config disagree -> fail, do not guess. + if bid is not None and name.endswith("self_attn.g_proj.weight"): + heads = (self.hparams.get("num_attention_heads_per_layer") + or [self.hparams["num_attention_heads"]] * self.hparams["num_hidden_layers"]) + n_head = heads[bid] + head_dim = self.hparams["head_dim"] + gate_type = self._attn_gate_types()[bid] + expected = n_head * head_dim if gate_type == "per_element" else n_head + out_features = int(data_torch.shape[0]) + if out_features != expected: + raise ValueError( + f"Laguna layer {bid}: g_proj output width {out_features} contradicts the " + f"declared {gate_type} gate (expected {expected}); weights and config disagree.") + + yield from TextModel.modify_tensors(self, data_torch, name, bid) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 91c006278..e5d3196ef 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -162,6 +162,7 @@ models = [ {"name": "granite-embed-multi-97m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-97m-multilingual-r2", }, {"name": "granite-embed-multi-311m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-311m-multilingual-r2", }, {"name": "mellum2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base"}, + {"name": "laguna", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/poolside/Laguna-XS.2", }, ] # some models are known to be broken upstream, so we will skip them as exceptions diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 63ac2ed1f..4391bd11d 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -507,6 +507,7 @@ class MODEL_ARCH(IntEnum): DOTS1 = auto() ARCEE = auto() AFMOE = auto() + LAGUNA = auto() ERNIE4_5 = auto() ERNIE4_5_MOE = auto() HUNYUAN_MOE = auto() @@ -1088,6 +1089,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.DOTS1: "dots1", MODEL_ARCH.ARCEE: "arcee", MODEL_ARCH.AFMOE: "afmoe", + MODEL_ARCH.LAGUNA: "laguna", MODEL_ARCH.ERNIE4_5: "ernie4_5", MODEL_ARCH.ERNIE4_5_MOE: "ernie4_5-moe", MODEL_ARCH.FALCON_H1: "falcon-h1", @@ -3823,6 +3825,31 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_POST_NORM, MODEL_TENSOR.FFN_EXP_PROBS_B, ], + MODEL_ARCH.LAGUNA: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + ], MODEL_ARCH.ERNIE4_5: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 9efb36f8a..b5707f11f 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -479,6 +479,7 @@ class TensorNameMap: "model.layers.{bid}.mlp.e_score_correction", # exaone-moe "model.layers.{bid}.block_sparse_moe.gate.e_score_correction", # kimi "model.layers.{bid}.moe.router_bias", # step3.5 expert selection bias + "model.layers.{bid}.mlp.experts.e_score_correction", # laguna ), # Feed-forward up diff --git a/models/templates/poolside-Laguna-S-2.1.jinja b/models/templates/poolside-Laguna-S-2.1.jinja new file mode 100644 index 000000000..75c5f4cec --- /dev/null +++ b/models/templates/poolside-Laguna-S-2.1.jinja @@ -0,0 +1,93 @@ +{#- Iteration on laguna_glm_thinking_v8/chat_template.jinja -#} +{#- No formatting instructions -#} +{{- "〈|EOS|〉" -}} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set add_generation_prompt = add_generation_prompt | default(false) -%} + +{#- ───── header (system message) ───── -#} +{#- A caller-supplied system message with empty content opts out of the default below, producing no block — used to train without a system message. -#} +{%- set system_message = "You are a helpful, conversationally-fluent assistant made by Poolside. You are here to be helpful to users through natural language conversations." -%} +{%- if messages and messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} + {%- set messages = messages[1:] -%} +{%- endif -%} + +{%- set has_sys = system_message and system_message.strip() -%} +{%- if has_sys or tools or enable_thinking -%} + {{- "" -}} + + {%- if has_sys -%} + {{- system_message.rstrip() -}} + {%- if tools -%}{{- "\n\n" -}}{%- endif -%} + {%- endif -%} + + {%- if tools -%} + {{- "### Tools\n\n" -}} + {{- "You may call functions to assist with the user query.\n" -}} + {{- "All available function signatures are listed below:\n" -}} + {{- "\n" -}} + {%- for tool in tools -%} + {{- (tool | tojson) ~ "\n" -}} + {%- endfor -%} + {{- "" -}} + {%- endif -%} + + {{- "\n" -}} +{%- endif -%} + +{#- ───── main loop ───── -#} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{- "" + content + "\n" -}} + {%- elif message.role == "assistant" -%} + {%- generation -%} + {{- "" -}} + {#- Extract reasoning content from message.reasoning (vLLM field name) or message.reasoning_content -#} + {%- set reasoning_content = '' -%} + {%- if message.reasoning is string -%} + {%- set reasoning_content = message.reasoning -%} + {%- elif message.reasoning_content is string -%} + {%- set reasoning_content = message.reasoning_content -%} + {%- endif -%} + {#- Display reasoning content for all messages if enable_thinking -#} + {%- if enable_thinking -%} + {{- '' + reasoning_content + '' -}} + {%- else -%} + {{- '' -}} + {%- endif -%} + {#- Display main content (trailing newline only when no tool_calls follow) -#} + {%- if content -%} + {{- content -}} + {%- endif -%} + {%- if message.tool_calls -%} + {%- for tool_call in message.tool_calls -%} + {%- set function_data = tool_call.function -%} + {{- '' + function_data.name -}} + {%- set _args = function_data.arguments -%} + {%- for k, v in _args.items() -%} + {{- "" ~ k ~ "" -}} + {{- "" -}}{{- v | tojson(ensure_ascii=False) if v is not string else v -}}{{- "" -}} + {%- endfor -%} + {{- "" -}} + {%- endfor -%} + {%- endif -%} + {{- "\n" -}} + {%- endgeneration -%} + {%- elif message.role == "tool" -%} + {{- "" + content + "\n" -}} + {%- elif message.role == "system" -%} + {#- Render additional system messages (the first one, if any, is handled separately in the header and was sliced off above) -#} + {{- "" + content + "\n" -}} + {%- endif -%} +{%- endfor -%} +{#- ───── generation prompt ───── -#} +{%- if add_generation_prompt -%} + {{- "" -}} + {#- ───── Include reasoning mode directive ───── -#} + {%- if enable_thinking -%} + {{- '' -}} + {%- else -%} + {{- '' -}} + {%- endif -%} +{%- endif -%} \ No newline at end of file diff --git a/models/templates/poolside-Laguna-XS-2.1.jinja b/models/templates/poolside-Laguna-XS-2.1.jinja new file mode 100644 index 000000000..d45f23f70 --- /dev/null +++ b/models/templates/poolside-Laguna-XS-2.1.jinja @@ -0,0 +1,132 @@ +{#- Copied from laguna_glm_thinking_v4/chat_template.jinja -#} +{#- Removes prefix that references token, and replaces message.reasoning_content reference with message.reasoning -#} +{{- "〈|EOS|〉" -}} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set render_assistant_messages_raw = render_assistant_messages_raw | default(false) -%} +{%- set add_generation_prompt = add_generation_prompt | default(false) -%} + +{#- ───── header (system message) ───── -#} +{%- set system_message = "" -%} +{%- if messages and messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} +{%- endif -%} + +{%- if (system_message and system_message.strip()) or tools -%} + {{- "\n" -}} + + {%- if system_message and system_message.strip() -%} + {{- "\n" -}} + {{- system_message.rstrip() -}} + {%- endif -%} + + {%- if tools -%} + {{- "\n\n### Tools\n\n" -}} + {%- set ns = namespace(tool_string="You may call functions to assist with the user query.\n" + ~ "All available function signatures are listed below:\n" + ~ "\n") -%} + {%- for tool in tools -%} + {%- set ns.tool_string = ns.tool_string ~ (tool | tojson) ~ "\n" -%} + {%- endfor -%} + {%- if enable_thinking -%} + {%- set tool_string = ns.tool_string + "\n\n" ~ + "Wrap your thinking in '', '' tags, followed by a function call. For each function call, return an unescaped XML-like object with function name and arguments within '' and '' tags, like here:\n" ~ + " your thoughts here \n" ~ + "function-name\nargument-key\nvalue-of-argument-key\n" ~ + "" -%} + {%- else -%} + {%- set tool_string = ns.tool_string + "\n\n" ~ + "For each function call, return an unescaped XML-like object " ~ + "with function name and arguments within '' and '' tags, like here:\n" ~ + "function-name\nargument-key\nvalue-of-argument-key\n" ~ + "" -%} + {%- endif -%} + {{- tool_string -}} + {%- endif -%} + + {{- "\n\n" -}} +{%- endif -%} + +{#- ───── main loop ───── -#} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{- "\n" + content + "\n\n" -}} + {%- elif message.role == "assistant" -%} + {%- generation -%} + {{- "\n" -}} + {%- if render_assistant_messages_raw -%} + {#- Raw mode: prepend the generation prompt token, then dump content verbatim. -#} + {#- The generation prompt is when enable_thinking, otherwise. -#} + {#- Only prepend if content doesn't already start with it. -#} + {%- if enable_thinking -%} + {%- if not content.startswith('') -%} + {{- '' -}} + {%- endif -%} + {%- else -%} + {%- if not content.startswith('') -%} + {{- '' -}} + {%- endif -%} + {%- endif -%} + {{- content -}} + {#- Append closing tag if content doesn't already end with it. -#} + {%- if not content.endswith('\n') and not content.endswith('') -%} + {{- '\n' -}} + {%- endif -%} + {{- "\n" -}} + {%- else -%} + {#- Extract reasoning content from message.reasoning (vLLM field name) or message.reasoning_content, or from tags -#} + {%- set reasoning_content = '' %} + {%- if message.reasoning is string %} + {%- set reasoning_content = message.reasoning %} + {%- elif message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {#- Always strip tags from content if present to avoid duplication -#} + {%- if '' in content %} + {%- if not reasoning_content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {#- Display reasoning content for all messages -#} + {%- if reasoning_content -%} + {{- '\n' + reasoning_content.strip() + '\n\n' -}} + {%- else -%} + {{- '\n' -}} + {%- endif -%} + {#- Display main content -#} + {%- if content.strip() -%} + {{- content.strip() ~ "\n" -}} + {%- endif -%} + {%- if message.tool_calls -%} + {%- for tool_call in message.tool_calls -%} + {%- set function_data = tool_call.function -%} + {{- '' + function_data.name }} + {% set _args = function_data.arguments %} + {%- for k, v in _args.items() -%} + {{- "" ~ k ~ "\n" -}} + {{- ""}}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{{ "\n" -}} + {%- endfor -%} + {{- "\n" -}} + {%- endfor -%} + {%- endif -%} + {{- "\n" -}} + {%- endif -%} + {%- endgeneration -%} + {%- elif message.role == "tool" -%} + {{- "\n" + content + "\n\n" -}} + {%- elif message.role == "system" and loop.index0 != 0 -%} + {#- Render additional system messages (skip the first one which is handled separately in the header) -#} + {{- "\n" + content + "\n\n" -}} + {%- endif -%} +{%- endfor -%} +{#- ───── generation prompt ───── -#} +{%- if add_generation_prompt -%} + {{- "\n" -}} + {#- ───── Include reasoning mode directive ───── -#} + {%- if not enable_thinking %} + {{- '' -}} + {%- else %} + {{- '' -}} + {%- endif %} +{%- endif -%} diff --git a/models/templates/poolside-Laguna-XS.2.jinja b/models/templates/poolside-Laguna-XS.2.jinja new file mode 100644 index 000000000..4baa3fded --- /dev/null +++ b/models/templates/poolside-Laguna-XS.2.jinja @@ -0,0 +1,132 @@ +{#- Iteration on laguna_glm_thinking_v5/chat_template.jinja -#} +{#- Adds a default system message (used when no system message is provided in `messages`). -#} +{{- "〈|EOS|〉" -}} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set render_assistant_messages_raw = render_assistant_messages_raw | default(false) -%} +{%- set add_generation_prompt = add_generation_prompt | default(false) -%} + +{#- ───── header (system message) ───── -#} +{%- set system_message = "You are a helpful, conversationally-fluent assistant made by Poolside. You are here to be helpful to users through natural language conversations." -%} +{%- if messages and messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} +{%- endif -%} + +{%- if (system_message and system_message.strip()) or tools -%} + {{- "\n" -}} + + {%- if system_message and system_message.strip() -%} + {{- "\n" -}} + {{- system_message.rstrip() -}} + {%- endif -%} + + {%- if tools -%} + {{- "\n\n### Tools\n\n" -}} + {%- set ns = namespace(tool_string="You may call functions to assist with the user query.\n" + ~ "All available function signatures are listed below:\n" + ~ "\n") -%} + {%- for tool in tools -%} + {%- set ns.tool_string = ns.tool_string ~ (tool | tojson) ~ "\n" -%} + {%- endfor -%} + {%- if enable_thinking -%} + {%- set tool_string = ns.tool_string + "\n\n" ~ + "Wrap your thinking in '', '' tags, followed by a function call. For each function call, return an unescaped XML-like object with function name and arguments within '' and '' tags, like here:\n" ~ + " your thoughts here \n" ~ + "function-name\nargument-key\nvalue-of-argument-key\n" ~ + "" -%} + {%- else -%} + {%- set tool_string = ns.tool_string + "\n\n" ~ + "For each function call, return an unescaped XML-like object " ~ + "with function name and arguments within '' and '' tags, like here:\n" ~ + "function-name\nargument-key\nvalue-of-argument-key\n" ~ + "" -%} + {%- endif -%} + {{- tool_string -}} + {%- endif -%} + + {{- "\n\n" -}} +{%- endif -%} + +{#- ───── main loop ───── -#} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{- "\n" + content + "\n\n" -}} + {%- elif message.role == "assistant" -%} + {%- generation -%} + {{- "\n" -}} + {%- if render_assistant_messages_raw -%} + {#- Raw mode: prepend the generation prompt token, then dump content verbatim. -#} + {#- The generation prompt is when enable_thinking, otherwise. -#} + {#- Only prepend if content doesn't already start with it. -#} + {%- if enable_thinking -%} + {%- if not content.startswith('') -%} + {{- '' -}} + {%- endif -%} + {%- else -%} + {%- if not content.startswith('') -%} + {{- '' -}} + {%- endif -%} + {%- endif -%} + {{- content -}} + {#- Append closing tag if content doesn't already end with it. -#} + {%- if not content.endswith('\n') and not content.endswith('') -%} + {{- '\n' -}} + {%- endif -%} + {{- "\n" -}} + {%- else -%} + {#- Extract reasoning content from message.reasoning (vLLM field name) or message.reasoning_content, or from tags -#} + {%- set reasoning_content = '' %} + {%- if message.reasoning is string %} + {%- set reasoning_content = message.reasoning %} + {%- elif message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {#- Always strip tags from content if present to avoid duplication -#} + {%- if '' in content %} + {%- if not reasoning_content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {#- Display reasoning content for all messages -#} + {%- if reasoning_content -%} + {{- '\n' + reasoning_content.strip() + '\n\n' -}} + {%- else -%} + {{- '\n' -}} + {%- endif -%} + {#- Display main content -#} + {%- if content.strip() -%} + {{- content.strip() ~ "\n" -}} + {%- endif -%} + {%- if message.tool_calls -%} + {%- for tool_call in message.tool_calls -%} + {%- set function_data = tool_call.function -%} + {{- '' + function_data.name }} + {% set _args = function_data.arguments %} + {%- for k, v in _args.items() -%} + {{- "" ~ k ~ "\n" -}} + {{- ""}}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{{ "\n" -}} + {%- endfor -%} + {{- "\n" -}} + {%- endfor -%} + {%- endif -%} + {{- "\n" -}} + {%- endif -%} + {%- endgeneration -%} + {%- elif message.role == "tool" -%} + {{- "\n" + content + "\n\n" -}} + {%- elif message.role == "system" and loop.index0 != 0 -%} + {#- Render additional system messages (skip the first one which is handled separately in the header) -#} + {{- "\n" + content + "\n\n" -}} + {%- endif -%} +{%- endfor -%} +{#- ───── generation prompt ───── -#} +{%- if add_generation_prompt -%} + {{- "\n" -}} + {#- ───── Include reasoning mode directive ───── -#} + {%- if not enable_thinking %} + {{- '' -}} + {%- else %} + {{- '' -}} + {%- endif %} +{%- endif -%} diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 72968607d..89fa097fe 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -108,6 +108,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_DOTS1, "dots1" }, { LLM_ARCH_ARCEE, "arcee" }, { LLM_ARCH_AFMOE, "afmoe" }, + { LLM_ARCH_LAGUNA, "laguna" }, { LLM_ARCH_ERNIE4_5, "ernie4_5" }, { LLM_ARCH_ERNIE4_5_MOE, "ernie4_5-moe" }, { LLM_ARCH_HUNYUAN_MOE, "hunyuan-moe" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index b74d53af4..0ef95b01b 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -113,6 +113,7 @@ enum llm_arch { LLM_ARCH_DOTS1, LLM_ARCH_ARCEE, LLM_ARCH_AFMOE, + LLM_ARCH_LAGUNA, LLM_ARCH_ERNIE4_5, LLM_ARCH_ERNIE4_5_MOE, LLM_ARCH_HUNYUAN_MOE, diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index a3928523b..867e1fe4d 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -28,6 +28,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: case LLM_ARCH_MELLUM: + case LLM_ARCH_LAGUNA: return false; default: return true; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 4c10e4126..7e5bab26c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -250,6 +250,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_arcee(params); case LLM_ARCH_AFMOE: return new llama_model_afmoe(params); + case LLM_ARCH_LAGUNA: + return new llama_model_laguna(params); case LLM_ARCH_ERNIE4_5: return new llama_model_ernie4_5(params); case LLM_ARCH_ERNIE4_5_MOE: @@ -2549,6 +2551,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_COGVLM: case LLM_ARCH_PANGU_EMBED: case LLM_ARCH_AFMOE: + case LLM_ARCH_LAGUNA: case LLM_ARCH_QWEN3NEXT: case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index fdd447147..d926eee6e 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -496,6 +496,12 @@ struct llm_tokenizer_bpe : llm_tokenizer { "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\\r\\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", }; break; + case LLAMA_VOCAB_PRE_TYPE_LAGUNA: + regex_exprs = { + "[^\\n]+|[\\n]+", + "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + break; case LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE: regex_exprs = { // original regex from tokenizer.json @@ -2342,6 +2348,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "afmoe") { pre_type = LLAMA_VOCAB_PRE_TYPE_AFMOE; clean_spaces = false; + } else if ( + tokenizer_pre == "laguna") { + pre_type = LLAMA_VOCAB_PRE_TYPE_LAGUNA; + clean_spaces = false; } else if ( tokenizer_pre == "minimax-m2") { pre_type = LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2; diff --git a/src/llama-vocab.h b/src/llama-vocab.h index 707cd4bac..b7c289263 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -64,6 +64,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54, LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55, + LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, }; struct LLM_KV; diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp new file mode 100644 index 000000000..fb55ec12f --- /dev/null +++ b/src/models/laguna.cpp @@ -0,0 +1,332 @@ +// Laguna (poolside): sigmoid-routed MoE with a score-correction bias, one shared +// expert, a softplus attention output gate, QK-norm, and per-layer-type RoPE +// (YaRN on full-attention layers, plain RoPE on sliding-window layers). XS.2 is +// hybrid full/SWA with a per-head gate; M.1 is full-attention with a per-element +// gate. Shares the MoE/gate structure with afmoe. + +#include "models.h" + +void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + + // Laguna ships one shared expert and stores its size directly (routed and + // shared experts may differ), so read the size from expert_shared_feed_forward_length. + // The count is not in the config; default to 1 but read the key if present. + hparams.n_expert_shared = 1; + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); + if (hparams.n_ff_shexp == 0) { + // Weightless fixtures (test-llama-archs) omit this key; derive a nonzero + // size so the shared expert is still built. Real GGUFs always carry the + // exact value (routed and shared FF lengths may differ). + hparams.n_ff_shexp = hparams.n_ff_exp * hparams.n_expert_shared; + } + + // Sliding-window attention is OPTIONAL. XS.2 is hybrid (full / SWA / SWA / + // SWA repeating, period 4 starting with full); M.1 has no sliding window + // (all layers full attention). When sliding_window is absent or zero we + // leave swa_type = NONE and skip the SWA-specific per-layer-type RoPE. + hparams.n_swa = 0; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); + if (hparams.n_swa > 0) { + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + + uint32_t swa_period = 4; + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); + hparams.set_swa_pattern(swa_period, /*dense_first=*/true); // XS.2: FULL at il%4==0 + + // Per-layer-type RoPE: full layers use YaRN θ=500000 over 64 dims; + // SWA layers use default RoPE θ=10000 over 128 dims. Base load_hparams + // already reads ROPE_FREQ_BASE and ROPE_DIMENSION_COUNT into the + // non-SWA fields; we explicitly pull the SWA mirrors here. + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = 1.0f; // SWA uses plain RoPE (no YaRN scaling); do NOT inherit full layers 1/factor + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT_SWA, hparams.n_rot_swa, false); + } + + // Default the expert gating function to SIGMOID when the key is absent + // (matches the HF reference). + if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { + hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; + } + + switch (hparams.n_layer()) { + case 40: type = LLM_TYPE_30B_A3B; break; // Laguna-XS.2 + case 70: type = LLM_TYPE_230B_A10B; break; // Laguna-M.1 + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + // tied embeddings fallback + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_shexp = hparams.n_ff_shexp; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + // Per-layer head count — Laguna varies n_head between full and SWA + // layers (48 vs 64 in XS.2). KV head count is uniform. + const int64_t n_head_il = hparams.n_head(i); + const int64_t n_head_kv_il = hparams.n_head_kv(i); + const int64_t n_embd_q_il = n_embd_head_k * n_head_il; + const int64_t n_embd_k_il = n_embd_head_k * n_head_kv_il; + const int64_t n_embd_v_il = n_embd_head_v * n_head_kv_il; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_q_il, n_embd_k_il, n_embd_v_il, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q_il, n_embd}, 0); + + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + + // Attention output gate. XS.2 is per-head (g_proj -> n_head, one scalar + // per head broadcast over head_dim at multiply time); M.1 is per-element + // (g_proj -> n_head*head_dim, like afmoe). Detect from the stored tensor + // shape so a single arch handles both; the graph mirrors this check. + // Gate width selects per-head vs per-element. Real GGUFs always carry the + // gate tensor, so read the width from it and require EXACTLY one of the two + // valid widths -- never guess between them. Weightless fixtures + // (test-llama-archs) have no gate tensor; fall back to the per-head layout so + // the per-head reshape path is still exercised. + const int64_t n_gate_per_head = n_head_il; + const int64_t n_gate_per_elem = n_embd_head_k * n_head_il; + const ggml_tensor * gate_meta = ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_GATE, "weight", i).str().c_str()); + int64_t n_gate_out; + if (gate_meta != nullptr) { + n_gate_out = gate_meta->ne[1]; + if (n_gate_out != n_gate_per_head && n_gate_out != n_gate_per_elem) { + GGML_ABORT("Laguna: unexpected attention gate width %lld at layer %d " + "(expected %lld per-head or %lld per-element)", + (long long) n_gate_out, i, (long long) n_gate_per_head, (long long) n_gate_per_elem); + } + } else { + n_gate_out = n_gate_per_head; + } + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_gate_out}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + if ((uint32_t)i >= hparams.n_layer_dense_lead) { + // MoE layer + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + + // Always-on shared expert. + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0); + } else { + // Dense layer (the leading n_layer_dense_lead layers) + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + } + } +} + +std::unique_ptr llama_model_laguna::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + // No MuP embedding scale (laguna omits this; afmoe scales by sqrt(hidden)). + + ggml_tensor * inp_pos = build_inp_pos(); + // XS.2 is hybrid SWA -> interleaved-SWA KV input; M.1 is all-full -> plain + // KV input. Pick the matching input (and build_attn overload) per swa_type. + const bool has_swa = hparams.swa_type != LLAMA_SWA_TYPE_NONE; + llm_graph_input_attn_kv * inp_attn_kv = has_swa ? nullptr : build_attn_inp_kv(); + llm_graph_input_attn_kv_iswa * inp_attn_iswa = has_swa ? build_attn_inp_kv_iswa() : nullptr; + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + for (int il = 0; il < n_layer; ++il) { + const bool is_swa_il = hparams.is_swa(il); + const int64_t n_head_il = hparams.n_head(il); + const int64_t n_head_kv_il = hparams.n_head_kv(il); + + // Per-layer-type RoPE config. SWA layers run plain rope (no YaRN), + // achieved by zeroing the YaRN ext/beta params for those layers. + const int n_rot_l = is_swa_il ? hparams.n_rot_swa : n_rot; + const float freq_base_l = is_swa_il ? hparams.rope_freq_base_train_swa : freq_base; + const float freq_scale_l = is_swa_il ? hparams.rope_freq_scale_train_swa : freq_scale; + const float ext_factor_l = is_swa_il ? 0.0f : ext_factor; + // YaRN magnitude scaling (mscale) is already handled by the framework: + // llama_context pre-divides cparams.yarn_attn_factor by (1 + 0.1*ln(factor)) + // to cancel ggml rope_yarn's internal mscale *= 1 + 0.1*ln(1/freq_scale). + // Pass attn_factor straight through (like every other arch); SWA layers run + // plain RoPE (ext_factor 0, no mscale) so force 1.0 there. + const float attn_factor_l = is_swa_il ? 1.0f : attn_factor; + const float beta_fast_l = is_swa_il ? 0.0f : beta_fast; + const float beta_slow_l = is_swa_il ? 0.0f : beta_slow; + const int n_ctx_orig_l = is_swa_il ? hparams.n_ctx_train : n_ctx_orig; + + ggml_tensor * inpSA = inpL; + + // Pre-norm + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // Self-attention + { + ggml_tensor * attn_inp = cur; // saved for the gate projection + + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head_il, n_head_kv_il, il); + + // g_proj on the *pre-attention* hidden state (matches HF + // reference: gate is computed from the same `hidden_states` + // input as q/k/v, not from the attn output). + ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp); + cb(gate, "attn_gate_proj", il); + + // QK RMSNorm at head_dim level (Qwen3 style) + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + cb(Kcur, "Kcur_normed", il); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, + n_rot_l, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, + ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, + n_rot_l, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, + ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + cb(Qcur, "Qcur_rope", il); + cb(Kcur, "Kcur_rope", il); + + cur = has_swa + ? build_attn(inp_attn_iswa, + NULL, NULL, NULL, // o_proj deferred until after gating + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il) + : build_attn(inp_attn_kv, + NULL, NULL, NULL, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + + // Softplus output gate (the unary kernel computes softplus in fp32 + // and casts back). Two shapes, distinguished by the g_proj output + // dim (matching the load-time detection): + // XS.2 per-head : gate [n_head_il, n_tokens] -> reshape to + // [1, n_head_il, n_tokens] and broadcast over + // head_dim against cur [head_dim, n_head, T]. + // M.1 per-element : gate [n_head_il*head_dim, n_tokens] spans the + // full attention output -> direct ggml_mul. + gate = ggml_softplus(ctx0, gate); + cb(gate, "attn_gate_softplus", il); + + const int64_t n_tokens = cur->ne[1]; + if (model.layers[il].wqkv_gate->ne[1] == n_head_il) { + cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head_il, n_tokens); + gate = ggml_reshape_3d(ctx0, gate, 1, n_head_il, n_tokens); + cur = ggml_mul(ctx0, cur, gate); + cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head_il, n_tokens); + } else { + cur = ggml_mul(ctx0, cur, gate); + } + cb(cur, "attn_gated", il); + + cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); + cb(cur, "attn_o_proj", il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // Pre-norm only (no post-attn norm) + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t)il >= hparams.n_layer_dense_lead) { + // MoE: sigmoid routing + score-correction bias + sum-norm + + // routed_scaling_factor (all handled by build_moe_ffn). + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, + hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "ffn_moe_out", il); + + // Always-on shared expert, summed in parallel. + ggml_tensor * ffn_shexp = build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } else { + // Dense FFN for the leading n_layer_dense_lead layers (XS.2: 1, M.1: 3) + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } + + // No post-ffn norm + cur = ggml_add(ctx0, cur, ffn_inp); + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/models.h b/src/models/models.h index a86ae05aa..916b83929 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1681,6 +1681,19 @@ struct llama_model_afmoe : public llama_model_base { }; +struct llama_model_laguna : public llama_model_base { + llama_model_laguna(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_ernie4_5 : public llama_model_base { llama_model_ernie4_5(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 78e42c65a..4218f8d57 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -57,6 +57,15 @@ static void test_seed_oss_tool_with_reasoning(testing & t); static void test_nemotron_analysis(testing & t); static void test_nemotron_reasoning_detection(testing & t); static void test_nemotron_tool_format(testing & t); +static void test_laguna_analysis(testing & t); +static void test_laguna_reasoning_detection(testing & t); +static void test_laguna_tool_format(testing & t); +static void test_laguna_s_analysis(testing & t); +static void test_laguna_s_reasoning_detection(testing & t); +static void test_laguna_s_tool_format(testing & t); +static void test_laguna_xs2_analysis(testing & t); +static void test_laguna_xs2_reasoning_detection(testing & t); +static void test_laguna_xs2_tool_format(testing & t); // CohereForAI template analysis tests static void test_cohere_reasoning_detection(testing & t); @@ -101,6 +110,9 @@ int main(int argc, char * argv[]) { t.test("seed_oss_diffs", test_seed_oss_tool_analysis); t.test("cohere", test_cohere_analysis); t.test("nemotron", test_nemotron_analysis); + t.test("laguna", test_laguna_analysis); + t.test("laguna-s", test_laguna_s_analysis); + t.test("laguna-xs2", test_laguna_xs2_analysis); t.test("smollm3", test_smollm3_analysis); t.test("standard_json_tools", test_standard_json_tools_formats); t.test("normalize_quotes_to_json", test_normalize_quotes_to_json); @@ -1378,6 +1390,94 @@ static void test_nemotron_tool_format(testing & t) { t.assert_true("should support tools", analysis.jinja_caps.supports_tools); } +// ============================================================================ +// Laguna Template Analysis Tests +// ============================================================================ +static common_chat_template load_laguna_template(testing & t) { + return load_template(t, "models/templates/poolside-Laguna-XS-2.1.jinja"); +} + +static void test_laguna_reasoning_detection(testing & t) { + common_chat_template tmpl = load_laguna_template(t); + struct autoparser analysis; + analysis.analyze_template(tmpl); + // Laguna's template renders reasoning delimiters with formatting whitespace + // ("\n") that the model does not emit; the Laguna patch trims them. + t.assert_equal("reasoning_start should be ''", "", analysis.reasoning.start); + t.assert_equal("reasoning_end should be ''", "", analysis.reasoning.end); + t.assert_equal("reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode); +} + +static void test_laguna_tool_format(testing & t) { + common_chat_template tmpl = load_laguna_template(t); + struct autoparser analysis; + analysis.analyze_template(tmpl); + t.assert_equal("arg_value_suffix should be ''", "", analysis.tools.arguments.value_suffix); +} + +static void test_laguna_stop_string(testing & t) { + // The turn terminator can be emitted as ordinary text tokens + // (not the single eot token), so it must also be a literal stop string. + common_chat_template tmpl = load_laguna_template(t); + struct autoparser analysis; + analysis.analyze_template(tmpl); + bool has_stop = false; + for (const auto & stop : analysis.additional_stops) { + if (stop == "") { has_stop = true; break; } + } + t.assert_true("Laguna additional_stops contains ", has_stop); +} + +static void test_laguna_analysis(testing & t) { + t.test("Laguna reasoning detection", test_laguna_reasoning_detection); + t.test("Laguna tool format", test_laguna_tool_format); + t.test("Laguna stop string", test_laguna_stop_string); +} + +static common_chat_template load_laguna_s_template(testing & t) { + return load_template(t, "models/templates/poolside-Laguna-S-2.1.jinja"); +} +static void test_laguna_s_reasoning_detection(testing & t) { + common_chat_template tmpl = load_laguna_s_template(t); + struct autoparser analysis; + analysis.analyze_template(tmpl); + t.assert_equal("Laguna-S(v8) reasoning_start should be ''", "", analysis.reasoning.start); + t.assert_equal("Laguna-S(v8) reasoning_end should be ''", "", analysis.reasoning.end); + t.assert_equal("Laguna-S(v8) reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode); +} +static void test_laguna_s_tool_format(testing & t) { + common_chat_template tmpl = load_laguna_s_template(t); + struct autoparser analysis; + analysis.analyze_template(tmpl); + t.assert_equal("Laguna-S(v8) arg_value_suffix should be ''", "", analysis.tools.arguments.value_suffix); +} +static void test_laguna_s_analysis(testing & t) { + t.test("Laguna-S(v8) reasoning detection", test_laguna_s_reasoning_detection); + t.test("Laguna-S(v8) tool format", test_laguna_s_tool_format); +} + +static common_chat_template load_laguna_xs2_template(testing & t) { + return load_template(t, "models/templates/poolside-Laguna-XS.2.jinja"); +} +static void test_laguna_xs2_reasoning_detection(testing & t) { + common_chat_template tmpl = load_laguna_xs2_template(t); + struct autoparser analysis; + analysis.analyze_template(tmpl); + t.assert_equal("Laguna-XS.2(v5) reasoning_start should be ''", "", analysis.reasoning.start); + t.assert_equal("Laguna-XS.2(v5) reasoning_end should be ''", "", analysis.reasoning.end); + t.assert_equal("Laguna-XS.2(v5) reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode); +} +static void test_laguna_xs2_tool_format(testing & t) { + common_chat_template tmpl = load_laguna_xs2_template(t); + struct autoparser analysis; + analysis.analyze_template(tmpl); + t.assert_equal("Laguna-XS.2(v5) arg_value_suffix should be ''", "", analysis.tools.arguments.value_suffix); +} +static void test_laguna_xs2_analysis(testing & t) { + t.test("Laguna-XS.2(v5) reasoning detection", test_laguna_xs2_reasoning_detection); + t.test("Laguna-XS.2(v5) tool format", test_laguna_xs2_tool_format); +} + static common_chat_template load_cohere_template(testing & t) { return load_template(t, "models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja"); } diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 57d33a627..86c3051c5 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -362,6 +362,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_STEP35: case LLM_ARCH_MISTRAL4: case LLM_ARCH_MELLUM: + case LLM_ARCH_LAGUNA: return true; default: return false;