diff --git a/conversion/__init__.py b/conversion/__init__.py index 94d6a49fb..4d58bcd10 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -255,6 +255,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "SeedOssForCausalLM": "olmo", "SmallThinkerForCausalLM": "smallthinker", "SmolLM3ForCausalLM": "llama", + "Spark2_5ForCausalLM": "spark2_5", "SolarOpenForCausalLM": "glm", "StableLMEpochForCausalLM": "stablelm", "StableLmForCausalLM": "stablelm", diff --git a/conversion/base.py b/conversion/base.py index c1ecf1c65..dc1083ead 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1543,6 +1543,9 @@ class TextModel(ModelBase): if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7": # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B res = "lfm2" + if chkhsh == "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed": + # ref: https://huggingface.co/XHToken/Spark-X2.5-1.7B + res = "spark2_5" if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B res = "llama-bpe" diff --git a/conversion/spark2_5.py b/conversion/spark2_5.py new file mode 100644 index 000000000..44a0bd262 --- /dev/null +++ b/conversion/spark2_5.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from collections.abc import Iterable +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf + + +@ModelBase.register("Spark2_5ForCausalLM") +@ModelBase.example("XHToken/Spark-X2.5-1.7B") +class Spark2_5Model(TextModel): + model_arch = gguf.MODEL_ARCH.SPARK2_5 + + def set_gguf_parameters(self) -> None: + super().set_gguf_parameters() + + hparams = self.hparams + layer_types = hparams["layer_types"] + if len(layer_types) != self.block_count: + raise ValueError( + f"Spark2_5 layer_types length {len(layer_types)} != num_hidden_layers {self.block_count}" + ) + if any(layer_type not in ("sliding_attention", "full_attention") for layer_type in layer_types): + raise ValueError(f"Spark2_5 has unsupported layer_types: {layer_types}") + if hparams.get("gate_attn_act_mode") != "sigmoid" or hparams.get("headwise_attn_output_gate") is not True: + raise ValueError("Spark2_5 conversion requires head-wise sigmoid attention gates") + if hparams.get("hidden_act") != "gelu": + raise ValueError(f"Spark2_5 conversion requires GELU, got {hparams.get('hidden_act')!r}") + + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern( + [layer_type == "sliding_attention" for layer_type in layer_types] + ) + + head_dim = hparams["head_dim"] + full_rope = self.rope_parameters["full_attention"] + swa_rope = self.rope_parameters["sliding_attention"] + self.gguf_writer.add_rope_dimension_count( + int(head_dim * float(full_rope["partial_rotary_factor"])) + ) + self.gguf_writer.add_rope_dimension_count_swa( + int(head_dim * float(swa_rope["partial_rotary_factor"])) + ) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name.endswith(".self_attn.q_k_v_proj.weight"): + if bid is None: + raise ValueError(f"Spark2_5 fused QKV tensor has no block id: {name}") + yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid), data_torch + return + + if name.endswith(".self_attn.g_proj.weight"): + if bid is None: + raise ValueError(f"Spark2_5 attention gate tensor has no block id: {name}") + expected = self.hparams["num_attention_heads"] + if data_torch.shape[0] != expected: + raise ValueError( + f"Spark2_5 layer {bid} attention gate width {data_torch.shape[0]} != head count {expected}" + ) + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index c4141afa6..6af74cd87 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -191,6 +191,7 @@ pre_computed_hashes = [ {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"}, # lfm2 variants {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"}, + {"name": "spark2_5", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/XHToken/Spark-X2.5-1.7B", "chkhsh": "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed"}, ] diff --git a/docs/autoparser.md b/docs/autoparser.md index b5e32621d..2a7ea00b4 100644 --- a/docs/autoparser.md +++ b/docs/autoparser.md @@ -514,6 +514,7 @@ The following templates have active tests in `tests/test-chat.cpp`: | Mistral Small 3.2 | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` with call ID | | Devstral | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` without call ID | | StepFun 3.5 Flash | TAG_WITH_TAGGED | `` format | +| Spark2.5 | TAG_WITH_TAGGED | `name......` format | ## Adding Support for New Templates diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 399d31f1d..486f3586d 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -619,6 +619,7 @@ class MODEL_ARCH(IntEnum): PADDLEOCR = auto() MIMO2 = auto() STEP35 = auto() + SPARK2_5 = auto() LLAMA_EMBED = auto() MAINCODER = auto() KIMI_LINEAR = auto() @@ -1373,6 +1374,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.PADDLEOCR: "paddleocr", MODEL_ARCH.MIMO2: "mimo2", MODEL_ARCH.STEP35: "step35", + MODEL_ARCH.SPARK2_5: "spark2_5", MODEL_ARCH.LLAMA_EMBED: "llama-embed", MODEL_ARCH.MAINCODER: "maincoder", MODEL_ARCH.KIMI_LINEAR: "kimi-linear", @@ -5231,6 +5233,19 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], + MODEL_ARCH.SPARK2_5: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], MODEL_ARCH.LLAMA_EMBED: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/models/templates/README.md b/models/templates/README.md index 3a649b8f4..022a5e278 100644 --- a/models/templates/README.md +++ b/models/templates/README.md @@ -23,4 +23,6 @@ These templates can be updated with the following commands: ./scripts/get_chat_template.py Qwen/Qwen3-0.6B > models/templates/Qwen-Qwen3-0.6B.jinja ./scripts/get_chat_template.py zai-org/GLM-4.5 > models/templates/zai-org-GLM-4.5.jinja ./scripts/get_chat_template.py deepseek-ai/DeepSeek-V3.1 > models/templates/deepseek-ai-DeepSeek-V3.1.jinja +./scripts/get_chat_template.py XHToken/Spark-X2.5-1.7B > models/templates/Spark2.5.jinja +./scripts/get_chat_template.py XHToken/Spark-X2.5-4B > models/templates/Spark2.5.jinja ``` diff --git a/models/templates/Spark2.5.jinja b/models/templates/Spark2.5.jinja new file mode 100644 index 000000000..54aa34ff2 --- /dev/null +++ b/models/templates/Spark2.5.jinja @@ -0,0 +1,110 @@ +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} + +{%- set enable_thinking = enable_thinking | default(true) %} + +{#- Render a string or a list of text blocks. -#} +{%- macro render_content(content, context_name) %} + {%- if content is string %} + {{- content }} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- elif content is iterable and content is not mapping %} + {%- for block in content %} + {%- if block.type == 'text' %} + {{- block.text }} + {%- else %} + {{- raise_exception('Unsupported ' ~ context_name ~ ' content block type: ' ~ (block.type | string)) }} + {%- endif %} + {%- endfor %} + {%- else %} + {{- raise_exception(context_name ~ ' content must be a string or a list of text blocks') }} + {%- endif %} +{%- endmacro %} + +{#- Default system prompt. -#} +{%- set default_system = 'you are a helpful assistant.' %} + +{#- The first message-level system is placed in the initial system block. -#} +{%- set ns = namespace(initial_system='') %} +{%- if messages[0].role == 'system' %} + {%- set ns.initial_system = render_content(messages[0].content, 'system') %} +{%- endif %} + +{#- System block. -#} +{{- '<｜start▁of▁sentence｜><|System|>' + '\n' + default_system }} +{%- if tools %} + {{- '## Tools' + '\n' + 'You have access to the following functions:' + '\n' + '' }} + {%- for tool in tools %} + {{- '\n' + tool.function | tojson }} + {%- endfor %} + {{- '\n' + '' }} +{%- endif %} +{%- if ns.initial_system %} + {{- '\n\n' + ns.initial_system }} +{%- endif %} +{{- '<｜end▁of▁sentence｜>' }} + +{#- Conversation turns. -#} +{%- for message in messages %} + {%- if message.role == 'system' %} + {#- The first system message was consumed by the initial block. -#} + {%- if not loop.first %} + {{- '<｜start▁of▁sentence｜><|System|>\n' + render_content(message.content, 'system') + '<｜end▁of▁sentence｜>' }} + {%- endif %} + {%- elif message.role == 'user' %} + {{- '<｜start▁of▁sentence｜><|User|>' + render_content(message.content, 'user') + '<｜end▁of▁sentence｜>' }} + {%- elif message.role == 'assistant' %} + {%- set assistant_content = render_content(message.content, 'assistant') %} + {%- if message.reasoning_content is defined and message.reasoning_content %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- set reasoning_content = '' %} + {%- endif %} + {{- '<｜start▁of▁sentence｜><|Bot|>' }} + {%- if reasoning_content %} + {{- '' + reasoning_content + '' }} + {%- else %} + {{- '' }} + {%- endif %} + {%- if assistant_content %} + {{- assistant_content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls is not none %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function.arguments is not mapping %} + {{- raise_exception('tool_call.function.arguments must be a dictionary; normalize JSON strings before apply_chat_template') }} + {%- endif %} + {%- set args = tool_call.function.arguments %} + {{- '' + tool_call.function.name }} + {%- for k, v in args.items() %} + {{- '' ~ k ~ '' ~ (v if v is string else v | tojson) ~ '' }} + {%- endfor %} + {{- '' }} + {%- endfor %} + {%- endif %} + {{- '<｜end▁of▁sentence｜>' }} + {%- elif message.role == 'tool' %} + {%- if loop.previtem is undefined or loop.previtem.role != 'tool' %} + {{- '<｜start▁of▁sentence｜><|Tool|>' }} + {%- endif %} + {{- '' ~ message.content ~ '' }} + {%- if loop.nextitem is undefined or loop.nextitem.role != 'tool' %} + {{- '<｜end▁of▁sentence｜>' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unsupported message role: ' ~ message.role) }} + {%- endif %} +{%- endfor %} + +{#- Generation prompt. -#} +{%- if add_generation_prompt %} + {{- '<｜start▁of▁sentence｜><|Bot|>' }} + {%- if enable_thinking is defined and enable_thinking %} + {{- '' }} + {%- endif %} + {%- if enable_thinking is defined and not enable_thinking %} + {{- '' }} + {%- endif %} +{%- endif %} diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index d06be641a..15f651919 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -146,6 +146,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_PADDLEOCR, "paddleocr" }, { LLM_ARCH_MIMO2, "mimo2" }, { LLM_ARCH_STEP35, "step35" }, + { LLM_ARCH_SPARK2_5, "spark2_5" }, { LLM_ARCH_LLAMA_EMBED, "llama-embed" }, { LLM_ARCH_MAINCODER, "maincoder" }, { LLM_ARCH_KIMI_LINEAR, "kimi-linear" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 62dfa5d81..f1d173a57 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -147,6 +147,7 @@ enum llm_arch { LLM_ARCH_PADDLEOCR, LLM_ARCH_MIMO2, LLM_ARCH_STEP35, + LLM_ARCH_SPARK2_5, LLM_ARCH_LLAMA_EMBED, LLM_ARCH_MAINCODER, LLM_ARCH_KIMI_LINEAR, diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index df2a46d93..66f8bdec3 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -27,6 +27,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { case LLM_ARCH_APERTUS: case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: + case LLM_ARCH_SPARK2_5: case LLM_ARCH_MUSE_GLIMMER: case LLM_ARCH_MELLUM: case LLM_ARCH_LAGUNA: diff --git a/src/llama-model.cpp b/src/llama-model.cpp index b837e2765..0e0036781 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -338,6 +338,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_kimi_k3(params); case LLM_ARCH_STEP35: return new llama_model_step35(params); + case LLM_ARCH_SPARK2_5: + return new llama_model_spark2_5(params); default: throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'"); } @@ -2999,6 +3001,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_QWEN3NEXT: case LLM_ARCH_MIMO2: case LLM_ARCH_STEP35: + case LLM_ARCH_SPARK2_5: case LLM_ARCH_TALKIE: case LLM_ARCH_MELLUM: return LLAMA_ROPE_TYPE_NEOX; diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index c0c34cdd8..a69801f08 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -325,6 +325,14 @@ struct llm_tokenizer_bpe : llm_tokenizer { "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+[\r\n]*|\\s*[\r\n]+|\\s+(?!\\S)|\\s+", }; break; + case LLAMA_VOCAB_PRE_TYPE_SPARK2_5: + regex_exprs = { + "\\p{N}{1,3}", + "[一-龥぀-ゟ゠-ヿ]+", + "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+|[\r\n]|\\s+(?!\\S)|\\s+", + "\\p{N}", + }; + break; case LLAMA_VOCAB_PRE_TYPE_YOUTU: regex_exprs = { "[가-힣ㄱ-ㆎ]+|[！…“”‘’—：；，、-〿︰-﹏]+|[ㄅ-ㄯ]+|[一-龥぀-ゟ゠-ヿ]+", @@ -2170,6 +2178,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "deepseek-v3") { pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM; clean_spaces = false; + } else if ( + tokenizer_pre == "spark2_5") { + pre_type = LLAMA_VOCAB_PRE_TYPE_SPARK2_5; + clean_spaces = false; } else if ( tokenizer_pre == "youtu") { pre_type = LLAMA_VOCAB_PRE_TYPE_YOUTU; diff --git a/src/llama-vocab.h b/src/llama-vocab.h index e02ea78ff..65293c026 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -66,6 +66,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55, LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, LLAMA_VOCAB_PRE_TYPE_HY_V4 = 57, + LLAMA_VOCAB_PRE_TYPE_SPARK2_5 = 58, }; struct LLM_KV; diff --git a/src/models/models.h b/src/models/models.h index 93a6b3494..50e9a235c 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2606,3 +2606,16 @@ struct llama_model_step35 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; + + +struct llama_model_spark2_5 : public llama_model_base { + llama_model_spark2_5(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; diff --git a/src/models/spark2-5.cpp b/src/models/spark2-5.cpp new file mode 100644 index 000000000..107448777 --- /dev/null +++ b/src/models/spark2-5.cpp @@ -0,0 +1,146 @@ +#include "models.h" + +void llama_model_spark2_5::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + + switch (hparams.n_layer()) { + case 28: type = LLM_TYPE_1_7B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_spark2_5::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == nullptr) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + const int64_t n_head_i = hparams.n_head(i); + const int64_t n_head_kv_i = hparams.n_head_kv(i); + const int64_t n_embd_q = hparams.n_embd_head_k(i) * n_head_i; + const int64_t n_embd_k = hparams.n_embd_head_k(i) * n_head_kv_i; + const int64_t n_embd_v = hparams.n_embd_head_v(i) * n_head_kv_i; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_k, n_embd_v, 0); + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_i}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + } +} + +std::unique_ptr llama_model_spark2_5::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_spark2_5::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + GGML_ASSERT(hparams.swa_type == LLAMA_SWA_TYPE_STANDARD); + + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + auto * inp_attn = build_attn_inp_kv_iswa(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + ggml_tensor * cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + const int64_t n_head_i = hparams.n_head(il); + const int64_t n_head_kv_i = hparams.n_head_kv(il); + const int64_t n_rot_i = hparams.n_rot(il); + const float freq_base_i = model.get_rope_freq_base(cparams, il); + const float freq_scale_i = model.get_rope_freq_scale(cparams, il); + + ggml_tensor * attn_inp = cur; + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head_i, n_head_kv_i, il); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, + n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, + n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur_rope", il); + cb(Kcur, "Kcur_rope", il); + + cur = build_attn(inp_attn, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + + ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp); + gate = ggml_sigmoid(ctx0, gate); + cb(gate, "attn_gate", il); + + const int64_t n_tokens_i = cur->ne[1]; + cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head_i, n_tokens_i); + gate = ggml_reshape_3d(ctx0, gate, 1, n_head_i, n_tokens_i); + cur = ggml_mul(ctx0, cur, gate); + cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head_i, n_tokens_i); + cb(cur, "attn_gated", il); + + cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); + cb(cur, "attn_out_proj", il); + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, nullptr, nullptr, + model.layers[il].ffn_gate, nullptr, nullptr, + model.layers[il].ffn_down, nullptr, nullptr, + nullptr, + LLM_FFN_GELU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + ggml_tensor * cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 7918f0ffc..f27c91e4d 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -4405,6 +4405,100 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); } + // Spark2.5 uses tagged arguments with forced-open thinking. + { + auto tst = peg_tester("models/templates/Spark2.5.jinja", detailed_debug); + + tst.test("Hello, world!\nWhat's up?") + .enable_thinking(false) + .expect(message_assist) + .expect_reconstruction() + .run(); + + tst.test("I'm\nthinkingHello, world!\nWhat's up?") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .expect(message_assist_thoughts) + .expect_reconstruction() + .run(); + + tst.test( + "special_function" + "arg11" + "") + .enable_thinking(false) + .tools({ special_function_tool }) + .expect(message_assist_call) + .expect_reconstruction() + .run(); + + tst.test( + "I'm\nthinking" + "special_function" + "arg11" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ special_function_tool }) + .expect(message_assist_call_thoughts) + .expect_reconstruction() + .run(); + + tst.test( + "special_function" + "arg11" + "" + "special_function_with_opt" + "arg11" + "arg22" + "") + .enable_thinking(false) + .parallel_tool_calls(true) + .tools({ special_function_tool, special_function_tool_with_optional_param }) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .expect_reconstruction() + .run(); + + tst.test( + "Preparing updates." + "magic_int" + "ref42" + "name上海" + "" + "amount" + "orig2.5" + "" + "toggle" + "enabledtrue" + "" + "set_config" + "config{\"source\": \"spark\", \"options\": {\"strict\": true}}" + "" + "nested_args" + "tags[\"alpha\", \"测试\"]" + "entries[{\"id\": 1, \"label\": \"first\"}, {\"id\": 2, \"label\": \"第二\"}]" + "" + "empty_args" + "") + .enable_thinking(false) + .parallel_tool_calls(true) + .tools({ magic_int_tool, amount_tool, toggle_tool, config_tool, nested_args_tool, empty_args_tool }) + .expect_content("Preparing updates.") + .expect_tool_calls({ + { "magic_int", R"({"ref": 42, "name": "上海"})", {} }, + { "amount", R"({"orig": 2.5})", {} }, + { "toggle", R"({"enabled": true})", {} }, + { "set_config", R"({"config": {"source": "spark", "options": {"strict": true}}})", {} }, + { "nested_args", R"({"tags": ["alpha", "测试"], "entries": [{"id": 1, "label": "first"}, {"id": 2, "label": "第二"}]})", {} }, + { "empty_args", "{}", {} }, + }) + .expect_reconstruction() + .run(); + } + // Verify the throw path produces a readable error message, not std::out_of_range. // #20424 introduced effective_input = generation_prompt + input, but the throw // uses input.substr(result.end) where result.end is in effective_input space. diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 0f3d1c79a..dbed9846f 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -237,7 +237,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); // SWA pattern: every 5th layer is full attention (matches E2B layer_types) ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); - } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || + } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE) { std::vector pattern; pattern.reserve(n_layer);