From 23f7aabc16c72f6f707dba227251c35d7bd1ed96 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 16 Oct 2026 12:00:41 -0600 Subject: [PATCH] model : Granite Switch Decision (#30676) * feat(convert): Add decision.chat to support decision models that can also chat Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(convert): Add conversion for Granite Switch as decision model The template follows from the usage sample: https://huggingface.co/ibm-granite/granite-switch-4.1-8b#guardian-and-decision-side-by-side Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(common): Add support for chat+decision models in common Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(common): Add granite switch decision model enum Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(server): Plumb through decision_chat bool to support decision+chat models Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(server): Add Granite Switch decision support Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * docs: Update server README for granite switch decisions / chat+decisions Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * Revert "feat(convert): Add decision.chat to support decision models that can also chat" This reverts commit 952ef567647e53ac9beda7a1f79d6927240d068d. * Revert "feat(common): Add support for chat+decision models in common" This reverts commit 93ed782e3a8b2fd5968732c6525c7ad29ad4bea2. * Revert "feat(server): Plumb through decision_chat bool to support decision+chat models" This reverts commit 97e9af60ddaccbd82f1eb0a488adff43b16ee095. * fix: If Granite Switch supports decisions, it always supports text too Branch: GraniteSwitchDecision AI-usage: none Signed-off-by: Gabe Goodhart * docs: Remove reference to decision.chat in server README Branch: GraniteSwitchDecision AI-usage: none Signed-off-by: Gabe Goodhart --- common/common.cpp | 1 + common/common.h | 1 + conversion/granite.py | 31 ++++++++++++++++++++++++++++++- gguf-py/gguf/constants.py | 1 + tools/server/README.md | 4 ++-- tools/server/server-common.cpp | 2 ++ tools/server/server-decision.cpp | 4 ++-- tools/server/server-decision.h | 1 + 8 files changed, 40 insertions(+), 5 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index c5db22832..b59e66e3b 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1174,6 +1174,7 @@ static const std::map COMMON_DECISION_TYPE_NA { COMMON_DECISION_TYPE_VLLM_D3, "vllm-d3" }, { COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" }, { COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" }, + { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"}, // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname" }; diff --git a/common/common.h b/common/common.h index 2cd6e2ec9..0c2d5e46f 100644 --- a/common/common.h +++ b/common/common.h @@ -970,6 +970,7 @@ enum common_decision_type { COMMON_DECISION_TYPE_VLLM_D3, // same as pplx-decider, non-causal attention COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout + COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported }; diff --git a/conversion/granite.py b/conversion/granite.py index 4cbeb7617..d2ef7c836 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -8,7 +8,7 @@ import torch if TYPE_CHECKING: from torch import Tensor -from .base import MmprojModel, ModelBase, gguf, logger +from .base import MmprojModel, ModelBase, gguf, jinja_str_or_json, logger from .llama import LlamaModel from .mamba import Mamba2Model @@ -284,10 +284,39 @@ class GraniteSwitchModel(GraniteMoeModel): self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank) self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"]) self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"]) + if self._has_decision_adapter(): + self.gguf_writer.add_decision_type(gguf.DecisionType.GRANITE_SWITCH) router_gain = float(self.hparams.get("control_token_gain", 15.0)) self.gguf_writer.add_adapter_router_gain(router_gain) logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain) + def _has_decision_adapter(self) -> bool: + return "decision" in self.hparams.get("adapter_names", []) + + def set_vocab(self): + super().set_vocab() + if self._has_decision_adapter(): + self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}]) + + def _systemone_template(self) -> str: + # prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token + # (it replaces the first token of "### DECISION"), the answer is the label token that comes next + description = jinja_str_or_json("o.description") + option = ( + "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}" + "{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}" + "{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}" + ) + return ( + "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}" + "<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n" + "<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions") + + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}" + "{{ '\n\nOptions:\n' }}" + "{% for o in options %}" + option + "{{ '\n' }}{% endfor %}" + "{{ '\nReply with exactly one option label.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>' }}" + ) + def _lora_a(self, data: Tensor) -> Tensor: # on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in] a = data.squeeze(1) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 00b51165c..4261e76e3 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6283,6 +6283,7 @@ class DecisionType: VLLM_D3 = "vllm-d3" # same as pplx-decider, non-causal attention LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout + GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switch class VisionProjectorType: diff --git a/tools/server/README.md b/tools/server/README.md index 7d697633e..e4a0c418b 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1723,9 +1723,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly. -The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. +The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. -For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. +For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`. *Image input:* diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index c32c0d1a5..5577c339d 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -161,6 +161,8 @@ std::vector server_model_output_modalities(common_decision_type dec case COMMON_DECISION_TYPE_LFM2_D1: case COMMON_DECISION_TYPE_LFM2_D1_OMNI: return {"decisions"}; + case COMMON_DECISION_TYPE_GRANITE_SWITCH: + return {"text", "decisions"}; default: // fallback when there is no decision type or the metadata is bad return {"text"}; diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index b3c19e1f0..3905c81b3 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) { bias[level] = std::strtof(val, nullptr); } - if (model_type == COMMON_DECISION_TYPE_OPENJEV) { + if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) { // one letter per option, each must be a single token const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; for (const char c : letters) { @@ -753,7 +753,7 @@ void server_decision_context::fill_task( const std::string prompt = render(state, questions, question, variant, files.size()); - if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) { + if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) { // lev reads the ratings of a noul question at its first labels, not at the digits task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question)); } diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h index bfe9fd0ee..ed9724429 100644 --- a/tools/server/server-decision.h +++ b/tools/server/server-decision.h @@ -39,6 +39,7 @@ struct server_decision_context { bool can_share_prompt() const { switch (type) { case COMMON_DECISION_TYPE_OPENJEV: + case COMMON_DECISION_TYPE_GRANITE_SWITCH: case COMMON_DECISION_TYPE_LEV: case COMMON_DECISION_TYPE_KEV: case COMMON_DECISION_TYPE_NIMBLE: