model : Granite Switch Decision (#30676)
* feat(convert): Add decision.chat to support decision models that can also chat Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(convert): Add conversion for Granite Switch as decision model The template follows from the usage sample: https://huggingface.co/ibm-granite/granite-switch-4.1-8b#guardian-and-decision-side-by-side Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(common): Add support for chat+decision models in common Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(common): Add granite switch decision model enum Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(server): Plumb through decision_chat bool to support decision+chat models Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(server): Add Granite Switch decision support Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * docs: Update server README for granite switch decisions / chat+decisions Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * Revert "feat(convert): Add decision.chat to support decision models that can also chat" This reverts commit 952ef567647e53ac9beda7a1f79d6927240d068d. * Revert "feat(common): Add support for chat+decision models in common" This reverts commit 93ed782e3a8b2fd5968732c6525c7ad29ad4bea2. * Revert "feat(server): Plumb through decision_chat bool to support decision+chat models" This reverts commit 97e9af60ddaccbd82f1eb0a488adff43b16ee095. * fix: If Granite Switch supports decisions, it always supports text too Branch: GraniteSwitchDecision AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * docs: Remove reference to decision.chat in server README Branch: GraniteSwitchDecision AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
This commit is contained in:
1 parent
7182ef71e6
commit
23f7aabc16
8 files changed
+40
-5
No files matched your search
@@ -1174,6 +1174,7 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
|
||||
{ COMMON_DECISION_TYPE_VLLM_D3, "vllm-d3" },
|
||||
{ COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },
|
||||
{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },
|
||||
{ COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
|
||||
// for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
|
||||
};
|
||||
|
||||
|
||||
@@ -970,6 +970,7 @@ enum common_decision_type {
|
||||
COMMON_DECISION_TYPE_VLLM_D3, // same as pplx-decider, non-causal attention
|
||||
COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
|
||||
COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
|
||||
COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
|
||||
+30
-1
@@ -8,7 +8,7 @@ import torch
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import MmprojModel, ModelBase, gguf, logger
|
||||
from .base import MmprojModel, ModelBase, gguf, jinja_str_or_json, logger
|
||||
|
||||
from .llama import LlamaModel
|
||||
from .mamba import Mamba2Model
|
||||
@@ -284,10 +284,39 @@ class GraniteSwitchModel(GraniteMoeModel):
|
||||
self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank)
|
||||
self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"])
|
||||
self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"])
|
||||
if self._has_decision_adapter():
|
||||
self.gguf_writer.add_decision_type(gguf.DecisionType.GRANITE_SWITCH)
|
||||
router_gain = float(self.hparams.get("control_token_gain", 15.0))
|
||||
self.gguf_writer.add_adapter_router_gain(router_gain)
|
||||
logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain)
|
||||
|
||||
def _has_decision_adapter(self) -> bool:
|
||||
return "decision" in self.hparams.get("adapter_names", [])
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
if self._has_decision_adapter():
|
||||
self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
|
||||
|
||||
def _systemone_template(self) -> str:
|
||||
# prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token
|
||||
# (it replaces the first token of "### DECISION"), the answer is the label token that comes next
|
||||
description = jinja_str_or_json("o.description")
|
||||
option = (
|
||||
"{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}"
|
||||
"{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}"
|
||||
"{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}"
|
||||
)
|
||||
return (
|
||||
"{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}"
|
||||
"<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n"
|
||||
"<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions")
|
||||
+ "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"
|
||||
"{{ '\n\nOptions:\n' }}"
|
||||
"{% for o in options %}" + option + "{{ '\n' }}{% endfor %}"
|
||||
"{{ '\nReply with exactly one option label.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>' }}"
|
||||
)
|
||||
|
||||
def _lora_a(self, data: Tensor) -> Tensor:
|
||||
# on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in]
|
||||
a = data.squeeze(1)
|
||||
|
||||
@@ -6283,6 +6283,7 @@ class DecisionType:
|
||||
VLLM_D3 = "vllm-d3" # same as pplx-decider, non-causal attention
|
||||
LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type
|
||||
LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout
|
||||
GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switch
|
||||
|
||||
|
||||
class VisionProjectorType:
|
||||
|
||||
@@ -1723,9 +1723,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
|
||||
|
||||
The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
|
||||
|
||||
The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
|
||||
The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
|
||||
|
||||
For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available.
|
||||
For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
|
||||
|
||||
*Image input:*
|
||||
|
||||
|
||||
@@ -161,6 +161,8 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
|
||||
case COMMON_DECISION_TYPE_LFM2_D1:
|
||||
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
|
||||
return {"decisions"};
|
||||
case COMMON_DECISION_TYPE_GRANITE_SWITCH:
|
||||
return {"text", "decisions"};
|
||||
default:
|
||||
// fallback when there is no decision type or the metadata is bad
|
||||
return {"text"};
|
||||
|
||||
@@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) {
|
||||
bias[level] = std::strtof(val, nullptr);
|
||||
}
|
||||
|
||||
if (model_type == COMMON_DECISION_TYPE_OPENJEV) {
|
||||
if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {
|
||||
// one letter per option, each must be a single token
|
||||
const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
|
||||
for (const char c : letters) {
|
||||
@@ -753,7 +753,7 @@ void server_decision_context::fill_task(
|
||||
|
||||
const std::string prompt = render(state, questions, question, variant, files.size());
|
||||
|
||||
if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
|
||||
if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
|
||||
// lev reads the ratings of a noul question at its first labels, not at the digits
|
||||
task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ struct server_decision_context {
|
||||
bool can_share_prompt() const {
|
||||
switch (type) {
|
||||
case COMMON_DECISION_TYPE_OPENJEV:
|
||||
case COMMON_DECISION_TYPE_GRANITE_SWITCH:
|
||||
case COMMON_DECISION_TYPE_LEV:
|
||||
case COMMON_DECISION_TYPE_KEV:
|
||||
case COMMON_DECISION_TYPE_NIMBLE:
|
||||
|
||||
Reference in new issue
Block a user