models : add Nemotron Super 3.5 VL and Nemotron Bolt 74B A9B support (#30520)
* add support * remove unneeded bos fix * add video flag * fix flake error * remove video-specific budgeting * convert : fix lilicorr-dflash bid clash * spec : enforce lilicorr n_tokens condition * spec : fix lilicorr walk [no release]
This commit is contained in:
1 parent
a11b6530b5
commit
cd0a5b6a18
15 files changed
+265
-22
No files matched your search
@@ -1338,7 +1338,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
||||
GGML_ASSERT(lattice && "DFlash2 selector produced no lattice");
|
||||
|
||||
int32_t predecessor = 0;
|
||||
for (int32_t i = 1; i < n_block_tokens; ++i) {
|
||||
for (int32_t i = 1; i < (is_dflash2 ? n_block_tokens : params.n_max + 1); ++i) {
|
||||
const float * row = lattice + (size_t) (beg + i) * n_embd_dec;
|
||||
const int32_t top_k = is_dflash2 ? selector_top_k : lilicorr_top_k;
|
||||
const float * scores = row + top_k + (size_t) predecessor * top_k;
|
||||
|
||||
@@ -353,6 +353,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
|
||||
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
|
||||
"Mistral3ForConditionalGeneration": "llava",
|
||||
"NemotronH_Omni_Reasoning_V3": "nemotron",
|
||||
"NemotronH_Nano_VL_V2": "nemotron",
|
||||
"MuseGlimmerForConditionalGeneration": "muse_glimmer",
|
||||
"PaddleOCRVisionModel": "ernie",
|
||||
|
||||
+1
-1
@@ -568,7 +568,7 @@ class ModelBase:
|
||||
tensors_to_remove.append(base_name + "_zero_point")
|
||||
else:
|
||||
raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
|
||||
elif quant_method == "modelopt":
|
||||
elif quant_method in ("modelopt", "modelopt_mixed"):
|
||||
dequant_fp8()
|
||||
elif quant_method is not None:
|
||||
raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")
|
||||
|
||||
+132
-12
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -89,19 +89,22 @@ class LiLiCorrDraftModel(DFlashModel):
|
||||
self.gguf_writer.add_lilicorr_vector_eps(float(dflash_config["lilicorr_vector_eps"]))
|
||||
self.gguf_writer.add_lilicorr_logit_scale(float(dflash_config["lilicorr_logit_scale"]))
|
||||
|
||||
def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:
|
||||
if name.startswith("model.lilicorr.layers."):
|
||||
parts = name.split(".", 4)
|
||||
if not parts[4].startswith("attn.in_proj."):
|
||||
parts[3] = str(self.block_count + int(parts[3]))
|
||||
name = ".".join(parts)
|
||||
elif name.startswith("model.lilicorr.feature_norm."):
|
||||
feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
|
||||
name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.")
|
||||
elif name.startswith("model.lilicorr.feature_mlp."):
|
||||
feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
|
||||
name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.")
|
||||
return super().map_tensor_name(name, try_suffixes)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("model.lilicorr."):
|
||||
if name.startswith("model.lilicorr.layers."):
|
||||
parts = name.split(".", 4)
|
||||
if not parts[4].startswith("attn.in_proj."):
|
||||
parts[3] = str(self.block_count + int(parts[3]))
|
||||
name = ".".join(parts)
|
||||
elif name.startswith("model.lilicorr.feature_norm."):
|
||||
feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
|
||||
name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.")
|
||||
elif name.startswith("model.lilicorr.feature_mlp."):
|
||||
feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
|
||||
name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.")
|
||||
yield self.map_tensor_name(name), data_torch
|
||||
return
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
@@ -251,6 +254,113 @@ class NemotronNanoV2VLModel(MmprojModel):
|
||||
yield mapped_name, tensor
|
||||
|
||||
|
||||
@ModelBase.register("NemotronH_Omni_Reasoning_V3")
|
||||
class NemotronOmniVLModel(NemotronNanoV2VLModel):
|
||||
_omni_tensor_prefixes = (
|
||||
"vision_model.embeddings.",
|
||||
"vision_model.encoder.layer.",
|
||||
"vision_projector.mlp1.",
|
||||
)
|
||||
|
||||
def get_vision_config(self) -> dict[str, Any] | None:
|
||||
vision_config = dict(self.global_config["vision_config"])
|
||||
vision_config["intermediate_size"] = int(vision_config["hidden_size"] * vision_config["mlp_ratio"])
|
||||
vision_config["image_size"] = 512 # warmup only; the full position table is resized at runtime
|
||||
return vision_config
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
preprocessor_config = self.preprocessor_config
|
||||
|
||||
preprocessor_config["image_mean"] = preprocessor_config["norm_mean"]
|
||||
preprocessor_config["image_std"] = preprocessor_config["norm_std"]
|
||||
|
||||
MmprojModel.set_gguf_parameters(self)
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_OMNI)
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
|
||||
self.gguf_writer.add_vision_use_gelu(True)
|
||||
downsample_ratio = self.global_config.get("downsample_ratio", 0.5)
|
||||
self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio))
|
||||
patch_area = self.hparams["patch_size"] ** 2
|
||||
self.gguf_writer.add_vision_min_pixels(preprocessor_config["min_num_patches"] * patch_area)
|
||||
self.gguf_writer.add_vision_max_pixels(preprocessor_config["max_num_patches"] * patch_area)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
# Transformers can rename RADIO and projector tensors in Omni checkpoints.
|
||||
# Accept both checkpoint layouts.
|
||||
if name == "vision_model.summary_idxs":
|
||||
# This buffer is only used by RadioModel.summary. The multimodal
|
||||
# projector consumes RadioModel.features for both images and video.
|
||||
return None
|
||||
if name.startswith(cls._omni_tensor_prefixes):
|
||||
return name, gen
|
||||
|
||||
if "radio_model.model.patch_generator.video_embedder" in name or name.startswith("vision_projector.vision_final_layernorm."):
|
||||
return MmprojModel.filter_tensors((name, gen))
|
||||
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if new_name in ("v.position_embd.weight", "v.class_embd"):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith(self._omni_tensor_prefixes):
|
||||
if name == "vision_model.embeddings.video_patch_projection.weight":
|
||||
data_torch = data_torch.reshape(
|
||||
self.hparams["hidden_size"],
|
||||
6,
|
||||
self.hparams["patch_size"],
|
||||
self.hparams["patch_size"],
|
||||
)
|
||||
yield gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch
|
||||
return
|
||||
|
||||
if name.startswith("vision_projector.mlp1."):
|
||||
projector_mapping = {
|
||||
"vision_projector.mlp1.norm.weight": "mlp1.0.weight",
|
||||
"vision_projector.mlp1.linear1.weight": "mlp1.1.weight",
|
||||
"vision_projector.mlp1.linear2.weight": "mlp1.3.weight",
|
||||
}
|
||||
name = projector_mapping[name]
|
||||
elif name == "vision_model.embeddings.patch_projection.weight":
|
||||
data_torch = data_torch.reshape(
|
||||
self.hparams["hidden_size"],
|
||||
3,
|
||||
self.hparams["patch_size"],
|
||||
self.hparams["patch_size"],
|
||||
)
|
||||
elif name == "vision_model.embeddings.position_embedding":
|
||||
data_torch = data_torch.squeeze(0)
|
||||
name += ".weight"
|
||||
elif ".layer_scale" in name and name.endswith(".lambda1"):
|
||||
name = name.removesuffix("lambda1") + "weight"
|
||||
|
||||
yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
|
||||
return
|
||||
|
||||
if "patch_generator.pos_embed" in name:
|
||||
# Keep the full table for runtime interpolation.
|
||||
yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
|
||||
return
|
||||
|
||||
if "patch_generator.video_embedder" in name:
|
||||
patch_size = self.hparams["patch_size"]
|
||||
n_embd = self.hparams["hidden_size"]
|
||||
temporal_patch_size = self.global_config["video_temporal_patch_size"]
|
||||
if temporal_patch_size != 2:
|
||||
raise ValueError("Current implementation only supports a temporal patch size of 2")
|
||||
data_torch = data_torch.reshape(n_embd, temporal_patch_size * 3, patch_size, patch_size)
|
||||
new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1"
|
||||
yield new_name, data_torch
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("NemotronForCausalLM")
|
||||
@ModelBase.example("nvidia/Minitron-4B-Base")
|
||||
class NemotronModel(TextModel):
|
||||
@@ -382,6 +492,8 @@ class NemotronHModel(GraniteHybridModel):
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if name.startswith("language_model."):
|
||||
name = name[len("language_model."):]
|
||||
if name.startswith("mtp."):
|
||||
# --no-mtp: drop the MTP head entirely
|
||||
if cls.no_mtp:
|
||||
@@ -531,6 +643,14 @@ class NemotronHModel(GraniteHybridModel):
|
||||
"mtp.layers.1.final_layernorm.weight": "model.layers.{bid}.shared_head.norm.weight",
|
||||
}
|
||||
|
||||
def _generate_nvfp4_tensors(self):
|
||||
if self._mtp_bid is not None:
|
||||
for name in list(self.model_tensors):
|
||||
if name.startswith("mtp.layers.1.mixer"):
|
||||
new_name = f"backbone.layers.{self._mtp_bid}.{name.split('.', 3)[3]}"
|
||||
self.model_tensors[new_name] = self.model_tensors.pop(name)
|
||||
super()._generate_nvfp4_tensors()
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# mtp.layers.0: NextN input fusion + attention
|
||||
# mtp.layers.1: MoE + final head norm
|
||||
|
||||
@@ -6265,6 +6265,7 @@ class VisionProjectorType:
|
||||
GLM5V = "glm5v"
|
||||
YOUTUVL = "youtuvl"
|
||||
NEMOTRON_V2_VL = "nemotron_v2_vl"
|
||||
NEMOTRON_OMNI = "nemotron_omni"
|
||||
QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder
|
||||
QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
|
||||
POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder
|
||||
|
||||
@@ -1735,6 +1735,7 @@ class TensorNameMap:
|
||||
"vision_model.class_embedding", # llama 4
|
||||
"model.vision.patch_embedding.cls_embedding", # cogvlm
|
||||
"vision_model.radio_model.model.patch_generator.cls_token.token", # Nemotron Nano v2 VL
|
||||
"vision_model.embeddings.cls_register_token", # Nemotron Omni
|
||||
"model.vision_model.embeddings.class_embedding", # Deepseek-OCR
|
||||
),
|
||||
|
||||
@@ -1756,6 +1757,7 @@ class TensorNameMap:
|
||||
"model.vision_model.embeddings.patch_embedding", # Deepseek-OCR CLIP
|
||||
"siglip2.vision_model.embeddings.patch_embedding",
|
||||
"vision_model.radio_model.model.patch_generator.embedder", # Nemotron Nano v2 VL
|
||||
"vision_model.embeddings.patch_projection", # Nemotron Omni
|
||||
"model.vision_tower.patch_embedder.input_proj", # gemma4
|
||||
"vision_tower.patch_embed.patchifier.proj", # dots.ocr
|
||||
"vision_encoder.patch_embed.proj", # dots3note
|
||||
@@ -1788,6 +1790,7 @@ class TensorNameMap:
|
||||
"model.vision.patch_embedding.position_embedding", # cogvlm
|
||||
"visual.embeddings.position_embedding", # glm4v
|
||||
"vision_model.radio_model.model.patch_generator.pos_embed", # Nemotron Nano v2 VL
|
||||
"vision_model.embeddings.position_embedding", # Nemotron Omni
|
||||
"model.vision_tower.patch_embedder.position_embedding_table", # gemma4
|
||||
"vision_model.positional_embedding", # Step3-VL
|
||||
"model.vision_embedder.pos_embedding", # gemma4 unified
|
||||
@@ -1818,6 +1821,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_Q: (
|
||||
"vision_model.encoder.layer.{bid}.attention.attention.query", # Nemotron Omni
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj",
|
||||
"model.vision_tower.encoder.layers.{bid}.self_attn.q_proj", # minicpmv4_6
|
||||
@@ -1846,6 +1850,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_K: (
|
||||
"vision_model.encoder.layer.{bid}.attention.attention.key", # Nemotron Omni
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj",
|
||||
"model.vision_tower.encoder.layers.{bid}.self_attn.k_proj", # minicpmv4_6
|
||||
@@ -1874,6 +1879,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_V: (
|
||||
"vision_model.encoder.layer.{bid}.attention.attention.value", # Nemotron Omni
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj",
|
||||
"model.vision_tower.encoder.layers.{bid}.self_attn.v_proj", # minicpmv4_6
|
||||
@@ -1894,6 +1900,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_INPUT_NORM: (
|
||||
"vision_model.encoder.layer.{bid}.norm1", # Nemotron Omni
|
||||
"vision.blocks.{bid}.norm1", # deepseek4v
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm1",
|
||||
@@ -1920,6 +1927,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_O: (
|
||||
"vision_model.encoder.layer.{bid}.attention.output.dense", # Nemotron Omni
|
||||
"vision.blocks.{bid}.attn.wo", # deepseek4v
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj",
|
||||
@@ -1952,6 +1960,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
|
||||
"vision_model.encoder.layer.{bid}.norm2", # Nemotron Omni
|
||||
"vision.blocks.{bid}.norm2", # deepseek4v
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm2",
|
||||
@@ -1979,6 +1988,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_UP: (
|
||||
"vision_model.encoder.layer.{bid}.mlp.fc1", # Nemotron Omni
|
||||
"vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1)
|
||||
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
|
||||
@@ -2016,6 +2026,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN: (
|
||||
"vision_model.encoder.layer.{bid}.mlp.fc2", # Nemotron Omni
|
||||
"vision.blocks.{bid}.mlp.w2", # deepseek4v
|
||||
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
|
||||
@@ -2074,12 +2085,14 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_LAYER_SCALE_1: (
|
||||
"vision_model.encoder.layer.{bid}.layer_scale1", # Nemotron Omni
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.ls1", # InternVL
|
||||
"model.vision_tower.encoder.layer.{bid}.lambda_1", # Intern-S1
|
||||
"vision_model.transformer.resblocks.{bid}.ls_1", # Step3-VL
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_LAYER_SCALE_2: (
|
||||
"vision_model.encoder.layer.{bid}.layer_scale2", # Nemotron Omni
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.ls2", # InternVL
|
||||
"model.vision_tower.encoder.layer.{bid}.lambda_2", # Intern-S1
|
||||
"vision_model.transformer.resblocks.{bid}.ls_2", # Step3-VL
|
||||
@@ -2114,6 +2127,7 @@ class TensorNameMap:
|
||||
"siglip2.vision_model.post_layernorm",
|
||||
"model.qwen2_model.model.model.norm", # Deepseek-OCR-2 qwen2
|
||||
"model.vision_tower.ln_post", # muse-glimmer
|
||||
"vision_projector.vision_final_layernorm", # Nemotron Omni Reasoning V3
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_MM_POST_NORM: (
|
||||
|
||||
@@ -930,7 +930,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
|
||||
res->t_inp_tokens = inp_tokens;
|
||||
|
||||
ggml_tensor * inp_anchor = nullptr;
|
||||
if (model.lilicorr_pass_hidden_proj && ubatch.embd_state) {
|
||||
if (model.lilicorr_pass_hidden_proj && ubatch.embd_state && n_tokens == (int64_t) hparams.dflash_block_size * ubatch.n_seqs_unq) {
|
||||
// LiliCorr sets inp_anchor to pass the context features to the lattice scorer
|
||||
auto inp = std::make_unique<llm_graph_input_embd_h>(n_embd, n_embd);
|
||||
inp->tokens = inp_tokens;
|
||||
@@ -1100,7 +1100,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
|
||||
|
||||
if (model.dflash_selector_hidden) {
|
||||
build_dflash2_selector(*this, model, inp_tokens);
|
||||
} else if (model.lilicorr_pass_hidden_proj && ubatch.embd_state) {
|
||||
} else if (model.lilicorr_pass_hidden_proj && ubatch.embd_state && n_tokens == (int64_t) hparams.dflash_block_size * ubatch.n_seqs_unq) {
|
||||
build_lilicorr_scores(*this, model, tok_embd, inp_anchor);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,21 +109,26 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {
|
||||
layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
|
||||
} else {
|
||||
if (n_expert != 0) {
|
||||
const auto * router_meta = ml.get_tensor_meta(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i).str().c_str());
|
||||
const int64_t n_expert_layer = router_meta ? router_meta->ne[1] : n_expert;
|
||||
if (n_expert_layer < hparams.n_expert_used(i) || n_expert_layer > n_expert) {
|
||||
throw std::runtime_error("Invalid per-layer Nemotron expert count");
|
||||
}
|
||||
// Use per-layer n_ff_exp; fall back to n_ff/n_expert_used if absent (existing GGUFs).
|
||||
const int64_t n_ff_exp_i = hparams.n_ff_exp(i)
|
||||
? (int64_t)hparams.n_ff_exp(i)
|
||||
: hparams.n_ff(i) / (int64_t)hparams.n_expert_used(i);
|
||||
const int64_t n_ff_shexp = hparams.n_ff_shexp;
|
||||
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, trunk_flags);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert }, trunk_flags);
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert_layer}, trunk_flags);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert_layer}, trunk_flags);
|
||||
|
||||
// MoE branch
|
||||
layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, TENSOR_NOT_REQUIRED);
|
||||
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert}, trunk_flags);
|
||||
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert}, trunk_flags);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert_layer}, trunk_flags);
|
||||
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert_layer}, trunk_flags);
|
||||
|
||||
// Shared expert branch
|
||||
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, trunk_flags);
|
||||
@@ -178,6 +183,15 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {
|
||||
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, mtp_flags);
|
||||
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, mtp_flags);
|
||||
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, mtp_flags);
|
||||
|
||||
layer.ffn_down_exps_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_up_exps_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_down_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "input_scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_up_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "input_scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_down_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_up_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_down_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "input_scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_up_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "input_scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,7 +318,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur,
|
||||
nullptr, // no gate
|
||||
model.layers[il].ffn_down_exps,
|
||||
model.layers[il].ffn_exp_probs_b,
|
||||
n_expert, (int64_t)hparams.n_expert_used(il),
|
||||
model.layers[il].ffn_up_exps->ne[2], (int64_t)hparams.n_expert_used(il),
|
||||
LLM_FFN_RELU_SQR, hparams.expert_weights_norm,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
|
||||
|
||||
@@ -46,6 +46,7 @@ add_library(mtmd
|
||||
models/kimivl.cpp
|
||||
models/kimik25.cpp
|
||||
models/nemotron-v2-vl.cpp
|
||||
models/nemotron-omni.cpp
|
||||
models/muse-glimmer.cpp
|
||||
models/llama4.cpp
|
||||
models/llava.cpp
|
||||
|
||||
@@ -498,6 +498,7 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_YASA2,
|
||||
PROJECTOR_TYPE_KIMIK25,
|
||||
PROJECTOR_TYPE_NEMOTRON_V2_VL,
|
||||
PROJECTOR_TYPE_NEMOTRON_OMNI,
|
||||
PROJECTOR_TYPE_HUNYUANVL,
|
||||
PROJECTOR_TYPE_PARAKEET,
|
||||
PROJECTOR_TYPE_EXAONE4_5,
|
||||
@@ -569,6 +570,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_YASA2, "yasa2"},
|
||||
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
|
||||
{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
|
||||
{ PROJECTOR_TYPE_NEMOTRON_OMNI, "nemotron_omni"},
|
||||
{ PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"},
|
||||
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
|
||||
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
|
||||
@@ -904,11 +906,13 @@ static void clip_log_internal(enum ggml_log_level level, const char * format, ..
|
||||
struct clip_image_f32_batch {
|
||||
std::vector<clip_image_f32> entries;
|
||||
bool is_audio = false;
|
||||
bool is_video = false;
|
||||
|
||||
clip_image_f32_batch clone() const {
|
||||
clip_image_f32_batch new_batch{
|
||||
/* entries */ {},
|
||||
/* is_audio */ is_audio,
|
||||
/* is_video */ is_video,
|
||||
};
|
||||
new_batch.entries.reserve(entries.size());
|
||||
for (const auto & entry : entries) {
|
||||
|
||||
@@ -566,7 +566,7 @@ struct clip_model {
|
||||
// embeddings
|
||||
ggml_tensor * class_embedding = nullptr;
|
||||
ggml_tensor * patch_embeddings_0 = nullptr;
|
||||
ggml_tensor * patch_embeddings_1 = nullptr; // second Conv2D kernel when we decouple Conv3D along temporal dimension (Qwen2VL)
|
||||
ggml_tensor * patch_embeddings_1 = nullptr; // additional patch kernel; interpretation is model-specific (Qwen2VL, Nemotron Omni)
|
||||
ggml_tensor * patch_bias = nullptr;
|
||||
ggml_tensor * position_embeddings = nullptr;
|
||||
ggml_tensor * norm_embd_w = nullptr;
|
||||
|
||||
@@ -1042,6 +1042,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
|
||||
{
|
||||
builder = std::make_unique<clip_graph_nemotron_v2_vl>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_nemotron_omni>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_LLAMA4:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_llama4>(ctx, img);
|
||||
@@ -1523,6 +1527,16 @@ struct clip_model_loader {
|
||||
{
|
||||
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
{
|
||||
hparams.ffn_op = FFN_GELU_ERF;
|
||||
log_ffn_op = "gelu_erf";
|
||||
hparams.image_resize_pad = PAD_NONE;
|
||||
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
|
||||
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
|
||||
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
|
||||
hparams.warmup_image_size = hparams.image_size;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
{
|
||||
get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor);
|
||||
@@ -3269,6 +3283,7 @@ struct clip_model_loader {
|
||||
model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
{
|
||||
model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
|
||||
model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
|
||||
@@ -4320,6 +4335,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
|
||||
case PROJECTOR_TYPE_COHERE2V:
|
||||
case PROJECTOR_TYPE_INTERNVL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
case PROJECTOR_TYPE_LLAMA4:
|
||||
{
|
||||
// both X and Y are downscaled by the scale factor
|
||||
@@ -5428,6 +5444,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
|
||||
case PROJECTOR_TYPE_COHERE2V:
|
||||
case PROJECTOR_TYPE_INTERNVL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
case PROJECTOR_TYPE_QWEN2A:
|
||||
case PROJECTOR_TYPE_QWEN3A:
|
||||
case PROJECTOR_TYPE_GLMA:
|
||||
@@ -6149,6 +6166,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
|
||||
return ctx->model.mm_3_w->ne[1]; // out_proj output dim
|
||||
case PROJECTOR_TYPE_INTERNVL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
return ctx->model.mm_3_w->ne[1];
|
||||
case PROJECTOR_TYPE_LLAMA4:
|
||||
return ctx->model.mm_model_proj->ne[1];
|
||||
@@ -6225,6 +6243,7 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {
|
||||
case PROJECTOR_TYPE_QWEN2VL:
|
||||
case PROJECTOR_TYPE_QWEN25VL:
|
||||
case PROJECTOR_TYPE_QWEN3VL:
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
case PROJECTOR_TYPE_LING3VL:
|
||||
return 2;
|
||||
default:
|
||||
|
||||
@@ -109,6 +109,12 @@ struct clip_graph_nemotron_v2_vl : clip_graph {
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_nemotron_omni : clip_graph {
|
||||
clip_graph_nemotron_omni(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * build_inp_with_temporal_merge();
|
||||
};
|
||||
|
||||
struct clip_graph_llama4 : clip_graph {
|
||||
clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() {
|
||||
if (n_batch == 1 && !img_batch->is_video) {
|
||||
return build_inp();
|
||||
}
|
||||
|
||||
GGML_ASSERT(n_batch == 1 || n_batch == 2);
|
||||
GGML_ASSERT(model.patch_embeddings_1 != nullptr);
|
||||
GGML_ASSERT(model.patch_embeddings_1->ne[2] == 6);
|
||||
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
if (n_batch == 1) {
|
||||
// pad the last video frame inside the graph
|
||||
inp_raw = ggml_concat(ctx0, inp_raw, inp_raw, 2);
|
||||
} else {
|
||||
inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.ny(), 6, 1);
|
||||
}
|
||||
|
||||
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, 1);
|
||||
return ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_nemotron_omni::build() {
|
||||
GGML_ASSERT(model.class_embedding != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
|
||||
const int n_registers = model.class_embedding->ne[1];
|
||||
const int n_pos = n_patches + n_registers;
|
||||
|
||||
ggml_tensor * inp = build_inp_with_temporal_merge();
|
||||
|
||||
inp = ggml_add(ctx0, inp, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
|
||||
cb(inp, "inp_pos", -1);
|
||||
|
||||
inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
|
||||
|
||||
ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, nullptr);
|
||||
|
||||
cur = ggml_view_2d(ctx0, cur,
|
||||
n_embd, n_patches,
|
||||
ggml_row_size(cur->type, n_embd),
|
||||
n_registers * ggml_row_size(cur->type, n_embd));
|
||||
|
||||
cur = build_patch_merge_permute(cur, model.hparams.n_merge);
|
||||
|
||||
{
|
||||
cur = build_norm(cur, model.mm_0_w, nullptr, NORM_TYPE_RMS, 1e-5, -1);
|
||||
cur = build_ffn(cur, model.mm_1_w, nullptr, nullptr, nullptr, model.mm_3_w, nullptr, FFN_RELU_SQR, -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -898,6 +898,12 @@ struct mtmd_context {
|
||||
{
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_NEMOTRON_OMNI:
|
||||
{
|
||||
img_beg = "<img>";
|
||||
img_end = "</img>";
|
||||
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_LFM2:
|
||||
{
|
||||
// multi-tile:
|
||||
@@ -1607,6 +1613,7 @@ struct mtmd_tokenizer {
|
||||
|
||||
clip_image_f32_batch batch_f32;
|
||||
batch_f32.is_audio = false;
|
||||
batch_f32.is_video = bitmaps[0]->mergeable;
|
||||
batch_f32.entries = std::move(preproc_out.entries);
|
||||
// do NOT use preproc_out from this point on, it's moved
|
||||
|
||||
|
||||
Reference in new issue
Block a user