llama: share the nextn tensor flags between models (#30097)
* llama: share the nextn tensor flags between models Follow-up of the TODO in glm5-next: move the trunk-only and MTP-only detection that each model copied into a nextn_flags helper of llama_model_base. It probes the first trunk layer and the first NextN layer, and adds TENSOR_SKIP when MTP is not loaded. qwen4exp probes hc_attn_norm since it has no attn_norm. deepseek4, nemotron-h, qwen35, qwen35moe, qwen3next and qwen4exp now also accept a trunk-only file, like the other models. * llama: avoid capturing structured bindings in the nextn flags Lambdas that capture structured bindings need C++20, and GCC 15 rejects them under -Werror, so the models read the trunk and MTP flags into plain variables.
This commit is contained in:
1 parent
988190680d
commit
448147d42a
18 files changed
+76
-123
No files matched your search
@@ -3461,6 +3461,27 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,
|
||||
}
|
||||
}
|
||||
|
||||
llama_model_base::nextn_flags_t llama_model_base::nextn_flags(llama_model_loader & ml, llm_tensor trunk_probe) const {
|
||||
int trunk = 0;
|
||||
int mtp = 0;
|
||||
|
||||
// a file without the first trunk layer is MTP-only, a file without the first NextN layer is trunk-only
|
||||
if (hparams.n_layer_nextn > 0) {
|
||||
if (ml.get_weight(tn(trunk_probe, "weight", 0).str().c_str()) == nullptr) {
|
||||
trunk = TENSOR_NOT_REQUIRED;
|
||||
}
|
||||
if (ml.get_weight(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", hparams.n_layer()).str().c_str()) == nullptr) {
|
||||
mtp = TENSOR_NOT_REQUIRED;
|
||||
}
|
||||
}
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp |= TENSOR_SKIP;
|
||||
}
|
||||
|
||||
return { trunk, mtp };
|
||||
}
|
||||
|
||||
void llama_model_base::load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first) {
|
||||
if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) {
|
||||
return;
|
||||
|
||||
@@ -849,6 +849,13 @@ struct llama_model_base : public llama_model {
|
||||
int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
|
||||
int flags);
|
||||
|
||||
// helper: tensor flags for a file that holds the trunk, the NextN layers, or both
|
||||
struct nextn_flags_t {
|
||||
int trunk; // TENSOR_NOT_REQUIRED when the file holds only the NextN layers
|
||||
int mtp; // TENSOR_NOT_REQUIRED when the file holds only the trunk, TENSOR_SKIP when MTP is not loaded
|
||||
};
|
||||
nextn_flags_t nextn_flags(llama_model_loader & ml, llm_tensor trunk_probe = LLM_TENSOR_ATTN_NORM) const;
|
||||
|
||||
// helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern
|
||||
void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false);
|
||||
|
||||
|
||||
@@ -64,15 +64,9 @@ void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader & ml) {
|
||||
const int64_t qk_head_dim = hparams.n_embd_head_k_mla();
|
||||
const int64_t v_head_dim = hparams.n_embd_head_v_mla();
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
auto & layer = layers[il];
|
||||
|
||||
@@ -40,18 +40,9 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {
|
||||
void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
// Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP
|
||||
// tensors live in a separate file. Mark MTP tensors NOT_REQUIRED so the
|
||||
// trunk loads cleanly.
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
|
||||
|
||||
|
||||
@@ -56,15 +56,9 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
const int64_t n_expert_shared = hparams.n_expert_shared;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
const bool is_mla = hparams.is_mla();
|
||||
|
||||
|
||||
@@ -46,15 +46,9 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) {
|
||||
void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
const bool is_mla = hparams.is_mla();
|
||||
if (!is_mla) {
|
||||
|
||||
@@ -93,9 +93,9 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) {
|
||||
const int64_t hc_dim = hc_mult * n_embd;
|
||||
const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult;
|
||||
|
||||
const bool mtp_only = (n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
const int mtp_flags = ml.load_mtp ? 0 : TENSOR_SKIP;
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
|
||||
|
||||
+3
-13
@@ -75,19 +75,9 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
const int64_t n_expert_shared = hparams.n_expert_shared;
|
||||
|
||||
// MTP-only: the GGUF carries only the NextN/MTP block(s) (user split target/draft).
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
// Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP
|
||||
// tensors live in a separate file (or were stripped at conversion). Mark
|
||||
// MTP tensors NOT_REQUIRED so the trunk loads cleanly.
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
const bool is_mla = hparams.is_mla();
|
||||
if (!is_mla) {
|
||||
|
||||
@@ -29,15 +29,9 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
const int64_t n_expert_shared = hparams.n_expert_shared;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers");
|
||||
GGML_ASSERT(hparams.n_expert_used() > 0 && "n_expert_used must be > 0 for GLM4_MOE MoE layers");
|
||||
|
||||
@@ -64,16 +64,9 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {
|
||||
const int64_t hc = hparams.dsv4_hc_mult;
|
||||
const int64_t hc_mix_dim = (2 + hc)*hc;
|
||||
|
||||
// TODO: consolidate the trunk_only/mtp_only detection and flags shared with glm4-moe and qwen35moe
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
|
||||
|
||||
+3
-12
@@ -22,18 +22,9 @@ void llama_model_hy_v3::load_arch_hparams(llama_model_loader & ml) {
|
||||
void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
// Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP
|
||||
// tensors live in a separate file (e.g. user split target/draft). Mark
|
||||
// MTP tensors NOT_REQUIRED so the trunk loads cleanly.
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
|
||||
|
||||
@@ -25,15 +25,9 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {
|
||||
void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
|
||||
|
||||
@@ -49,9 +49,9 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {
|
||||
void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const bool mtp_only = hparams.n_layer_nextn > 0 && ml.get_weight("blk.0.attn_norm.weight") == nullptr;
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
const int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
// mamba2 Mixer SSM params
|
||||
// NOTE: int64_t for tensor dimensions
|
||||
|
||||
@@ -33,9 +33,9 @@ void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) {
|
||||
void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
|
||||
|
||||
|
||||
@@ -36,9 +36,9 @@ void llama_model_qwen35moe::load_arch_hparams(llama_model_loader & ml) {
|
||||
void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
|
||||
|
||||
|
||||
@@ -35,9 +35,9 @@ void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) {
|
||||
throw std::runtime_error(arch_name() + " model cannot have zero experts");
|
||||
}
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
|
||||
|
||||
|
||||
@@ -175,10 +175,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
|
||||
const int64_t hc_dim = hc * n_embd;
|
||||
const int64_t hc_lr = hparams.hc_low_rank;
|
||||
|
||||
// an MTP-only file carries the MTP block, the embeddings and the LM head, but no trunk
|
||||
const bool mtp_only = n_layer_nextn > 0 && ml.get_weight(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", 0).str().c_str()) == nullptr;
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
const int mtp_flags = ml.load_mtp ? 0 : TENSOR_SKIP;
|
||||
const auto nf = nextn_flags(ml, LLM_TENSOR_HC_ATTN_NORM);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
|
||||
|
||||
|
||||
+3
-12
@@ -37,18 +37,9 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) {
|
||||
void llama_model_step35::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
|
||||
// Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP
|
||||
// tensors live in a separate file (e.g. user split target/draft). Mark
|
||||
// MTP tensors NOT_REQUIRED so the trunk loads cleanly.
|
||||
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
|
||||
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
|
||||
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
|
||||
|
||||
if (!ml.load_mtp) {
|
||||
mtp_flags |= TENSOR_SKIP;
|
||||
}
|
||||
const auto nf = nextn_flags(ml);
|
||||
const int trunk_flags = nf.trunk;
|
||||
const int mtp_flags = nf.mtp;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
|
||||
|
||||
Reference in new issue
Block a user