model: apply .scale tensors automatically + fix correctness (#32669)
* model: apply .scale tensors automatically * rm build_moe_ffn "_s" overload * use llama_prec_policy * fix ci * build_lora_mm_id: rm w_s
This commit is contained in:
1 parent
e50c254a22
commit
19bc41dbb4
39 files changed
+164
-290
No files matched your search
@@ -113,6 +113,11 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {
|
||||
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED | flags);
|
||||
}
|
||||
|
||||
// numerical issues with F16 accumulators
|
||||
prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
|
||||
prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
|
||||
prec_policy.set_acc(layer.ffn_down_shexp, GGML_PREC_BF16);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user