model : use exact GELU for ModernBERT encoders (#30108)
* model : use exact GELU for ModernBERT encoders Assisted-by: Codex * model : keep tanh GELU aliases on ggml_geglu Assisted-by: Claude Opus 5.5 * model : map gelu_python to ggml_geglu_erf Assisted-by: Claude Opus 5.5
This commit is contained in:
4 files changed
+22
-13
No files matched your search
@@ -17,10 +17,10 @@ void llama_model_modern_bert::load_arch_hparams(llama_model_loader & ml) {
|
||||
|
||||
// Some ModernBert derivatives (e.g. IBM Granite Embedding 97m R2) use
|
||||
// SiLU/SwiGLU in the FFN instead of the default GELU/GeGLU.
|
||||
hparams.llm_ffn_op = LLM_FFN_GEGLU;
|
||||
hparams.llm_ffn_op = LLM_FFN_GEGLU_ERF;
|
||||
std::string hidden_act;
|
||||
if (ml.get_key(LLM_KV_HIDDEN_ACT, hidden_act, false)) {
|
||||
hparams.llm_ffn_op = llm_ffn_op_type_from_string(hidden_act, LLM_FFN_GEGLU);
|
||||
hparams.llm_ffn_op = llm_ffn_op_type_from_string(hidden_act, LLM_FFN_GEGLU_ERF);
|
||||
}
|
||||
|
||||
// GGUFs without a classifier pooling type use mean (gte-reranker-modernbert-base)
|
||||
|
||||
Reference in new issue
Block a user