qwen3 : bidirectional attention and dense_2 output projection (#30607)

* qwen3 : bidirectional attention and dense_2 output projection

Assisted-by: Grok

* convert : read use_bidirectional_attention in base

Assisted-by: Grok

* Update conversion/base.py

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
fzowlandSigbjørn Skjæret authored and GitHub committed 2026-10-19 09:29:02 +03:00
1 parent 2b57148b08
commit 412a363c65
4 files changed
+11 -1

No files matched your search

+1 -1
View File
@@ -1625,7 +1625,7 @@ class TextModel(ModelBase):
self.gguf_writer.add_head_count_kv(n_head_kv)
logger.info(f"gguf: key-value head count = {n_head_kv}")
if self.hparams.get("is_causal") is False:
if self.hparams.get("is_causal") is False or self.hparams.get("use_bidirectional_attention") is True:
self.gguf_writer.add_causal_attention(False)
logger.info("gguf: causal attention = False")
+5
View File
@@ -253,6 +253,11 @@ class Qwen3Model(Qwen2Model):
yield from super().modify_tensors(data_torch, name, bid)
return
# output embedding projection (voyage-4-nano)
if name == "linear.weight":
self.gguf_writer.add_embedding_length_out(data_torch.shape[0])
name = "dense_2_out.weight"
yield from super().modify_tensors(data_torch, name, bid)
+1
View File
@@ -2929,6 +2929,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.DENSE_2_OUT, # voyage-4-nano
],
MODEL_ARCH.QWEN3MOE: [
MODEL_TENSOR.TOKEN_EMBD,
+4
View File
@@ -28,6 +28,10 @@ void llama_model_qwen3::load_arch_tensors(llama_model_loader &) {
// output rerank head
cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);
// output embedding projection
dense_2_out_layers = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "weight"), {n_embd, hparams.n_embd_out()}, TENSOR_NOT_REQUIRED);
dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"), {hparams.n_embd_out() }, TENSOR_NOT_REQUIRED);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];