qwen3 : bidirectional attention and dense_2 output projection (#30607)
* qwen3 : bidirectional attention and dense_2 output projection Assisted-by: Grok * convert : read use_bidirectional_attention in base Assisted-by: Grok * Update conversion/base.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> --------- Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
1 parent
2b57148b08
commit
412a363c65
4 files changed
+11
-1
No files matched your search
+1
-1
@@ -1625,7 +1625,7 @@ class TextModel(ModelBase):
|
||||
self.gguf_writer.add_head_count_kv(n_head_kv)
|
||||
logger.info(f"gguf: key-value head count = {n_head_kv}")
|
||||
|
||||
if self.hparams.get("is_causal") is False:
|
||||
if self.hparams.get("is_causal") is False or self.hparams.get("use_bidirectional_attention") is True:
|
||||
self.gguf_writer.add_causal_attention(False)
|
||||
logger.info("gguf: causal attention = False")
|
||||
|
||||
|
||||
@@ -253,6 +253,11 @@ class Qwen3Model(Qwen2Model):
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
return
|
||||
|
||||
# output embedding projection (voyage-4-nano)
|
||||
if name == "linear.weight":
|
||||
self.gguf_writer.add_embedding_length_out(data_torch.shape[0])
|
||||
name = "dense_2_out.weight"
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
|
||||
@@ -2929,6 +2929,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.FFN_GATE,
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
MODEL_TENSOR.DENSE_2_OUT, # voyage-4-nano
|
||||
],
|
||||
MODEL_ARCH.QWEN3MOE: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
|
||||
@@ -28,6 +28,10 @@ void llama_model_qwen3::load_arch_tensors(llama_model_loader &) {
|
||||
// output rerank head
|
||||
cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);
|
||||
|
||||
// output embedding projection
|
||||
dense_2_out_layers = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "weight"), {n_embd, hparams.n_embd_out()}, TENSOR_NOT_REQUIRED);
|
||||
dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"), {hparams.n_embd_out() }, TENSOR_NOT_REQUIRED);
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = layers[i];
|
||||
|
||||
|
||||
Reference in new issue
Block a user