diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 66c51adde..15ec05408 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2185,6 +2185,10 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } + if (per_layer_tok_embd_s) { + ggml_backend_tensor_get(per_layer_tok_embd_s, &per_layer_tok_embd_scale, 0, sizeof(per_layer_tok_embd_scale)); + } + load_tensors_hadamard(); return true; diff --git a/src/llama-model.h b/src/llama-model.h index f3a665410..27c804eb6 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -882,6 +882,7 @@ struct llama_model { struct ggml_tensor * altup_unembd_proj = nullptr; struct ggml_tensor * per_layer_tok_embd = nullptr; struct ggml_tensor * per_layer_tok_embd_s = nullptr; // optional scalar scale of an FP8 table + float per_layer_tok_embd_scale = 1.0f; // value of per_layer_tok_embd_s, read after load // limite value embeddings struct ggml_tensor * value_embd = nullptr; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index faa4a928f..7bf9b3fee 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -217,6 +217,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { { hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY); // an FP8 table keeps its checkpoint bytes, so its per-tensor scale is applied after the gather per_layer_tok_embd_s = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "scale"), { 1 }, TENSOR_NOT_REQUIRED); + if (per_layer_tok_embd_s && per_layer_tok_embd_s->type != GGML_TYPE_F32) { + throw std::runtime_error(format("%s must be F32", per_layer_tok_embd_s->name)); + } } auto load_block = [&](int il, int flags) { @@ -1411,7 +1414,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple( // gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows); if (model.per_layer_tok_embd_s) { - emb = ggml_mul(ctx0, emb, model.per_layer_tok_embd_s); + emb = ggml_scale(ctx0, emb, model.per_layer_tok_embd_scale); } emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens); cb(emb, "ple_embd", -1);