qwen4exp : fix graph reallocation from the n-gram table scale (#32483)

* qwen4exp : apply the n-gram table scale with ggml_scale

* read the scale once after load

* remove comment
This commit is contained in:
Aman Gupta authored and GitHub committed 2026-11-18 00:35:54 +08:00
1 parent 955c4e8ee5
commit ad9d9e8f50
3 files changed
+9 -1

No files matched your search

+4
View File
@@ -2185,6 +2185,10 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
}
if (per_layer_tok_embd_s) {
ggml_backend_tensor_get(per_layer_tok_embd_s, &per_layer_tok_embd_scale, 0, sizeof(per_layer_tok_embd_scale));
}
load_tensors_hadamard();
return true;
+1
View File
@@ -882,6 +882,7 @@ struct llama_model {
struct ggml_tensor * altup_unembd_proj = nullptr;
struct ggml_tensor * per_layer_tok_embd = nullptr;
struct ggml_tensor * per_layer_tok_embd_s = nullptr; // optional scalar scale of an FP8 table
float per_layer_tok_embd_scale = 1.0f; // value of per_layer_tok_embd_s, read after load
// limite value embeddings
struct ggml_tensor * value_embd = nullptr;
+4 -1
View File
@@ -217,6 +217,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
{ hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY);
// an FP8 table keeps its checkpoint bytes, so its per-tensor scale is applied after the gather
per_layer_tok_embd_s = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "scale"), { 1 }, TENSOR_NOT_REQUIRED);
if (per_layer_tok_embd_s && per_layer_tok_embd_s->type != GGML_TYPE_F32) {
throw std::runtime_error(format("%s must be F32", per_layer_tok_embd_s->name));
}
}
auto load_block = [&](int il, int flags) {
@@ -1411,7 +1414,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(
// gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does
ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
if (model.per_layer_tok_embd_s) {
emb = ggml_mul(ctx0, emb, model.per_layer_tok_embd_s);
emb = ggml_scale(ctx0, emb, model.per_layer_tok_embd_scale);
}
emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens);
cb(emb, "ple_embd", -1);