From d9ee007a06b63e9b2380ba28663de4bcc7b60260 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 22 Oct 2026 16:49:38 +0300 Subject: [PATCH] model : fix qwen4exp layer inputs for DFlash drafters (#30927) qwen4exp published the hc-wide residual [n_embd, hc, n_tokens] as t_layer_inp, but the drafter reads layer inputs as [n_embd, n_tokens] rows: llama_context sizes each embd_layer_inp buffer as n_embd * n_batch and the DFlash encoder input is n_extract * n_embd wide. Evaluating a DFlash draft over a qwen4exp target therefore aborts on the first request in llama_context::extract_layer_inputs. Collapse the streams with a plain mean before exposing them, as deepseek4 already does for its hyper-connection residual, and build the mean only for the layers a drafter actually requested. Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) --- src/models/qwen4exp.cpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index ec0b438fa..fc9acd43e 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -308,6 +308,20 @@ std::unique_ptr llama_model_qwen4exp::build_arch_graph(const return std::make_unique(*this, params); } +// DFlash drafters read the layer inputs as [n_embd, T] rows, so collapse the streams +static ggml_tensor * hc_stream_mean(ggml_context * ctx, ggml_tensor * x) { + const int64_t n_embd = x->ne[0]; + const int64_t hc = x->ne[1]; + const int64_t nt = x->ne[2]; + + ggml_tensor * acc = ggml_view_2d(ctx, x, n_embd, nt, x->nb[2], 0); + for (int64_t s = 1; s < hc; ++s) { + acc = ggml_add(ctx, acc, ggml_view_2d(ctx, x, n_embd, nt, x->nb[2], s*x->nb[1])); + } + + return ggml_scale(ctx, acc, 1.0f / (float) hc); +} + // Hyper-connections keep hc parallel residual streams [n_embd, hc, T] in place of layer norms. // Returns the mixed [n_embd, T] stream; `inject` gets the [hc, T] scatter weights. ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( @@ -447,7 +461,11 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa ggml_build_forward_expand(gf, res_hc); for (int il = 0; il < n_layer; ++il) { - res->t_layer_inp[il] = res_hc; + if ((size_t) il < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[il]) { + res->t_layer_inp[il] = hc_stream_mean(ctx0, res_hc); + cb(res->t_layer_inp[il], "layer_inp", il); + ggml_build_forward_expand(gf, res->t_layer_inp[il]); + } if (hparams.is_ple(il)) { res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il);