From e853ec9a8c72e91e8fb0eaa68e1dfe02f3c396f0 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Wed, 4 Nov 2026 17:23:25 +0800 Subject: [PATCH] llama : follow ctx_other devices and flash attention in Gemma4 assistant (#31635) * llama : follow ctx_other devices and flash attention in Gemma4 assistant * llama : include for std::any_of --- src/llama-context.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index eb9756996..85c32c995 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -14,6 +14,7 @@ #include "llama-sampler.h" #include "llama.h" +#include #include #include #include @@ -332,6 +333,18 @@ llama_context::llama_context( cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED; cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO; + // the KV cache is shared with ctx_other, so follow its flash attention setting instead of resolving it here: + // the probe disables it when the layers and the shared KV are on different devices, which breaks a quantized V cache + if (model.arch == LLM_ARCH_GEMMA4_ASSISTANT) { + const bool flash_attn_other = cparams.ctx_other->get_cparams().flash_attn; + if (!cparams.auto_fa && cparams.flash_attn != flash_attn_other) { + LLAMA_LOG_WARN("%s: flash_attn = %d does not match the context the KV cache is shared with, using %d\n", + __func__, cparams.flash_attn, flash_attn_other); + } + cparams.flash_attn = flash_attn_other; + cparams.auto_fa = false; + } + cparams.fused_gdn_ar = true; cparams.fused_gdn_ch = true; cparams.auto_fgdn = false; @@ -459,6 +472,22 @@ llama_context::llama_context( backends.emplace_back(backend); } + // tensors of ctx_other (e.g. a shared KV cache) can be on devices that this model does not use + if (cparams.ctx_other) { + for (const auto & dev_other : llama_get_model(cparams.ctx_other)->devices) { + const bool found = std::any_of(model.devices.begin(), model.devices.end(), + [&](const llama_device & dev) { return dev.dev == dev_other.dev; }); + if (found) { + continue; + } + ggml_backend_t backend = ggml_backend_dev_init(dev_other.dev, nullptr); + if (backend == nullptr) { + throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev_other.dev))); + } + backends.emplace_back(backend); + } + } + // add ACCEL backends (such as BLAS) for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i);