llama : follow ctx_other devices and flash attention in Gemma4 assistant (#31635)

* llama : follow ctx_other devices and flash attention in Gemma4 assistant

* llama : include <algorithm> for std::any_of
This commit is contained in:
Aman Gupta authored and GitHub committed 2026-11-04 11:23:25 +02:00
1 parent feaf800060
commit e853ec9a8c
1 file changed
+29
+29
View File
@@ -14,6 +14,7 @@
#include "llama-sampler.h"
#include "llama.h"
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <cstring>
@@ -332,6 +333,18 @@ llama_context::llama_context(
cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED;
cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO;
// the KV cache is shared with ctx_other, so follow its flash attention setting instead of resolving it here:
// the probe disables it when the layers and the shared KV are on different devices, which breaks a quantized V cache
if (model.arch == LLM_ARCH_GEMMA4_ASSISTANT) {
const bool flash_attn_other = cparams.ctx_other->get_cparams().flash_attn;
if (!cparams.auto_fa && cparams.flash_attn != flash_attn_other) {
LLAMA_LOG_WARN("%s: flash_attn = %d does not match the context the KV cache is shared with, using %d\n",
__func__, cparams.flash_attn, flash_attn_other);
}
cparams.flash_attn = flash_attn_other;
cparams.auto_fa = false;
}
cparams.fused_gdn_ar = true;
cparams.fused_gdn_ch = true;
cparams.auto_fgdn = false;
@@ -459,6 +472,22 @@ llama_context::llama_context(
backends.emplace_back(backend);
}
// tensors of ctx_other (e.g. a shared KV cache) can be on devices that this model does not use
if (cparams.ctx_other) {
for (const auto & dev_other : llama_get_model(cparams.ctx_other)->devices) {
const bool found = std::any_of(model.devices.begin(), model.devices.end(),
[&](const llama_device & dev) { return dev.dev == dev_other.dev; });
if (found) {
continue;
}
ggml_backend_t backend = ggml_backend_dev_init(dev_other.dev, nullptr);
if (backend == nullptr) {
throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev_other.dev)));
}
backends.emplace_back(backend);
}
}
// add ACCEL backends (such as BLAS)
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);