llama : follow ctx_other devices and flash attention in Gemma4 assistant (#31635)
* llama : follow ctx_other devices and flash attention in Gemma4 assistant * llama : include <algorithm> for std::any_of
This commit is contained in:
1 parent
feaf800060
commit
e853ec9a8c
1 file changed
+29
@@ -14,6 +14,7 @@
|
||||
#include "llama-sampler.h"
|
||||
#include "llama.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cinttypes>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
@@ -332,6 +333,18 @@ llama_context::llama_context(
|
||||
cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED;
|
||||
cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO;
|
||||
|
||||
// the KV cache is shared with ctx_other, so follow its flash attention setting instead of resolving it here:
|
||||
// the probe disables it when the layers and the shared KV are on different devices, which breaks a quantized V cache
|
||||
if (model.arch == LLM_ARCH_GEMMA4_ASSISTANT) {
|
||||
const bool flash_attn_other = cparams.ctx_other->get_cparams().flash_attn;
|
||||
if (!cparams.auto_fa && cparams.flash_attn != flash_attn_other) {
|
||||
LLAMA_LOG_WARN("%s: flash_attn = %d does not match the context the KV cache is shared with, using %d\n",
|
||||
__func__, cparams.flash_attn, flash_attn_other);
|
||||
}
|
||||
cparams.flash_attn = flash_attn_other;
|
||||
cparams.auto_fa = false;
|
||||
}
|
||||
|
||||
cparams.fused_gdn_ar = true;
|
||||
cparams.fused_gdn_ch = true;
|
||||
cparams.auto_fgdn = false;
|
||||
@@ -459,6 +472,22 @@ llama_context::llama_context(
|
||||
backends.emplace_back(backend);
|
||||
}
|
||||
|
||||
// tensors of ctx_other (e.g. a shared KV cache) can be on devices that this model does not use
|
||||
if (cparams.ctx_other) {
|
||||
for (const auto & dev_other : llama_get_model(cparams.ctx_other)->devices) {
|
||||
const bool found = std::any_of(model.devices.begin(), model.devices.end(),
|
||||
[&](const llama_device & dev) { return dev.dev == dev_other.dev; });
|
||||
if (found) {
|
||||
continue;
|
||||
}
|
||||
ggml_backend_t backend = ggml_backend_dev_init(dev_other.dev, nullptr);
|
||||
if (backend == nullptr) {
|
||||
throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev_other.dev)));
|
||||
}
|
||||
backends.emplace_back(backend);
|
||||
}
|
||||
}
|
||||
|
||||
// add ACCEL backends (such as BLAS)
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
||||
|
||||
Reference in new issue
Block a user