Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

DeepseekV4: fix rollback with multi-seq (#26756)

Browse Source 
* DeepseekV4: fix rollback with multi-seq

* fix model loading

* make pending rollback single use

* only clear cache for seq_id for full load

* add assert for compress ratio

* make graph topology static

* pass true instead of flags in clear_compressed

* cont : clean-up + TODOs

---------

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
Aman Gupta
2026-08-23 13:57:49 +03:00
committed by GitHub
co-authored by Georgi Gerganov
parent d3371929bb
commit b0539c43ed
10 changed files with 345 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-llama-archs.cpp
+25 -4
View File
@@ -101,6 +101,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_head = 1;
n_ff = 96;
n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
} else if (arch == LLM_ARCH_DEEPSEEK4) {
n_embd = 128;
n_head = 1;
n_ff = 192;
n_layer = 3; // uncompressed + csa + hca, one layer of each ratio kind
} else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) {
n_embd = 160; // exercise per-head tensor split granularity with head size 80
} else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
@@ -203,6 +208,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
}
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);
}
} else if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, uint32_t(128));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(128));
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));
} else if (arch == LLM_ARCH_MINIMAX_M3) {
// partial rotary: n_rot must not exceed the indexer key length (64)
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));
@@ -239,6 +248,20 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
// MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the
// indexer head count is independent of the main attention head count.
if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 2.5f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(64));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 10000.0f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1e-6f);
ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 4, 128}));
}
ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(1));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, uint32_t(64));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
@@ -257,7 +280,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, uint32_t(2)); // sigmoid
ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid
ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));
}
@@ -395,6 +418,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_DEEPSEEK32:
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_GLM4_MOE:
case LLM_ARCH_GLM_DSA:
case LLM_ARCH_EXAONE_MOE:
@@ -480,9 +504,6 @@ static bool arch_supported(const llm_arch arch) {
if (arch == LLM_ARCH_DEEPSEEK2OCR) {
return false;
}
if (arch == LLM_ARCH_DEEPSEEK4) {
return false;
}
// FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI.
#ifdef GGML_USE_WEBGPU