Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Deepseek 4: -sm tensor (#26490)

Browse Source 
* DSV4: sm tensor

* set coarser granularity for head splits

* fix dspark

* add model saving for dsv4 + allow dflash to return on specific device

* add comment about dsv4 seq_rm

* simplify

* add shared expert delayed allreduce

* remove special test for dsv4
This commit is contained in:
Aman Gupta
2026-08-24 09:20:25 +03:00
committed by GitHub
parent c060ca974c
commit bf0a29cc16
7 changed files with 314 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-llama-archs.cpp
+27 -27
View File
@@ -102,10 +102,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_ff = 96;
n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
} else if (arch == LLM_ARCH_DEEPSEEK4) {
n_embd = 128;
n_head = 1;
n_ff = 192;
n_layer = 3; // uncompressed + csa + hca, one layer of each ratio kind
// head size 64 so that GPU flash attention kernels support the model
n_embd = 512;
n_head = 8;
n_ff = 1024;
n_layer = 4;
} else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) {
n_embd = 160; // exercise per-head tensor split granularity with head size 80
} else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
@@ -175,11 +176,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_kv);
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(1) : n_head_kv);
}
ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);
if (arch == LLM_ARCH_DEEPSEEK2
if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2);
} else if (arch == LLM_ARCH_DEEPSEEK2
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_DOTS3NOTE
@@ -208,10 +213,6 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
}
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);
}
} else if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, uint32_t(128));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(128));
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));
} else if (arch == LLM_ARCH_MINIMAX_M3) {
// partial rotary: n_rot must not exceed the indexer key length (64)
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));
@@ -221,7 +222,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(64) : uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, n_ctx/8);
@@ -248,26 +249,26 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
// MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the
// indexer head count is independent of the main attention head count.
if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 2.5f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(64));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 10000.0f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1e-6f);
ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 4, 128}));
}
ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(1));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 || arch == LLM_ARCH_DEEPSEEK4 ? n_head : uint32_t(1));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, uint32_t(64));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));
ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));
if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
ms.add_kv(LLM_KV_TOKENIZER_MODEL, "no_vocab");
// ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT, n_embd);
// ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd);
@@ -504,7 +505,6 @@ static bool arch_supported(const llm_arch arch) {
if (arch == LLM_ARCH_DEEPSEEK2OCR) {
return false;
}
// FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI.
#ifdef GGML_USE_WEBGPU
if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA) {