Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Tensor parallel] Fix meta tensor split state propagation (#27574)

Browse Source 
* ggml : fix meta tensor split state propagation

* Add test-llama-archs to CI
This commit is contained in:
Gaurav Garg
2026-08-23 18:49:12 +08:00
committed by GitHub
parent 8144f3192e
commit d3371929bb
4 changed files with 113 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-llama-archs.cpp
+11 -1
View File
@@ -101,6 +101,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_head = 1;
n_ff = 96;
n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
} else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) {
n_embd = 160; // exercise per-head tensor split granularity with head size 80
} else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
n_head = 4;
} else if (arch == LLM_ARCH_DEEPSEEK2
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
@@ -120,6 +124,12 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_vocab = 4096; // must be >= the hard-coded codec head size (3072)
}
uint32_t n_head_kv = n_head;
if (arch == LLM_ARCH_QWEN3) {
n_head_kv = 1; // MQA coverage
} else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
n_head_kv = 2; // GQA coverage
}
const uint32_t n_embd_head = n_embd / n_head;
ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE, llm_arch_name(arch));
@@ -160,7 +170,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head);
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_kv);
}
ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);