Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

qwen4exp: fix seq_cp, block position keying, mtmd input, cuda abort, add tests (#27941)

Browse Source 
* qwen4exp: follow up fixes

* -kvu NaN collapse fix

Assisted-by: Claude

* indexer cache ext.x/ext.y restore fix

Assisted-by: Claude

* kv-cells: rename seq_set to seq_get_all

seq_get is already taken by the single-id getter, so the suggested name
cannot be overloaded on return type alone.

Assisted-by: Claude

* memory-hybrid-idx: implement set_input_qsa on the memory class

The context held the whole implementation, where the pattern elsewhere is a
thin context forwarding to the memory class, as llama_kv_cache_context does
for set_input_kq_mask. The body reads no context state, so it moves unchanged
and the context keeps a forwarder.

Also shortens the seq_get_all comment as suggested.

* tests: check that a sequence state survives a save/restore round-trip

Saves seq 0, erases it, restores the blob and saves again, requiring the two
blobs to match. Compares blobs rather than generated text, which cannot see a
field dropped on the way back in.

Note this passes on master for qwen4exp, so it does not demonstrate the
ext.x/ext.y drop this PR fixes; reaching that needs 2D mrope content.

* tests: give the synthetic qwen4exp a PLE so the state test bites

has_cell_ext() is n_pos_per_embd() > 1 || ple_n_heads > 0, and the indexer
cache sets rope_type = NONE, so without a PLE it serializes no cell ext at
all and the round-trip test cannot see a dropped ext.x/ext.y. With one,
removing the ext_set restore in state_read_meta fails the test: 198 of
335692 bytes differ, first at offset 282092.

Loading such a model needed two fixes:

- the row count of per_layer_token_embd came from require_weight(), which a
  model synthesised from metadata alone has no file to answer. Derive it
  from the head ranges and prefer the file's padded count where there is one.
- the PLE conv history is a row of the recurrent cache, so a PLE on a full
  attention layer dereferenced a null p_l. Reject it at load time instead.

The meta mirror is skipped for qwen4exp. It returned NaN logits before this
fixture carried a PLE, which the nmse check passes since a NaN comparison is
false, and aborts with one. -sm tensor on real devices works.

Assisted-by: Claude

* llama: disable -sm tensor for qwen4exp

test-llama-archs skipped the tensor split for this arch from inside the
test, so the arch still advertised support it does not have. Declare it in
llm_arch_supports_sm_tensor instead and drop the test-side exception; the
existing llm_arch_supports_sm_tensor branch then does the skipping.

Assisted-by: Claude
This commit is contained in:
Daniel Han
2026-09-01 13:22:04 +03:00
committed by GitHub
parent d086dbb348
commit 36b1015438
8 changed files with 529 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-llama-archs.cpp
+25
View File
@@ -254,6 +254,30 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, uint32_t(8));
// without this the QSA layers fall back to dense and go uncovered
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>(n_layer, 4));
// has_cell_ext() needs ple_n_heads here: the indexer cache serializes no ext without it
const uint32_t ple_ngram_size = 3;
const uint32_t ple_heads_per_ngram = 2;
const uint32_t ple_n_heads = (ple_ngram_size - 1)*ple_heads_per_ngram;
GGML_ASSERT(n_embd % ple_n_heads == 0);
const uint32_t ple_head_dim = n_embd/ple_n_heads;
std::vector<uint64_t> ple_head_offsets(ple_n_heads);
std::vector<uint64_t> ple_head_vocab_sizes(ple_n_heads, n_vocab);
for (uint32_t h = 0; h < ple_n_heads; h++) {
ple_head_offsets[h] = uint64_t(h)*n_vocab;
}
// the PLE history lives in the recurrent cache, so it must sit on a linear attention layer
ms.add_kv(LLM_KV_PLE_LAYERS, std::vector<uint32_t>({ 0 }));
ms.add_kv(LLM_KV_PLE_NGRAM_SIZE, ple_ngram_size);
ms.add_kv(LLM_KV_PLE_HEADS_PER_NGRAM, ple_heads_per_ngram);
ms.add_kv(LLM_KV_PLE_CONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_PLE_EOS_TOKEN_ID, uint32_t(0));
ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, ple_head_dim);
ms.add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector<uint64_t>({ 1, 3, 5 }));
ms.add_kv(LLM_KV_PLE_HEAD_OFFSETS, ple_head_offsets);
ms.add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, ple_head_vocab_sizes);
}
// minimax-m3 keeps one indexer head per GQA head; the rest use a fixed 64 to match the fused
@@ -709,6 +733,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
std::string status_nmse = "\033[1;33mSKIP\033[0m";
std::string status_roundtrip = "\033[1;33mSKIP\033[0m";
char nmse_str[12] = {0};
bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty());
if (!skip) {
if (logits_cpu.empty()) {
tests/test-save-load-state.cpp
+65 -1
View File
@@ -449,7 +449,66 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_
}
// Run the full save/load test suite (tests 1-7) for a single model.
// Test 8: state blob round-trip
// compares blobs rather than generated text: a partially restored cell still decodes to plausible tokens
static bool test_state_roundtrip(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens) {
auto params_ctx = common_context_params_to_llama(params);
auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};
LOG("\n=== Test 8: state blob round-trip ===\n");
if (llama_decode(ctx.get(), llama_batch_get_one(const_cast<llama_token *>(tokens.data()), (int32_t) tokens.size()))) {
LOG_ERR("\n%s: failed to decode prompt\n", __func__);
return false;
}
std::vector<uint8_t> blob_a(llama_state_seq_get_size(ctx.get(), 0));
const size_t n_a = llama_state_seq_get_data(ctx.get(), blob_a.data(), blob_a.size(), 0);
if (n_a != blob_a.size()) {
LOG_ERR("\n%s: saved %zu bytes, expected %zu\n", __func__, n_a, blob_a.size());
return false;
}
if (!llama_memory_seq_rm(llama_get_memory(ctx.get()), 0, -1, -1)) {
LOG_ERR("\n%s: failed to erase seq 0\n", __func__);
return false;
}
if (llama_state_seq_set_data(ctx.get(), blob_a.data(), blob_a.size(), 0) != blob_a.size()) {
LOG_ERR("\n%s: failed to restore seq 0\n", __func__);
return false;
}
std::vector<uint8_t> blob_b(llama_state_seq_get_size(ctx.get(), 0));
const size_t n_b = llama_state_seq_get_data(ctx.get(), blob_b.data(), blob_b.size(), 0);
if (n_b != n_a) {
LOG_ERR("\n%s: re-saved %zu bytes, expected %zu\n", __func__, n_b, n_a);
return false;
}
size_t n_diff = 0;
size_t i_diff = 0;
for (size_t i = 0; i < n_a; i++) {
if (blob_a[i] != blob_b[i]) {
if (n_diff == 0) {
i_diff = i;
}
n_diff++;
}
}
if (n_diff > 0) {
LOG_ERR("\n%s: state changed across a restore: %zu of %zu bytes differ, first at offset %zu\n",
__func__, n_diff, n_a, i_diff);
return false;
}
LOG("\nPASS\n");
return true;
}
// Run the full save/load test suite (tests 1-8) for a single model.
// Returns true if all tests pass, false otherwise.
static bool run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) {
struct common_params params = base_params;
@@ -526,6 +585,11 @@ static bool run_save_load_tests_for_model(const std::string & model_path, const
return false;
}
// Test 8: state blob round-trip
if (!test_state_roundtrip(model, params, tokens)) {
return false;
}
LOG("\nAll tests passed.\n");
return true;