kv-cache: fix restoring mismatched KV cache rotation by saving exact rotation metadata (#28498)
* kv-cache: save exact KV rotation metadata, reject restoring mismatched rotation * tests : move the state rotation test to test-save-load-state the test is now part of the save/load test matrix and runs against every model under test, like the rest of the suite it probes the KV cache type combinations supported by the model and treats models that do not use attention rotation as passing vacuously Assisted-by: pi:llama.cpp/Qwen3.8-27B * cont : skip unsupported KV caches --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
1 parent
e5983d6704
commit
210791069b
4 files changed
+140
-34
No files matched your search
@@ -744,6 +744,92 @@ static bool test_state_restore_failure(struct llama_model * model, const struct
|
||||
}
|
||||
|
||||
|
||||
// Test 10: state rotation
|
||||
// a KV state saved with attention rotation enabled must restore only into a context with the same setting;
|
||||
// note: rotation is only active for quantized KV caches with a head size that is a multiple of 64,
|
||||
// for other models the restore into the rotation-disabled context is valid and the test passes vacuously
|
||||
static bool test_state_rotation(struct llama_model * model, const struct common_params & params) {
|
||||
LOGV(LOG_LEVEL_INFO, "\n=== Test 10: state rotation ===\n");
|
||||
|
||||
const std::string attn_rot_disable = common_get_env("LLAMA_ATTN_ROT_DISABLE");
|
||||
const auto make_context = [&](ggml_type type_k, ggml_type type_v, bool disable_rotation) {
|
||||
common_set_env("LLAMA_ATTN_ROT_DISABLE", disable_rotation ? "1" : "0");
|
||||
auto params_ctx = common_context_params_to_llama(params);
|
||||
params_ctx.n_ctx = 32;
|
||||
params_ctx.n_batch = 1;
|
||||
params_ctx.n_ubatch = 1;
|
||||
params_ctx.type_k = type_k;
|
||||
params_ctx.type_v = type_v;
|
||||
params_ctx.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;
|
||||
return llama_context_ptr(llama_init_from_model(model, params_ctx));
|
||||
};
|
||||
|
||||
std::vector<std::pair<ggml_type, ggml_type>> type_pairs;
|
||||
for (const auto & types : { std::pair{GGML_TYPE_Q8_0, GGML_TYPE_Q8_0} }) {
|
||||
if (make_context(types.first, types.second, false)) {
|
||||
type_pairs.push_back(types);
|
||||
}
|
||||
}
|
||||
if (type_pairs.empty()) {
|
||||
LOG_WRN("%s: no supported quantized KV cache type combination - skipping\n", __func__);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool success = true;
|
||||
for (const auto & types : type_pairs) {
|
||||
auto src = make_context(types.first, types.second, false);
|
||||
if (!src) {
|
||||
LOG_ERR("%s: failed to create source context\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
llama_token token = 0;
|
||||
if (llama_decode(src.get(), llama_batch_get_one(&token, 1))) {
|
||||
LOG_ERR("%s: failed to decode token\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
const size_t state_size = llama_state_seq_get_size(src.get(), 0);
|
||||
if (state_size == 0) {
|
||||
continue; // no KV state to test
|
||||
}
|
||||
|
||||
std::vector<uint8_t> state(state_size);
|
||||
if (llama_state_seq_get_data(src.get(), state.data(), state.size(), 0) != state.size()) {
|
||||
LOG_ERR("%s: failed to save sequence state\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
auto matching = make_context(types.first, types.second, false);
|
||||
if (!matching || llama_state_seq_set_data(matching.get(), state.data(), state.size(), 0) != state.size()) {
|
||||
LOG_ERR("%s: failed to restore matching rotation\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
auto mismatched = make_context(types.first, types.second, true);
|
||||
if (!mismatched) {
|
||||
LOG_ERR("%s: failed to create mismatched rotation context\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
if (llama_state_seq_set_data(mismatched.get(), state.data(), state.size(), 0) != 0) {
|
||||
LOG_TRC("%s: state restored into rotation-disabled context, model does not use attention rotation\n", __func__);
|
||||
}
|
||||
}
|
||||
common_set_env("LLAMA_ATTN_ROT_DISABLE", attn_rot_disable);
|
||||
|
||||
if (!success) {
|
||||
return false;
|
||||
}
|
||||
|
||||
LOGV(LOG_LEVEL_INFO, "\nPASS\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
struct test_suite {
|
||||
std::vector<test_status> results;
|
||||
|
||||
@@ -754,10 +840,10 @@ struct test_suite {
|
||||
|
||||
// column headers for the --models table, one per test, in the order they are run
|
||||
static const std::vector<const char *> test_names = {
|
||||
"baseline", "seq_rm", "state_load", "cp_h", "cp_d", "cp_h_s", "cp_d_s", "rt", "rf",
|
||||
"baseline", "seq_rm", "state_load", "cp_h", "cp_d", "cp_h_s", "cp_d_s", "rt", "rf", "rot",
|
||||
};
|
||||
|
||||
// Run the full save/load test suite (tests 1-9) for a single model.
|
||||
// Run the full save/load test suite (tests 1-10) for a single model.
|
||||
// Returns the per-test results.
|
||||
static test_suite run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) {
|
||||
test_suite suite;
|
||||
@@ -836,6 +922,9 @@ static test_suite run_save_load_tests_for_model(const std::string & model_path,
|
||||
// Test 9: state restore failure
|
||||
suite.results.push_back(test_state_restore_failure(model, params, tokens) ? test_status::PASS : test_status::FAIL);
|
||||
|
||||
// Test 10: state rotation
|
||||
suite.results.push_back(test_state_rotation(model, params) ? test_status::PASS : test_status::FAIL);
|
||||
|
||||
return suite;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user