diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cpp index 08c7c6772..33ee7143a 100644 --- a/tests/test-save-load-state.cpp +++ b/tests/test-save-load-state.cpp @@ -13,6 +13,21 @@ constexpr double NMSE_THRESHOLD = 1e-5; +enum class test_status { + PASS, + FAIL, + SKIP, +}; + +static const char * test_status_str(test_status status) { + switch (status) { + case test_status::PASS: return "\033[1;32mPASS\033[0m"; + case test_status::FAIL: return "\033[1;31mFAIL\033[0m"; + case test_status::SKIP: return "\033[1;33mSKIP\033[0m"; + } + return ""; +} + // normalized mean squared error = mse(a, b) / mse(a, 0) static double nmse(const std::vector & a, const std::vector & b) { GGML_ASSERT(a.size() == b.size()); @@ -78,7 +93,8 @@ static generation_result generate_tokens(llama_context * ctx, llama_sampler * sm auto next_token = llama_sampler_sample(smpl, ctx, -1); - LOG("%d ", next_token); + // LOG_LEVEL_INFO gate: visible in single-model mode, silenced in --models table mode + LOGV(LOG_LEVEL_INFO, "%d ", next_token); result.tokens.push_back(next_token); result.logits.push_back(std::move(logits)); @@ -126,7 +142,7 @@ static bool generate_tokens_compare( const auto next_token = llama_sampler_sample(smpl, ctx, -1); const auto expected_token = expected.tokens[i]; - LOG("%d ", next_token); + LOGV(LOG_LEVEL_INFO, "%d ", next_token); if (next_token != expected_token) { LOG_TRC("%s: sampled token %d differs from expected %d, using expected token\n", __func__, next_token, expected_token); } @@ -164,14 +180,14 @@ static generation_result test_baseline(struct llama_model * model, const struct return {}; } - LOG("\n=== Test 1: baseline ===\n"); + LOGV(LOG_LEVEL_INFO, "\n=== Test 1: baseline ===\n"); auto result = generate_tokens(ctx.get(), smpl.get(), n_past, params.n_predict, 0); if (result.empty()) { return {}; } - LOG("\n"); + LOGV(LOG_LEVEL_INFO, "\n"); return result; } @@ -196,7 +212,7 @@ static bool test_seq_rm_isolated( return false; } - LOG("\n=== Test 2: sequence removal isolation ===\n"); + LOGV(LOG_LEVEL_INFO, "\n=== Test 2: sequence removal isolation ===\n"); const size_t n_tokens = tokens.size() < 128 ? tokens.size() : 128; for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) { @@ -249,7 +265,7 @@ static bool test_seq_rm_isolated( return false; } - LOG("PASS\n"); + LOGV(LOG_LEVEL_INFO, "PASS\n"); return true; } @@ -268,7 +284,7 @@ static bool test_state_load(struct llama_model * model, const struct common_para auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)}; llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed)); - LOG("\n=== Test 3: state load ===\n"); + LOGV(LOG_LEVEL_INFO, "\n=== Test 3: state load ===\n"); // Load state from file llama_tokens unused_sts(tokens.size()); @@ -293,7 +309,7 @@ static bool test_state_load(struct llama_model * model, const struct common_para return false; } - LOG("\nPASS\n"); + LOGV(LOG_LEVEL_INFO, "\nPASS\n"); return true; } @@ -313,7 +329,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_par auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)}; llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed)); - LOG("\n=== Test 4: seq copy (host) ===\n"); + LOGV(LOG_LEVEL_INFO, "\n=== Test 4: seq copy (host) ===\n"); // Load state from file llama_tokens unused_sts(tokens.size()); @@ -359,7 +375,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_par return false; } - LOG("\nPASS\n"); + LOGV(LOG_LEVEL_INFO, "\nPASS\n"); return true; } @@ -379,7 +395,7 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)}; llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed)); - LOG("\n=== Test 5: seq copy (device) ===\n"); + LOGV(LOG_LEVEL_INFO, "\n=== Test 5: seq copy (device) ===\n"); // Load state from file llama_tokens unused_sts(tokens.size()); @@ -425,7 +441,7 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p return false; } - LOG("\nPASS\n"); + LOGV(LOG_LEVEL_INFO, "\nPASS\n"); return true; } @@ -442,7 +458,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_ params_ctx.kv_unified = true; auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)}; - LOG("\n=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host"); + LOGV(LOG_LEVEL_INFO, "\n=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host"); const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE; @@ -519,7 +535,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_ return false; } - LOG("\nPASS\n"); + LOGV(LOG_LEVEL_INFO, "\nPASS\n"); return true; } @@ -530,7 +546,7 @@ static bool test_state_roundtrip(struct llama_model * model, const struct common auto params_ctx = common_context_params_to_llama(params); auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)}; - LOG("\n=== Test 8: state blob round-trip ===\n"); + LOGV(LOG_LEVEL_INFO, "\n=== Test 8: state blob round-trip ===\n"); if (llama_decode(ctx.get(), llama_batch_get_one(const_cast(tokens.data()), (int32_t) tokens.size()))) { LOG_ERR("\n%s: failed to decode prompt\n", __func__); @@ -578,14 +594,29 @@ static bool test_state_roundtrip(struct llama_model * model, const struct common return false; } - LOG("\nPASS\n"); + LOGV(LOG_LEVEL_INFO, "\nPASS\n"); return true; } +struct test_suite { + std::vector results; + + bool all_passed() const { + return std::all_of(results.begin(), results.end(), [](test_status s) { return s == test_status::PASS; }); + } +}; + +// column headers for the --models table, one per test, in the order they are run +static const std::vector test_names = { + "baseline", "seq_rm", "state_load", "cp_h", "cp_d", "cp_h_s", "cp_d_s", "rt", +}; + // Run the full save/load test suite (tests 1-8) for a single model. -// Returns true if all tests pass, false otherwise. -static bool run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) { +// Returns the per-test results. +static test_suite run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) { + test_suite suite; + struct common_params params = base_params; params.model.path = model_path; @@ -594,7 +625,8 @@ static bool run_save_load_tests_for_model(const std::string & model_path, const if (model == nullptr) { LOG_ERR("%s: failed to init model '%s'\n", __func__, model_path.c_str()); - return false; + suite.results.assign(test_names.size(), test_status::SKIP); + return suite; } GGML_ASSERT(llama_init->context() == nullptr); @@ -626,51 +658,48 @@ static bool run_save_load_tests_for_model(const std::string & model_path, const // Test 1: baseline (saves state to disk) auto result_baseline = test_baseline(model, params, tokens); - if (result_baseline.empty()) { - return false; - } + suite.results.push_back(result_baseline.empty() ? test_status::FAIL : test_status::PASS); // Test 2: sequence removal isolation - if (!test_seq_rm_isolated(model, params, tokens)) { - return false; - } + suite.results.push_back(test_seq_rm_isolated(model, params, tokens) ? test_status::PASS : test_status::FAIL); - // Test 3: state load - if (!test_state_load(model, params, tokens, result_baseline)) { - return false; - } + if (!result_baseline.empty()) { + // Test 3: state load + suite.results.push_back(test_state_load(model, params, tokens, result_baseline) ? test_status::PASS : test_status::FAIL); - // Test 4: seq copy (host) - if (!test_seq_cp_host(model, params, tokens, result_baseline)) { - return false; - } + // Test 4: seq copy (host) + suite.results.push_back(test_seq_cp_host(model, params, tokens, result_baseline) ? test_status::PASS : test_status::FAIL); - // Test 5: seq copy (device) - if (!test_seq_cp_device(model, params, tokens, result_baseline)) { - return false; + // Test 5: seq copy (device) + suite.results.push_back(test_seq_cp_device(model, params, tokens, result_baseline) ? test_status::PASS : test_status::FAIL); + } else { + // tests 3-5 depend on the baseline result and the state file it saves + suite.results.push_back(test_status::SKIP); + suite.results.push_back(test_status::SKIP); + suite.results.push_back(test_status::SKIP); } // Test 6: seq copy (host, scatter) - if (!test_seq_cp_scatter(model, params, tokens, 6, false)) { - return false; - } + suite.results.push_back(test_seq_cp_scatter(model, params, tokens, 6, false) ? test_status::PASS : test_status::FAIL); // Test 7: seq copy (device, scatter) - if (!test_seq_cp_scatter(model, params, tokens, 7, true)) { - return false; - } + suite.results.push_back(test_seq_cp_scatter(model, params, tokens, 7, true) ? test_status::PASS : test_status::FAIL); // Test 8: state blob round-trip - if (!test_state_roundtrip(model, params, tokens)) { - return false; - } + suite.results.push_back(test_state_roundtrip(model, params, tokens) ? test_status::PASS : test_status::FAIL); - LOG("\nAll tests passed.\n"); - - return true; + return suite; } +static void print_usage(int /* argc */, char ** argv) { + LOG("\nexample usage:\n"); + LOG("\n %s -m your_model.gguf\n", argv[0]); + LOG("\n %s --models tests/test-models\n", argv[0]); + LOG("\n %s -m your_model.gguf -lv 5\n", argv[0]); + LOG("\n"); +} + int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); @@ -707,7 +736,7 @@ int main(int argc, char ** argv) { params.model.path = models_dir; } - if (!common_params_parse(fargc, filtered_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + if (!common_params_parse(fargc, filtered_argv.data(), params, LLAMA_EXAMPLE_COMMON, print_usage)) { return 1; } @@ -742,27 +771,59 @@ int main(int argc, char ** argv) { return 1; } - LOG_INF("%s: running save/load tests over %zu models in '%s'\n", __func__, models.size(), models_dir.c_str()); + auto col_width = [](const char * name) { return (int) std::max(strlen(name), (size_t) 4); }; + + size_t name_width = 5; // "Model" + for (const auto & model_path : models) { + name_width = std::max(name_width, std::filesystem::path(model_path).filename().string().size()); + } + + // silence everything but the table itself (LOG has verbosity LOG_LEVEL_OUTPUT = 0) + common_log_set_verbosity_thold(0); + + LOG("%-*s", (int) name_width, "Model"); + for (const auto & name : test_names) { + LOG(" %-*s", col_width(name), name); + } + LOG("\n"); + common_log_flush(common_log_main()); size_t n_pass = 0; size_t n_fail = 0; for (const auto & model_path : models) { - LOG("\n================================================================\n"); - LOG_INF("%s: model %s\n", __func__, model_path.c_str()); + const auto name = std::filesystem::path(model_path).filename().string(); - if (run_save_load_tests_for_model(model_path, params)) { + LOG("%-*s", (int) name_width, name.c_str()); + common_log_flush(common_log_main()); + + const test_suite suite = run_save_load_tests_for_model(model_path, params); + + for (size_t i = 0; i < suite.results.size(); i++) { + LOG(" %s%*s", test_status_str(suite.results[i]), col_width(test_names[i]) - 4, ""); + } + LOG("\n"); + common_log_flush(common_log_main()); + + if (suite.all_passed()) { n_pass++; } else { n_fail++; } } - LOG("\n================================================================\n"); + common_log_set_verbosity_thold(LOG_DEFAULT_LLAMA); + common_log_flush(common_log_main()); + LOG_INF("%s: summary: %zu passed, %zu failed (of %zu)\n", __func__, n_pass, n_fail, models.size()); return n_fail == 0 ? 0 : 1; } // single-model mode - return run_save_load_tests_for_model(params.model.path, params) ? 0 : 1; + const test_suite suite = run_save_load_tests_for_model(params.model.path, params); + const bool all_passed = suite.all_passed(); + if (all_passed) { + LOG("\nAll tests passed.\n"); + } + return all_passed ? 0 : 1; }