From 7879491cf4c1cfb25acbd78fae34a9e7a8e3a062 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 20 Oct 2026 10:40:15 +0300 Subject: [PATCH] tests : make MoE router selection independent of the backend noise (#30818) * tests : make MoE router selection independent of the backend noise The router scores are in the range [0, 1), so a random draw can leave the last expert (or expert group) still selected and the first one dropped tied within the noise between the backends. They then route a token to different experts and the logits differ by 4e-3 instead of the usual 5e-7. Beam is the only arch here with a real top-k (8 experts, 2 used) and a group top-k (4 groups, 2 used), so it is the only one that hits this: 2 of 20 seeds failed on an M2 Ultra, the other MoE archs select 2 of 2 and cannot tie. Fill the selection bias with a ramp of step 2 to give every expert a score range of its own, which turns the top-k into a function of the expert index alone. The bias affects only the selection, the weights still come from the unbiased probs, so the MoE output keeps its magnitude. Beam now matches the CPU reference within 4e-7 for all of 160 seeds. Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) * tests : tell the CPU and the GPU of an Apple SoC apart in the device table Both devices report the same description ("Apple M2 Ultra" here), so the device column of the test table had two indistinguishable rows and a failing row did not say whether the Metal device or the host CPU produced it. Add the device name, which is also what -b takes, unless the description already is the name. Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) --- tests/test-llama-archs.cpp | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 94a22ff2f..21d27e17d 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -85,10 +85,18 @@ static void fill_resid_gate(std::vector & dst, const char * name, const t } } +// a random router bias can tie within the noise between the backends, a ramp keeps the top-k selection the same +static void fill_exp_probs_b(std::vector & dst, const char *, const tensor_data_params &) { + for (size_t i = 0; i < dst.size(); i++) { + dst[i] = 2.0f*i; + } +} + // a tensor whose values hold a property of their own, matched on a substring of its name, first match wins static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = { - { "ssm_a", fill_ssm_a }, - { "_resid_", fill_resid_gate }, + { "ssm_a", fill_ssm_a }, + { "_resid_", fill_resid_gate }, + { "exp_probs_b", fill_exp_probs_b }, }; static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) { @@ -934,6 +942,18 @@ static int save_models(const std::string & arch_filter, const size_t seed, const return 0; } +// the CPU and the GPU of an Apple SoC carry the same description, the name tells them apart (CPU, MTL0) +static std::string get_device_label(ggml_backend_dev_t dev) { + const char * desc = ggml_backend_dev_description(dev); + const char * name = ggml_backend_dev_name(dev); + + if (strcmp(desc, name) == 0) { + return desc; + } + + return std::string(desc) + " (" + name + ")"; +} + static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const char * target_backend) { struct user_data_t { struct { @@ -986,7 +1006,7 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con if (target_backend != nullptr && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) { continue; } - dev_configs.emplace_back(std::vector{dev}, ggml_backend_dev_description(dev), LLAMA_SPLIT_MODE_LAYER); + dev_configs.emplace_back(std::vector{dev}, get_device_label(dev), LLAMA_SPLIT_MODE_LAYER); max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length()); // cpu-based devices cannot be used in tensor split mode