From ff6b8d176186ce1472a71800fcf3e6b221fcbb7b Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Wed, 18 Nov 2026 23:39:30 +0800 Subject: [PATCH] ggml-cpu : use all threads for large GET_ROWS (#32514) --- ggml/src/ggml-cpu/ggml-cpu.c | 6 ++---- tests/test-backend-ops.cpp | 10 ++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index a76af8487..30edcdbac 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2378,10 +2378,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { } break; case GGML_OP_GET_ROWS: { - // FIXME: get_rows can use additional threads, but the cost of launching additional threads - // decreases performance with GPU offloading - //n_tasks = n_threads; - n_tasks = 1; + // waking the other threads costs more than a small lookup, e.g. when this is the only CPU node of an offloaded model + n_tasks = ggml_nelements(node) >= 262144 ? n_threads : 1; } break; case GGML_OP_SCALE: case GGML_OP_SET: diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index fa1ffb853..b20e0d3fe 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10813,6 +10813,8 @@ static std::vector> make_test_cases_eval() { for (int n : {1, 3, 257}) { test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, n, 5, 3, 1, 1, false)); } + // many rows of a wide table, as a per-layer embedding lookup does + test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 100000, 32768, 1, 1, false)); for (ggml_type type : all_types) { for (int b : {1, 7}) { for (bool v : {false, true}) { @@ -13609,6 +13611,14 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, n, 1.0f)); } + // embedding lookups: one token (decode) and one batch of 2048 tokens (prefill) + for (int r : {16, 32768}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 2000000, r, 1, 1, false)); // Qwen4-Exp n-gram table, 16 rows per token + } + for (int r : {1, 2048}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_F16, 2560, 65536, r, 1, 1, false)); + } + // LoRA at prefill: B is [rank, n_out], so K = rank for (int64_t k : {16, 32, 48, 64}) { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 2048, 512, k, {1, 1}, {1, 1}));