ggml-cpu : use all threads for large GET_ROWS (#32514)

This commit is contained in:
Aman Gupta authored and GitHub committed 2026-11-18 23:39:30 +08:00
1 parent 98d67c8d53
commit ff6b8d1761
2 files changed
+12 -4

No files matched your search

+2 -4
View File
@@ -2378,10 +2378,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
} break;
case GGML_OP_GET_ROWS:
{
// FIXME: get_rows can use additional threads, but the cost of launching additional threads
// decreases performance with GPU offloading
//n_tasks = n_threads;
n_tasks = 1;
// waking the other threads costs more than a small lookup, e.g. when this is the only CPU node of an offloaded model
n_tasks = ggml_nelements(node) >= 262144 ? n_threads : 1;
} break;
case GGML_OP_SCALE:
case GGML_OP_SET:
+10
View File
@@ -10813,6 +10813,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
for (int n : {1, 3, 257}) {
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, n, 5, 3, 1, 1, false));
}
// many rows of a wide table, as a per-layer embedding lookup does
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 100000, 32768, 1, 1, false));
for (ggml_type type : all_types) {
for (int b : {1, 7}) {
for (bool v : {false, true}) {
@@ -13609,6 +13611,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, n, 1.0f));
}
// embedding lookups: one token (decode) and one batch of 2048 tokens (prefill)
for (int r : {16, 32768}) {
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 2000000, r, 1, 1, false)); // Qwen4-Exp n-gram table, 16 rows per token
}
for (int r : {1, 2048}) {
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F16, 2560, 65536, r, 1, 1, false));
}
// LoRA at prefill: B is [rank, n_out], so K = rank
for (int64_t k : {16, 32, 48, 64}) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 2048, 512, k, {1, 1}, {1, 1}));