ggml-cpu : use all threads for large GET_ROWS (#32514)
This commit is contained in:
1 parent
98d67c8d53
commit
ff6b8d1761
2 files changed
+12
-4
No files matched your search
@@ -2378,10 +2378,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
} break;
|
||||
case GGML_OP_GET_ROWS:
|
||||
{
|
||||
// FIXME: get_rows can use additional threads, but the cost of launching additional threads
|
||||
// decreases performance with GPU offloading
|
||||
//n_tasks = n_threads;
|
||||
n_tasks = 1;
|
||||
// waking the other threads costs more than a small lookup, e.g. when this is the only CPU node of an offloaded model
|
||||
n_tasks = ggml_nelements(node) >= 262144 ? n_threads : 1;
|
||||
} break;
|
||||
case GGML_OP_SCALE:
|
||||
case GGML_OP_SET:
|
||||
|
||||
@@ -10813,6 +10813,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
for (int n : {1, 3, 257}) {
|
||||
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, n, 5, 3, 1, 1, false));
|
||||
}
|
||||
// many rows of a wide table, as a per-layer embedding lookup does
|
||||
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 100000, 32768, 1, 1, false));
|
||||
for (ggml_type type : all_types) {
|
||||
for (int b : {1, 7}) {
|
||||
for (bool v : {false, true}) {
|
||||
@@ -13609,6 +13611,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
|
||||
test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, n, 1.0f));
|
||||
}
|
||||
|
||||
// embedding lookups: one token (decode) and one batch of 2048 tokens (prefill)
|
||||
for (int r : {16, 32768}) {
|
||||
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, 160, 2000000, r, 1, 1, false)); // Qwen4-Exp n-gram table, 16 rows per token
|
||||
}
|
||||
for (int r : {1, 2048}) {
|
||||
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F16, 2560, 65536, r, 1, 1, false));
|
||||
}
|
||||
|
||||
// LoRA at prefill: B is [rank, n_out], so K = rank
|
||||
for (int64_t k : {16, 32, 48, 64}) {
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 2048, 512, k, {1, 1}, {1, 1}));
|
||||
|
||||
Reference in new issue
Block a user