ggml : add batched tensor copy API (#30881)

* ggml : add generic batched tensor copy API

Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* llama : use batched tensor copy API in context io

Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* ggml : add batched tensor copy API

Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* llama : batch device io tensor copies

Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* ggml : add CUDA support for batched tensor APIs

Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* ggml : remove async tensor_n APIs and add null buffer hooks

Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* ggml : fix batched tensor 2D detection

Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* ggml : group batched tensor operations by buffer type

Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* cont : minor indentantion

* cont : const correctness

* tests : add mock buffer type tests for batched tensor API

Restructure test-backend-tensor to run a unified test set on each backend buffer type and on an ad-hoc mock buffer type. The mock set exercises the dispatch logic in ggml_backend_tensor_{set,get,copy}_n: hook invocation, per-tensor fallback, cpy_tensor_n all-or-nothing, and buft grouping.

Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)

* cont : add TODO
This commit is contained in:
Georgi Gerganov authored and GitHub committed 2026-10-29 20:52:17 +02:00
1 parent eb4e244be7
commit 418f905a4e
24 files changed
+1381 -21

No files matched your search

+64 -19
View File
@@ -2926,13 +2926,31 @@ private:
class llama_io_write_host : public llama_io_write_i {
public:
llama_io_write_host(
uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
uint8_t * p, size_t len) : ptr(p), base_ptr(p), buf_size(len) {}
~llama_io_write_host() {
// TODO: add backend support to batch tensor_get? or some other way to speed this up
for (const auto & winfo : winfos) {
ggml_backend_tensor_get(winfo.tensor, winfo.ptr, winfo.offset, winfo.size);
if (winfos.empty()) {
return;
}
std::vector<ggml_tensor *> tensors;
std::vector<size_t> offsets;
std::vector<size_t> sizes;
std::vector<size_t> data_offsets;
tensors.reserve(winfos.size());
offsets.reserve(winfos.size());
sizes.reserve(winfos.size());
data_offsets.reserve(winfos.size());
for (const auto & winfo : winfos) {
tensors.push_back(winfo.tensor);
offsets.push_back(winfo.offset);
sizes.push_back(winfo.size);
data_offsets.push_back(winfo.data_offset);
}
std::vector<const ggml_tensor *> const_tensors(tensors.begin(), tensors.end());
ggml_backend_tensor_get_n(const_tensors.data(), base_ptr, offsets.data(), sizes.data(), data_offsets.data(), winfos.size());
}
void write(const void * src, size_t size) override {
@@ -2951,7 +2969,7 @@ public:
}
// save the write for later during destruction
winfos.push_back({tensor, ptr, size, offset});
winfos.push_back({tensor, (size_t) (ptr - base_ptr), size, offset});
ptr += size;
size_written += size;
@@ -2964,12 +2982,13 @@ public:
private:
uint8_t * ptr;
uint8_t * base_ptr;
size_t buf_size = 0;
size_t size_written = 0;
struct write_info {
ggml_tensor * tensor;
uint8_t * ptr;
size_t data_offset;
size_t size;
size_t offset;
};
@@ -2978,13 +2997,30 @@ private:
class llama_io_read_host : public llama_io_read_i {
public:
llama_io_read_host(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
llama_io_read_host(const uint8_t * p, size_t len) : ptr(p), base_ptr(p), buf_size(len) {}
~llama_io_read_host() {
// flush the reads
for (const auto & rinfo : rinfos) {
ggml_backend_tensor_set(rinfo.tensor, rinfo.ptr, rinfo.offset, rinfo.size);
if (rinfos.empty()) {
return;
}
std::vector<ggml_tensor *> tensors;
std::vector<size_t> offsets;
std::vector<size_t> sizes;
std::vector<size_t> data_offsets;
tensors.reserve(rinfos.size());
offsets.reserve(rinfos.size());
sizes.reserve(rinfos.size());
data_offsets.reserve(rinfos.size());
for (const auto & rinfo : rinfos) {
tensors.push_back(rinfo.tensor);
offsets.push_back(rinfo.offset);
sizes.push_back(rinfo.size);
data_offsets.push_back(rinfo.data_offset);
}
ggml_backend_tensor_set_n(tensors.data(), base_ptr, offsets.data(), sizes.data(), data_offsets.data(), rinfos.size());
}
void read(void * dst, size_t size) override {
@@ -3003,7 +3039,7 @@ public:
}
// save for later during destruction
rinfos.push_back({tensor, ptr, size, offset});
rinfos.push_back({tensor, (size_t) (ptr - base_ptr), size, offset});
ptr += size;
size_read += size;
@@ -3020,12 +3056,13 @@ public:
private:
const uint8_t * ptr;
const uint8_t * base_ptr;
size_t buf_size = 0;
size_t size_read = 0;
struct read_info {
ggml_tensor * tensor;
const uint8_t * ptr;
size_t data_offset;
size_t size;
size_t offset;
};
@@ -3173,9 +3210,9 @@ public:
}
}
for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
ggml_backend_tensor_copy(mbuf_cur.org[i], mbuf_cur.cpy[i]);
}
std::vector<const ggml_tensor *> srcs(mbuf_cur.org.begin(), mbuf_cur.org.end());
std::vector<ggml_tensor *> dsts(mbuf_cur.cpy.begin(), mbuf_cur.cpy.end());
ggml_backend_tensor_copy_n(srcs.data(), dsts.data(), mbuf_cur.org.size());
}
}
@@ -3272,9 +3309,9 @@ public:
if (same_chunking) {
// same chunking: copy 1:1 by index
for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);
}
std::vector<const ggml_tensor *> srcs(mbuf_cur.cpy.begin(), mbuf_cur.cpy.end());
std::vector<ggml_tensor *> dsts(mbuf.org.begin(), mbuf.org.end());
ggml_backend_tensor_copy_n(srcs.data(), dsts.data(), mbuf_cur.cpy.size());
continue;
}
}
@@ -3299,6 +3336,11 @@ public:
size_t src_base = 0;
size_t dst_base = 0;
std::vector<const ggml_tensor *> chunk_srcs;
std::vector<ggml_tensor *> chunk_dsts;
chunk_srcs.reserve(mbuf_cur.cpy.size() + mbuf.org.size());
chunk_dsts.reserve(mbuf_cur.cpy.size() + mbuf.org.size());
while (src_pos < total) {
const auto & src_t = mbuf_cur.cpy[src_j];
const auto & dst_t = mbuf.org[dst_i];
@@ -3319,7 +3361,8 @@ public:
auto * dst_v = ggml_view_1d(ctx_scratch, dst_t, n_el, dst_off);
ggml_backend_view_init(dst_v);
ggml_backend_tensor_copy(src_v, dst_v);
chunk_srcs.push_back(src_v);
chunk_dsts.push_back(dst_v);
src_pos += n_copy;
dst_pos += n_copy;
@@ -3343,6 +3386,8 @@ public:
GGML_ASSERT(ggml_nbytes(mbuf.org[i]) == 0);
}
ggml_backend_tensor_copy_n(chunk_srcs.data(), chunk_dsts.data(), chunk_srcs.size());
ggml_free(ctx_scratch);
}