ggml : add batched tensor copy API (#30881)
* ggml : add generic batched tensor copy API
Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* llama : use batched tensor copy API in context io
Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* ggml : add batched tensor copy API
Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* llama : batch device io tensor copies
Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* ggml : add CUDA support for batched tensor APIs
Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* ggml : remove async tensor_n APIs and add null buffer hooks
Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* ggml : fix batched tensor 2D detection
Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* ggml : group batched tensor operations by buffer type
Assisted-by: pi:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* cont : minor indentantion
* cont : const correctness
* tests : add mock buffer type tests for batched tensor API
Restructure test-backend-tensor to run a unified test set on each backend buffer type and on an ad-hoc mock buffer type. The mock set exercises the dispatch logic in ggml_backend_tensor_{set,get,copy}_n: hook invocation, per-tensor fallback, cpy_tensor_n all-or-nothing, and buft grouping.
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* cont : add TODO
This commit is contained in:
1 parent
eb4e244be7
commit
418f905a4e
24 files changed
+1381
-21
No files matched your search
+64
-19
@@ -2926,13 +2926,31 @@ private:
|
||||
class llama_io_write_host : public llama_io_write_i {
|
||||
public:
|
||||
llama_io_write_host(
|
||||
uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
|
||||
uint8_t * p, size_t len) : ptr(p), base_ptr(p), buf_size(len) {}
|
||||
|
||||
~llama_io_write_host() {
|
||||
// TODO: add backend support to batch tensor_get? or some other way to speed this up
|
||||
for (const auto & winfo : winfos) {
|
||||
ggml_backend_tensor_get(winfo.tensor, winfo.ptr, winfo.offset, winfo.size);
|
||||
if (winfos.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<ggml_tensor *> tensors;
|
||||
std::vector<size_t> offsets;
|
||||
std::vector<size_t> sizes;
|
||||
std::vector<size_t> data_offsets;
|
||||
tensors.reserve(winfos.size());
|
||||
offsets.reserve(winfos.size());
|
||||
sizes.reserve(winfos.size());
|
||||
data_offsets.reserve(winfos.size());
|
||||
|
||||
for (const auto & winfo : winfos) {
|
||||
tensors.push_back(winfo.tensor);
|
||||
offsets.push_back(winfo.offset);
|
||||
sizes.push_back(winfo.size);
|
||||
data_offsets.push_back(winfo.data_offset);
|
||||
}
|
||||
|
||||
std::vector<const ggml_tensor *> const_tensors(tensors.begin(), tensors.end());
|
||||
ggml_backend_tensor_get_n(const_tensors.data(), base_ptr, offsets.data(), sizes.data(), data_offsets.data(), winfos.size());
|
||||
}
|
||||
|
||||
void write(const void * src, size_t size) override {
|
||||
@@ -2951,7 +2969,7 @@ public:
|
||||
}
|
||||
|
||||
// save the write for later during destruction
|
||||
winfos.push_back({tensor, ptr, size, offset});
|
||||
winfos.push_back({tensor, (size_t) (ptr - base_ptr), size, offset});
|
||||
|
||||
ptr += size;
|
||||
size_written += size;
|
||||
@@ -2964,12 +2982,13 @@ public:
|
||||
|
||||
private:
|
||||
uint8_t * ptr;
|
||||
uint8_t * base_ptr;
|
||||
size_t buf_size = 0;
|
||||
size_t size_written = 0;
|
||||
|
||||
struct write_info {
|
||||
ggml_tensor * tensor;
|
||||
uint8_t * ptr;
|
||||
size_t data_offset;
|
||||
size_t size;
|
||||
size_t offset;
|
||||
};
|
||||
@@ -2978,13 +2997,30 @@ private:
|
||||
|
||||
class llama_io_read_host : public llama_io_read_i {
|
||||
public:
|
||||
llama_io_read_host(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
|
||||
llama_io_read_host(const uint8_t * p, size_t len) : ptr(p), base_ptr(p), buf_size(len) {}
|
||||
|
||||
~llama_io_read_host() {
|
||||
// flush the reads
|
||||
for (const auto & rinfo : rinfos) {
|
||||
ggml_backend_tensor_set(rinfo.tensor, rinfo.ptr, rinfo.offset, rinfo.size);
|
||||
if (rinfos.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<ggml_tensor *> tensors;
|
||||
std::vector<size_t> offsets;
|
||||
std::vector<size_t> sizes;
|
||||
std::vector<size_t> data_offsets;
|
||||
tensors.reserve(rinfos.size());
|
||||
offsets.reserve(rinfos.size());
|
||||
sizes.reserve(rinfos.size());
|
||||
data_offsets.reserve(rinfos.size());
|
||||
|
||||
for (const auto & rinfo : rinfos) {
|
||||
tensors.push_back(rinfo.tensor);
|
||||
offsets.push_back(rinfo.offset);
|
||||
sizes.push_back(rinfo.size);
|
||||
data_offsets.push_back(rinfo.data_offset);
|
||||
}
|
||||
|
||||
ggml_backend_tensor_set_n(tensors.data(), base_ptr, offsets.data(), sizes.data(), data_offsets.data(), rinfos.size());
|
||||
}
|
||||
|
||||
void read(void * dst, size_t size) override {
|
||||
@@ -3003,7 +3039,7 @@ public:
|
||||
}
|
||||
|
||||
// save for later during destruction
|
||||
rinfos.push_back({tensor, ptr, size, offset});
|
||||
rinfos.push_back({tensor, (size_t) (ptr - base_ptr), size, offset});
|
||||
|
||||
ptr += size;
|
||||
size_read += size;
|
||||
@@ -3020,12 +3056,13 @@ public:
|
||||
|
||||
private:
|
||||
const uint8_t * ptr;
|
||||
const uint8_t * base_ptr;
|
||||
size_t buf_size = 0;
|
||||
size_t size_read = 0;
|
||||
|
||||
struct read_info {
|
||||
ggml_tensor * tensor;
|
||||
const uint8_t * ptr;
|
||||
size_t data_offset;
|
||||
size_t size;
|
||||
size_t offset;
|
||||
};
|
||||
@@ -3173,9 +3210,9 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
|
||||
ggml_backend_tensor_copy(mbuf_cur.org[i], mbuf_cur.cpy[i]);
|
||||
}
|
||||
std::vector<const ggml_tensor *> srcs(mbuf_cur.org.begin(), mbuf_cur.org.end());
|
||||
std::vector<ggml_tensor *> dsts(mbuf_cur.cpy.begin(), mbuf_cur.cpy.end());
|
||||
ggml_backend_tensor_copy_n(srcs.data(), dsts.data(), mbuf_cur.org.size());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3272,9 +3309,9 @@ public:
|
||||
|
||||
if (same_chunking) {
|
||||
// same chunking: copy 1:1 by index
|
||||
for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
|
||||
ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);
|
||||
}
|
||||
std::vector<const ggml_tensor *> srcs(mbuf_cur.cpy.begin(), mbuf_cur.cpy.end());
|
||||
std::vector<ggml_tensor *> dsts(mbuf.org.begin(), mbuf.org.end());
|
||||
ggml_backend_tensor_copy_n(srcs.data(), dsts.data(), mbuf_cur.cpy.size());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -3299,6 +3336,11 @@ public:
|
||||
size_t src_base = 0;
|
||||
size_t dst_base = 0;
|
||||
|
||||
std::vector<const ggml_tensor *> chunk_srcs;
|
||||
std::vector<ggml_tensor *> chunk_dsts;
|
||||
chunk_srcs.reserve(mbuf_cur.cpy.size() + mbuf.org.size());
|
||||
chunk_dsts.reserve(mbuf_cur.cpy.size() + mbuf.org.size());
|
||||
|
||||
while (src_pos < total) {
|
||||
const auto & src_t = mbuf_cur.cpy[src_j];
|
||||
const auto & dst_t = mbuf.org[dst_i];
|
||||
@@ -3319,7 +3361,8 @@ public:
|
||||
auto * dst_v = ggml_view_1d(ctx_scratch, dst_t, n_el, dst_off);
|
||||
ggml_backend_view_init(dst_v);
|
||||
|
||||
ggml_backend_tensor_copy(src_v, dst_v);
|
||||
chunk_srcs.push_back(src_v);
|
||||
chunk_dsts.push_back(dst_v);
|
||||
|
||||
src_pos += n_copy;
|
||||
dst_pos += n_copy;
|
||||
@@ -3343,6 +3386,8 @@ public:
|
||||
GGML_ASSERT(ggml_nbytes(mbuf.org[i]) == 0);
|
||||
}
|
||||
|
||||
ggml_backend_tensor_copy_n(chunk_srcs.data(), chunk_dsts.data(), chunk_srcs.size());
|
||||
|
||||
ggml_free(ctx_scratch);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user