diff --git a/ggml/src/ggml-cuda/batch-get.cu b/ggml/src/ggml-cuda/batch-copy.cu similarity index 58% rename from ggml/src/ggml-cuda/batch-get.cu rename to ggml/src/ggml-cuda/batch-copy.cu index 60c4747b9..920b2b15d 100644 --- a/ggml/src/ggml-cuda/batch-get.cu +++ b/ggml/src/ggml-cuda/batch-copy.cu @@ -1,17 +1,17 @@ -#include "batch-get.cuh" +#include "batch-copy.cuh" #include "common.cuh" #include "ggml-cpp.h" #include #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) -struct ggml_cuda_batch_get_descriptor { +struct ggml_cuda_batch_copy_descriptor { const unsigned char * src; unsigned char * dst; size_t size; }; -static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) { +static __global__ void ggml_cuda_batch_copy_kernel(const ggml_cuda_batch_copy_descriptor * copies, size_t count) { // Blocks along y cooperate on each descriptor. const size_t index = blockIdx.x; if (index >= count) { @@ -50,11 +50,11 @@ static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_desc } } -struct ggml_cuda_batch_get_mapping { +struct ggml_cuda_batch_copy_mapping { void * host = nullptr; unsigned char * device = nullptr; - ggml_cuda_batch_get_mapping(void * host, size_t size) { + ggml_cuda_batch_copy_mapping(void * host, size_t size) { const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable); if (err != cudaSuccess) { (void) cudaGetLastError(); @@ -70,10 +70,10 @@ struct ggml_cuda_batch_get_mapping { } } - ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete; - ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete; + ggml_cuda_batch_copy_mapping(const ggml_cuda_batch_copy_mapping &) = delete; + ggml_cuda_batch_copy_mapping & operator=(const ggml_cuda_batch_copy_mapping &) = delete; - ~ggml_cuda_batch_get_mapping() { + ~ggml_cuda_batch_copy_mapping() { if (host) { // Release only the registration acquired by this object. CUDA_CHECK(cudaHostUnregister(host)); @@ -82,11 +82,12 @@ struct ggml_cuda_batch_get_mapping { }; #endif -bool ggml_cuda_get_tensor_n(int device, - const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, +static bool ggml_cuda_copy_tensor_n(int device, cudaMemcpyKind direction, + const ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) { #if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) GGML_UNUSED(device); + GGML_UNUSED(direction); GGML_UNUSED(tensors); GGML_UNUSED(data); GGML_UNUSED(offsets); @@ -94,14 +95,17 @@ bool ggml_cuda_get_tensor_n(int device, GGML_UNUSED(n_tensors); return false; #else + const bool to_host = direction == cudaMemcpyDeviceToHost; // Provisional dispatch values based on experiments. constexpr size_t min_batch_reads = 128; - constexpr size_t max_gather_read_bytes = 128 * 1024; - // Collect non-empty CPU destination ranges. + constexpr size_t max_kernel_read_bytes = 128 * 1024; + // Collect destination ranges; repeated sources are allowed. std::vector> ranges; ranges.reserve(n_tensors); size_t payload_bytes = 0; - // Number of non-empty reads whose size is at most max_gather_read_bytes. + uintptr_t begin = UINTPTR_MAX; + uintptr_t end = 0; + // Number of non-empty reads whose size is at most max_kernel_read_bytes. size_t n_reads_small = 0; for (size_t i = 0; i < n_tensors; ++i) { if (sizes[i] == 0) { @@ -111,15 +115,19 @@ bool ggml_cuda_get_tensor_n(int device, GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]); GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]); payload_bytes += sizes[i]; - n_reads_small += sizes[i] <= max_gather_read_bytes; - ranges.emplace_back(address, address + sizes[i]); + n_reads_small += sizes[i] <= max_kernel_read_bytes; + begin = std::min(begin, address); + end = std::max(end, address + sizes[i]); + const uintptr_t destination = to_host ? address : (uintptr_t) tensors[i]->data + offsets[i]; + GGML_ASSERT(destination <= UINTPTR_MAX - sizes[i]); + ranges.emplace_back(destination, destination + sizes[i]); } if (ranges.empty()) { return true; } // Total number of non-empty reads. const size_t n_reads = ranges.size(); - if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) { + if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_copy_descriptor) || n_reads > INT_MAX) { return false; } if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) { @@ -127,72 +135,85 @@ bool ggml_cuda_get_tensor_n(int device, } std::sort(ranges.begin(), ranges.end()); for (size_t i = 1; i < ranges.size(); ++i) { - // Preserve the ordered copy path when CPU destinations overlap. + // Preserve the ordered copy path when destinations overlap. if (ranges[i].first < ranges[i - 1].second) { return false; } } - // Find the CPU span to register, including gaps between destinations. - const uintptr_t begin = ranges.front().first; - const uintptr_t end = ranges.back().second; - if (end - begin - payload_bytes > payload_bytes) { + // Limit gaps in the CPU span without underflow when sources overlap. + if (end - begin > payload_bytes && end - begin - payload_bytes > payload_bytes) { return false; } - // Register and map the existing CPU destination so the kernel can write to it directly. - ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin); + // Register and map the CPU span for direct kernel access. + ggml_cuda_batch_copy_mapping mapping((void *) begin, end - begin); if (!mapping.host) { return false; } - // Reads selected for gather; zero means all reads use DMA. - size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0; + // Reads selected for the kernel; zero means all reads use DMA. + size_t n_reads_kernel = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0; // Keep descriptor storage alive until the final stream synchronization. constexpr size_t max_batch_descriptors = 128 * 1024; - const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor); - ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr); + const size_t bytes = std::min(n_reads_kernel, max_batch_descriptors) * sizeof(ggml_cuda_batch_copy_descriptor); + ggml_backend_buffer_ptr descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr); if (!descriptors) { - n_reads_gather = 0; + n_reads_kernel = 0; } const auto host_type = ggml_backend_cuda_host_buffer_type(); - ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr); + ggml_backend_buffer_ptr host_descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr); if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) { - n_reads_gather = 0; + n_reads_kernel = 0; } - if (n_reads_gather) { - auto * copies = static_cast(ggml_backend_buffer_get_base(host_descriptors.get())); - auto * device_copies = static_cast(ggml_backend_buffer_get_base(descriptors.get())); + if (n_reads_kernel) { + auto * copies = static_cast(ggml_backend_buffer_get_base(host_descriptors.get())); + auto * device_copies = static_cast(ggml_backend_buffer_get_base(descriptors.get())); const int threads = 256; const size_t nsm = ggml_cuda_info().devices[device].nsm; size_t i = 0; - for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) { - const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors); + for (size_t first = 0; first < n_reads_kernel; first += max_batch_descriptors) { + const size_t n_batch = std::min(n_reads_kernel - first, max_batch_descriptors); size_t count = 0; for (; i < n_tensors && count < n_batch; ++i) { - if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) { + if (sizes[i] == 0 || sizes[i] > max_kernel_read_bytes) { continue; } - copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i], - mapping.device + ((uintptr_t) data[i] - begin), sizes[i]}; + auto * host = mapping.device + ((uintptr_t) data[i] - begin); + auto * tensor = (unsigned char *) tensors[i]->data + offsets[i]; + copies[count++] = {to_host ? tensor : host, to_host ? host : tensor, sizes[i]}; } const size_t blocks_per_copy = std::min(UINT16_MAX, std::max(1, (nsm + n_batch - 1) / n_batch)); const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy); // Stream ordering keeps each kernel ahead of the next descriptor overwrite. CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread)); - ggml_cuda_batch_get_kernel<<>>(device_copies, n_batch); + ggml_cuda_batch_copy_kernel<<>>(device_copies, n_batch); CUDA_CHECK(cudaGetLastError()); - if (first + n_batch < n_reads_gather) { + if (first + n_batch < n_reads_kernel) { // Wait before refilling the pinned host descriptor buffer. CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); } } } - // Keep the destination pinned for DMA too; overlapping destinations already declined above. + // Keep the CPU span pinned for DMA too; overlapping destinations already declined above. for (size_t i = 0; i < n_tensors; ++i) { - if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) { - CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread)); + if (sizes[i] && (!n_reads_kernel || sizes[i] > max_kernel_read_bytes)) { + void * host = const_cast(data[i]); + void * tensor = (char *) tensors[i]->data + offsets[i]; + CUDA_CHECK(cudaMemcpyAsync(to_host ? host : tensor, to_host ? tensor : host, sizes[i], direction, cudaStreamPerThread)); } } - // Wait for GPU writes before releasing the CPU mapping and descriptor buffers. + // Wait for transfers before releasing the CPU mapping and descriptor buffers. CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); return true; #endif } + +bool ggml_cuda_get_tensor_n(int device, + const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors) { + return ggml_cuda_copy_tensor_n(device, cudaMemcpyDeviceToHost, tensors, data, offsets, sizes, n_tensors); +} + +bool ggml_cuda_set_tensor_n(int device, + ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors) { + return ggml_cuda_copy_tensor_n(device, cudaMemcpyHostToDevice, tensors, data, offsets, sizes, n_tensors); +} diff --git a/ggml/src/ggml-cuda/batch-get.cuh b/ggml/src/ggml-cuda/batch-copy.cuh similarity index 55% rename from ggml/src/ggml-cuda/batch-get.cuh rename to ggml/src/ggml-cuda/batch-copy.cuh index eb9bf0cc8..bb7fca4c0 100644 --- a/ggml/src/ggml-cuda/batch-get.cuh +++ b/ggml/src/ggml-cuda/batch-copy.cuh @@ -5,3 +5,7 @@ bool ggml_cuda_get_tensor_n(int device, const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors); + +bool ggml_cuda_set_tensor_n(int device, + ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index c6622d1b3..2ce3d0b80 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -10,7 +10,7 @@ #include "ggml-cuda/arange.cuh" #include "ggml-cuda/argmax.cuh" #include "ggml-cuda/argsort.cuh" -#include "ggml-cuda/batch-get.cuh" +#include "ggml-cuda/batch-copy.cuh" #include "ggml-cuda/binbcast.cuh" #include "ggml-cuda/clamp.cuh" #include "ggml-cuda/col2im-1d.cuh" @@ -1117,6 +1117,11 @@ static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer, ggml_cuda_set_device(ctx->device); + if (!ggml_cuda_pinned_memory_disabled() && + ggml_cuda_set_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) { + return; + } + for (size_t i = 0; i < n_tensors; i++) { if (sizes[i] == 0) { continue; diff --git a/tests/test-backend-tensor.cpp b/tests/test-backend-tensor.cpp index 9685bc277..ed617939d 100644 --- a/tests/test-backend-tensor.cpp +++ b/tests/test-backend-tensor.cpp @@ -170,6 +170,40 @@ static bool test_tensor_n(ggml_backend_buffer_type_t buft) { } } + { + std::vector batch; + ggml_backend_buffer_ptr scatter_buf = alloc_tensors(ctx.get(), buft, { 64 }, batch); + if (!scatter_buf) { + return false; + } + batch.resize(128, batch[0]); + uint8_t input[32], expected[256], out[256]; + for (size_t i = 0; i < sizeof(input); i++) { + input[i] = (uint8_t) (i + 1); + } + std::vector offsets(batch.size()), sizes(batch.size(), 1), data_offsets(batch.size()); + for (bool overlap : { false, true }) { + memset(expected, 0, sizeof(expected)); + ggml_backend_tensor_set(batch[0], expected, 0, sizeof(expected)); + for (size_t i = 0; i < batch.size(); i++) { + offsets[i] = 2*((i*5) % batch.size()) + 1; + data_offsets[i] = i % sizeof(input); + } + // Repeated sources are valid; overlapping destinations must retain copy order. + if (overlap) { + offsets.back() = offsets.front(); + } + for (size_t i = 0; i < batch.size(); i++) { + expected[offsets[i]] = input[data_offsets[i]]; + } + ggml_backend_tensor_set_n(batch.data(), input, offsets.data(), sizes.data(), data_offsets.data(), batch.size()); + ggml_backend_tensor_get(batch[0], out, 0, sizeof(out)); + if (memcmp(out, expected, sizeof(out)) != 0) { + return false; + } + } + } + return true; } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 36082df89..f821ac6cb 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -353,6 +353,8 @@ struct server_slot { return false; } + const int64_t t_start = ggml_time_us(); + const size_t cur_size_tgt = llama_state_seq_get_size_ext(ctx_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE); const size_t cur_size_dft = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0; @@ -374,6 +376,8 @@ struct server_slot { return false; } + SLT_TRC(*this, "prompt cache save took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0); + return true; } diff --git a/tools/server/server-slot-sched.cpp b/tools/server/server-slot-sched.cpp index 5f8aa5c33..9f9779348 100644 --- a/tools/server/server-slot-sched.cpp +++ b/tools/server/server-slot-sched.cpp @@ -208,6 +208,8 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok if (it_best != states.end()) { SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); + const int64_t t_start = ggml_time_us(); + { auto & data = it_best->data.main; @@ -252,6 +254,8 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok prompt = std::move(it_best->prompt); states.erase(it_best); + + SRV_TRC("slot %d: prompt cache restore took %.2f ms\n", id_slot, (ggml_time_us() - t_start) / 1000.0); } return true;