789 lines
34 KiB
C++
789 lines
34 KiB
C++
#include "ggml-vulkan-common.h"
|
|
|
|
ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {
|
|
/* .get_name = */ ggml_backend_vk_buffer_type_name,
|
|
/* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,
|
|
/* .alloc_buffer_n = */ NULL,
|
|
/* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment,
|
|
/* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,
|
|
/* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size,
|
|
/* .get_alloc_size_n = */ NULL,
|
|
/* .is_host = */ NULL,
|
|
};
|
|
|
|
static std::vector<uint32_t> ggml_vk_find_memory_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) {
|
|
std::vector<uint32_t> indices;
|
|
|
|
for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) {
|
|
vk::MemoryType memory_type = mem_props->memoryTypes[i];
|
|
if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) &&
|
|
(flags & memory_type.propertyFlags) == flags &&
|
|
mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) {
|
|
indices.push_back(i);
|
|
}
|
|
}
|
|
return indices;
|
|
}
|
|
|
|
static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std::initializer_list<vk::MemoryPropertyFlags> & req_flags_list,
|
|
void *import_ptr = nullptr) {
|
|
VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags_list.begin()[0]) << ", " << to_string(req_flags_list.begin()[req_flags_list.size()-1]) << ")");
|
|
if (size > device->max_buffer_size) {
|
|
throw vk::OutOfDeviceMemoryError("Requested buffer size exceeds device buffer size limit");
|
|
}
|
|
|
|
vk_buffer buf = std::make_shared<vk_buffer_struct>();
|
|
|
|
if (size == 0) {
|
|
buf->size = 0;
|
|
return buf;
|
|
}
|
|
|
|
vk::BufferUsageFlags usage_flags = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst;
|
|
vk::MemoryAllocateFlags mem_flags {};
|
|
if (device->buffer_device_address) {
|
|
usage_flags |= vk::BufferUsageFlagBits::eShaderDeviceAddress;
|
|
mem_flags |= vk::MemoryAllocateFlagBits::eDeviceAddress;
|
|
}
|
|
|
|
vk::BufferCreateInfo buffer_create_info{
|
|
vk::BufferCreateFlags(),
|
|
size,
|
|
usage_flags,
|
|
vk::SharingMode::eExclusive,
|
|
0,
|
|
nullptr,
|
|
};
|
|
|
|
vk::ExternalMemoryBufferCreateInfo external_memory_bci;
|
|
if (import_ptr) {
|
|
external_memory_bci.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;
|
|
buffer_create_info.setPNext(&external_memory_bci);
|
|
}
|
|
|
|
buf->buffer = device->device.createBuffer(buffer_create_info);
|
|
|
|
vk::MemoryRequirements mem_req = device->device.getBufferMemoryRequirements(buf->buffer);
|
|
|
|
vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();
|
|
|
|
const vk::MemoryPriorityAllocateInfoEXT mem_priority_info { 1.0f };
|
|
|
|
vk::MemoryAllocateFlagsInfo mem_flags_info { mem_flags };
|
|
|
|
if (device->memory_priority) {
|
|
mem_flags_info.setPNext(&mem_priority_info);
|
|
}
|
|
|
|
if (import_ptr) {
|
|
vk::MemoryHostPointerPropertiesEXT host_pointer_props;
|
|
try {
|
|
host_pointer_props = device->device.getMemoryHostPointerPropertiesEXT(vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, import_ptr);
|
|
} catch (vk::SystemError& e) {
|
|
GGML_LOG_WARN("ggml_vulkan: Failed getMemoryHostPointerPropertiesEXT (%s)\n", e.what());
|
|
device->device.destroyBuffer(buf->buffer);
|
|
return {};
|
|
}
|
|
vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();
|
|
|
|
uint32_t memory_type_idx;
|
|
vk::MemoryPropertyFlags property_flags = *req_flags_list.begin();
|
|
for (memory_type_idx = 0; memory_type_idx < 32; ++memory_type_idx) {
|
|
if (!(host_pointer_props.memoryTypeBits & (1u << memory_type_idx))) {
|
|
continue;
|
|
}
|
|
if (!(mem_req.memoryTypeBits & (1u << memory_type_idx))) {
|
|
continue;
|
|
}
|
|
|
|
vk::MemoryType memory_type = mem_props.memoryTypes[memory_type_idx];
|
|
// check for visible+coherent+cached. Other flags (e.g. devicelocal) are allowed
|
|
if ((memory_type.propertyFlags & property_flags) == property_flags) {
|
|
property_flags = memory_type.propertyFlags;
|
|
break;
|
|
}
|
|
}
|
|
if (memory_type_idx == 32) {
|
|
GGML_LOG_WARN("ggml_vulkan: Memory type for host allocation not found\n");
|
|
device->device.destroyBuffer(buf->buffer);
|
|
return {};
|
|
}
|
|
|
|
buf->memory_property_flags = mem_props.memoryTypes[memory_type_idx].propertyFlags;
|
|
try {
|
|
vk::ImportMemoryHostPointerInfoEXT import_info;
|
|
import_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;
|
|
import_info.pHostPointer = import_ptr;
|
|
import_info.setPNext(&mem_flags_info);
|
|
buf->device_memory = device->device.allocateMemory({ size, memory_type_idx, &import_info });
|
|
} catch (const vk::SystemError& e) {
|
|
}
|
|
} else {
|
|
for (auto it = req_flags_list.begin(); it != req_flags_list.end(); it++) {
|
|
const auto & req_flags = *it;
|
|
|
|
const std::vector<uint32_t> memory_type_indices = ggml_vk_find_memory_properties(&mem_props, &mem_req, req_flags);
|
|
|
|
if (memory_type_indices.empty()) {
|
|
continue;
|
|
}
|
|
|
|
bool done = false;
|
|
|
|
for (auto mtype_it = memory_type_indices.begin(); mtype_it != memory_type_indices.end(); mtype_it++) {
|
|
try {
|
|
buf->device_memory = device->device.allocateMemory({ mem_req.size, *mtype_it, &mem_flags_info });
|
|
buf->memory_property_flags = mem_props.memoryTypes[*mtype_it].propertyFlags;
|
|
done = true;
|
|
break;
|
|
} catch (const vk::SystemError& e) {
|
|
// loop and retry
|
|
// during last attempt throw the exception
|
|
if (it + 1 == req_flags_list.end() && mtype_it + 1 == memory_type_indices.end()) {
|
|
device->device.destroyBuffer(buf->buffer);
|
|
throw e;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (done) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!buf->device_memory) {
|
|
device->device.destroyBuffer(buf->buffer);
|
|
throw vk::OutOfDeviceMemoryError("No suitable memory type found");
|
|
}
|
|
|
|
buf->ptr = nullptr;
|
|
|
|
if (import_ptr) {
|
|
buf->ptr = import_ptr;
|
|
} else {
|
|
if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
|
|
buf->ptr = device->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE);
|
|
}
|
|
}
|
|
|
|
device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0);
|
|
|
|
buf->device = device;
|
|
buf->size = size;
|
|
|
|
if (device->buffer_device_address) {
|
|
const vk::BufferDeviceAddressInfo addressInfo(buf->buffer);
|
|
buf->bda_addr = device->device.getBufferAddress(addressInfo);
|
|
}
|
|
|
|
device->memory_logger->log_allocation(buf, size);
|
|
|
|
return buf;
|
|
}
|
|
|
|
vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags) {
|
|
try {
|
|
return ggml_vk_create_buffer(device, size, {req_flags, fallback_flags});
|
|
} catch (const vk::SystemError& e) {
|
|
std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl;
|
|
std::cerr << "ggml_vulkan: " << e.what() << std::endl;
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) {
|
|
vk_buffer buf;
|
|
try {
|
|
if (device->prefer_host_memory) {
|
|
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
|
|
vk::MemoryPropertyFlagBits::eDeviceLocal});
|
|
} else if (device->uma) {
|
|
// On UMA, prefer host-visible memory so direct tensor borrowing works.
|
|
// If unavailable, fall back to device-local memory.
|
|
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
|
|
vk::MemoryPropertyFlagBits::eDeviceLocal,
|
|
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
|
|
} else if (device->disable_host_visible_vidmem) {
|
|
if (device->allow_sysmem_fallback) {
|
|
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal,
|
|
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
|
|
} else {
|
|
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal});
|
|
}
|
|
} else {
|
|
// use rebar if available, otherwise fallback to device only visible memory
|
|
if (device->allow_sysmem_fallback) {
|
|
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
|
|
vk::MemoryPropertyFlagBits::eDeviceLocal,
|
|
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
|
|
} else {
|
|
buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,
|
|
vk::MemoryPropertyFlagBits::eDeviceLocal});
|
|
}
|
|
}
|
|
} catch (const vk::SystemError& e) {
|
|
std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl;
|
|
std::cerr << "ggml_vulkan: " << e.what() << std::endl;
|
|
throw e;
|
|
}
|
|
|
|
return buf;
|
|
}
|
|
|
|
void ggml_vk_destroy_buffer(vk_buffer& buf) {
|
|
if (buf == nullptr) {
|
|
return;
|
|
}
|
|
|
|
if (buf->device != nullptr) {
|
|
buf->device->memory_logger->log_deallocation(buf);
|
|
}
|
|
|
|
buf.reset();
|
|
}
|
|
|
|
void * ggml_vk_host_malloc(vk_device& device, size_t size) {
|
|
VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")");
|
|
vk_buffer buf = ggml_vk_create_buffer(device, size,
|
|
{vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
|
|
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});
|
|
|
|
if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) {
|
|
fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n",
|
|
size/1024.0/1024.0);
|
|
device->device.freeMemory(buf->device_memory);
|
|
device->device.destroyBuffer(buf->buffer);
|
|
return nullptr;
|
|
}
|
|
|
|
std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);
|
|
device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf));
|
|
|
|
return buf->ptr;
|
|
}
|
|
|
|
void ggml_vk_host_free(vk_device& device, void* ptr) {
|
|
if (ptr == nullptr) {
|
|
return;
|
|
}
|
|
VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")");
|
|
std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);
|
|
|
|
vk_buffer buf;
|
|
size_t index;
|
|
for (size_t i = 0; i < device->pinned_memory.size(); i++) {
|
|
const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);
|
|
const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);
|
|
if (ptr >= addr && ptr < endr) {
|
|
buf = std::get<2>(device->pinned_memory[i]);
|
|
index = i;
|
|
break;
|
|
}
|
|
}
|
|
if (buf == nullptr) {
|
|
fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n");
|
|
return;
|
|
}
|
|
|
|
ggml_vk_destroy_buffer(buf);
|
|
|
|
device->pinned_memory.erase(device->pinned_memory.begin() + index);
|
|
}
|
|
|
|
void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) {
|
|
std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex);
|
|
buf = nullptr;
|
|
buf_offset = 0;
|
|
for (size_t i = 0; i < device->pinned_memory.size(); i++) {
|
|
const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);
|
|
const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);
|
|
if (ptr >= addr && ptr < endr) {
|
|
buf = std::get<2>(device->pinned_memory[i]);
|
|
buf_offset = ((const uint8_t *)ptr) - addr;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size) {
|
|
if (device->sync_staging == nullptr || device->sync_staging->size < size) {
|
|
VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");
|
|
ggml_vk_destroy_buffer(device->sync_staging);
|
|
device->sync_staging = ggml_vk_create_buffer_check(device, size,
|
|
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
|
|
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
|
|
}
|
|
}
|
|
|
|
void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) {
|
|
if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) {
|
|
VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");
|
|
ggml_vk_destroy_buffer(ctx->sync_staging);
|
|
ctx->sync_staging = ggml_vk_create_buffer_check(ctx->device, size,
|
|
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,
|
|
vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);
|
|
}
|
|
}
|
|
|
|
static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")");
|
|
GGML_ASSERT(!ggml_is_contiguous(tensor));
|
|
// Buffer is already mapped
|
|
if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
|
|
std::cerr << "ggml_vulkan: buffer_write_nc_async dst buffer is host_visible. Use synchronous write." << std::endl;
|
|
GGML_ABORT("fatal error");
|
|
}
|
|
// Check if src is pinned memory
|
|
vk_buffer buf = nullptr;
|
|
size_t buf_offset = 0;
|
|
ggml_vk_host_get(ctx->device, tensor->data, buf, buf_offset);
|
|
|
|
const uint64_t ne0 = tensor->ne[0];
|
|
const uint64_t ne1 = tensor->ne[1];
|
|
const uint64_t ne2 = tensor->ne[2];
|
|
const uint64_t ne3 = tensor->ne[3];
|
|
const uint64_t nb0 = tensor->nb[0];
|
|
const uint64_t nb1 = tensor->nb[1];
|
|
const uint64_t nb2 = tensor->nb[2];
|
|
const uint64_t nb3 = tensor->nb[3];
|
|
const ggml_type type = tensor->type;
|
|
const uint64_t ts = ggml_type_size(type);
|
|
const uint64_t bs = ggml_blck_size(type);
|
|
|
|
const uint64_t dstnb0 = ts;
|
|
const uint64_t dstnb1 = dstnb0*(ne0/bs);
|
|
const uint64_t dstnb2 = dstnb1*ne1;
|
|
const uint64_t dstnb3 = dstnb2*ne2;
|
|
|
|
const uint64_t ne = ggml_nelements(tensor);
|
|
|
|
if (buf != nullptr) {
|
|
// Memory is pinned, use as staging buffer
|
|
std::vector<vk::BufferCopy> slices;
|
|
|
|
for (uint64_t i3 = 0; i3 < ne3; i3++) {
|
|
for (uint64_t i2 = 0; i2 < ne2; i2++) {
|
|
// Find longest contiguous slice
|
|
if (ne1*nb1 == dstnb2) {
|
|
slices.push_back({ buf_offset + i3*nb3 + i2*nb2, offset + i3*dstnb3 + i2*dstnb2, dstnb2 });
|
|
} else {
|
|
for (uint64_t i1 = 0; i1 < ne1; i1++) {
|
|
if (ne0*nb0/bs == dstnb1) {
|
|
slices.push_back({ buf_offset + i3*nb3 + i2*nb2 + i1*nb1, offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, dstnb1 });
|
|
} else {
|
|
const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;
|
|
const uint64_t d_off = offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1;
|
|
for (uint64_t i0 = 0; i0 < ne0; i0++) {
|
|
slices.push_back({ s_off + i0*nb0, d_off + i0*dstnb0, dstnb0 });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
ggml_vk_sync_buffers(ctx, subctx);
|
|
subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);
|
|
return;
|
|
}
|
|
|
|
if (!sync_staging) {
|
|
GGML_ABORT("Asynchronous write to non-pinned memory not supported");
|
|
}
|
|
|
|
// Staging buffer required
|
|
vk_buffer& staging = ctx->device->sync_staging;
|
|
const uint64_t copy_size = ts*ne/bs;
|
|
ggml_vk_ensure_sync_staging_buffer(ctx->device, copy_size);
|
|
VkBufferCopy buf_copy{ 0, offset, copy_size };
|
|
|
|
ggml_vk_sync_buffers(ctx, subctx);
|
|
vkCmdCopyBuffer(subctx->s->buffer->buf, (VkBuffer)staging->buffer, (VkBuffer)dst->buffer, 1, &buf_copy);
|
|
|
|
for (uint64_t i3 = 0; i3 < ne3; i3++) {
|
|
for (uint64_t i2 = 0; i2 < ne2; i2++) {
|
|
// Find longest contiguous slice
|
|
if (ne1*nb1 == dstnb2) {
|
|
deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &subctx->in_memcpys);
|
|
} else {
|
|
for (uint64_t i1 = 0; i1 < ne1; i1++) {
|
|
if (ne0*nb0/bs == dstnb1) {
|
|
deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &subctx->in_memcpys);
|
|
} else {
|
|
const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;
|
|
const uint64_t d_off = i3*dstnb3 + i2*dstnb2 + i1*dstnb1;
|
|
for (uint64_t i0 = 0; i0 < ne0; i0++) {
|
|
deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &subctx->in_memcpys);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")");
|
|
// Check if src is pinned memory
|
|
vk_buffer buf = nullptr;
|
|
size_t buf_offset = 0;
|
|
ggml_vk_host_get(dst->device, src, buf, buf_offset);
|
|
|
|
if (buf != nullptr) {
|
|
// Memory is pinned, use as staging buffer
|
|
std::vector<vk::BufferCopy> slices(1);
|
|
if (width == spitch && width == dpitch) {
|
|
// Only do single write if stride is equal
|
|
slices[0].srcOffset = buf_offset;
|
|
slices[0].dstOffset = offset;
|
|
slices[0].size = width * height;
|
|
} else {
|
|
slices.resize(height);
|
|
for (size_t i = 0; i < height; i++) {
|
|
slices[i].srcOffset = buf_offset + i * spitch;
|
|
slices[i].dstOffset = offset + i * dpitch;
|
|
slices[i].size = width;
|
|
}
|
|
}
|
|
|
|
ggml_vk_sync_buffers(nullptr, subctx);
|
|
subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);
|
|
return true;
|
|
}
|
|
VK_LOG_DEBUG("STAGING");
|
|
|
|
if (!sync_staging) {
|
|
// copy was not handled caller needs to fall back
|
|
return false;
|
|
}
|
|
|
|
// Staging buffer required
|
|
const size_t staging_size = width * height;
|
|
ggml_vk_ensure_sync_staging_buffer(dst->device, staging_size);
|
|
|
|
vk_buffer& staging_buffer = dst->device->sync_staging;
|
|
|
|
std::vector<vk::BufferCopy> slices(1);
|
|
if (width == dpitch) {
|
|
slices[0].srcOffset = 0;
|
|
slices[0].dstOffset = offset;
|
|
slices[0].size = staging_size;
|
|
} else {
|
|
slices.resize(height);
|
|
for (size_t i = 0; i < height; i++) {
|
|
slices[i].srcOffset = i * width;
|
|
slices[i].dstOffset = offset + i * dpitch;
|
|
slices[i].size = width;
|
|
}
|
|
}
|
|
|
|
ggml_vk_sync_buffers(nullptr, subctx);
|
|
subctx->s->buffer->buf.copyBuffer(staging_buffer->buffer, dst->buffer, slices);
|
|
|
|
if (width == spitch) {
|
|
deferred_memcpy((uint8_t *)staging_buffer->ptr, src, staging_size, &subctx->in_memcpys);
|
|
} else {
|
|
for (size_t i = 0; i < height; i++) {
|
|
deferred_memcpy((uint8_t *)staging_buffer->ptr + i * width, (const uint8_t *) src + i * spitch, width, &subctx->in_memcpys);
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_write_async(" << size << ")");
|
|
return ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, size, size, size, 1, sync_staging);
|
|
}
|
|
|
|
void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_write_2d(" << width << ", " << height << ")");
|
|
// Buffer is already mapped
|
|
if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {
|
|
GGML_ASSERT(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
|
|
|
|
if (width == spitch && width == dpitch) {
|
|
memcpy((uint8_t *)dst->ptr + offset, src, width * height);
|
|
} else {
|
|
for (size_t i = 0; i < height; i++) {
|
|
memcpy((uint8_t *)dst->ptr + offset + i * dpitch, (const uint8_t *) src + i * spitch, width);
|
|
}
|
|
}
|
|
} else {
|
|
std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);
|
|
|
|
vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);
|
|
ggml_vk_ctx_begin(dst->device, subctx);
|
|
bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true);
|
|
GGML_ASSERT(ret);
|
|
ggml_vk_ctx_end(subctx);
|
|
|
|
for (auto& cpy : subctx->in_memcpys) {
|
|
memcpy(cpy.dst, cpy.src, cpy.n);
|
|
}
|
|
|
|
for (auto& mset : subctx->memsets) {
|
|
memset(mset.dst, mset.val, mset.n);
|
|
}
|
|
|
|
ggml_vk_submit(subctx, dst->device->fence);
|
|
VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);
|
|
dst->device->device.resetFences({ dst->device->fence });
|
|
ggml_vk_queue_command_pools_cleanup(dst->device);
|
|
}
|
|
}
|
|
|
|
void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_write(" << size << ")");
|
|
ggml_vk_buffer_write_2d(dst, offset, src, size, size, size, 1);
|
|
}
|
|
|
|
bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")");
|
|
GGML_ASSERT(width > 0);
|
|
GGML_ASSERT(height > 0);
|
|
GGML_ASSERT(src != nullptr);
|
|
|
|
// TODO: staging_offset is not used
|
|
|
|
// Check if dst is pinned memory
|
|
vk_buffer buf = nullptr;
|
|
size_t buf_offset = 0;
|
|
ggml_vk_host_get(src->device, dst, buf, buf_offset);
|
|
|
|
std::vector<vk::BufferCopy> slices(1);
|
|
if (width == spitch && width == dpitch) {
|
|
// Only do single write if stride is equal
|
|
slices[0].srcOffset = offset;
|
|
slices[0].dstOffset = buf_offset;
|
|
slices[0].size = width * height;
|
|
} else {
|
|
slices.resize(height);
|
|
for (size_t i = 0; i < height; i++) {
|
|
slices[i].srcOffset = offset + i * spitch;
|
|
slices[i].dstOffset = buf_offset + i * dpitch;
|
|
slices[i].size = width;
|
|
}
|
|
}
|
|
|
|
if (buf != nullptr) {
|
|
// Memory is pinned, use as staging buffer
|
|
ggml_vk_sync_buffers(nullptr, subctx);
|
|
subctx->s->buffer->buf.copyBuffer(src->buffer, buf->buffer, slices);
|
|
|
|
return true;
|
|
}
|
|
VK_LOG_DEBUG("STAGING");
|
|
|
|
if (!sync_staging) {
|
|
// copy was not handled caller needs to fall back
|
|
return false;
|
|
}
|
|
|
|
// Fall back to staging buffer
|
|
const size_t staging_size = width * height;
|
|
ggml_vk_ensure_sync_staging_buffer(src->device, staging_size);
|
|
|
|
vk_buffer& staging_buffer = src->device->sync_staging;
|
|
|
|
std::vector<vk::BufferCopy> staging_slices(1);
|
|
if (width == spitch) {
|
|
staging_slices[0].srcOffset = offset;
|
|
staging_slices[0].dstOffset = 0;
|
|
staging_slices[0].size = staging_size;
|
|
} else {
|
|
staging_slices.resize(height);
|
|
for (size_t i = 0; i < height; i++) {
|
|
staging_slices[i].srcOffset = offset + i * spitch;
|
|
staging_slices[i].dstOffset = i * width;
|
|
staging_slices[i].size = width;
|
|
}
|
|
}
|
|
|
|
ggml_vk_sync_buffers(nullptr, subctx);
|
|
subctx->s->buffer->buf.copyBuffer(src->buffer, staging_buffer->buffer, staging_slices);
|
|
|
|
if (width == dpitch) {
|
|
deferred_memcpy(dst, staging_buffer->ptr, staging_size, &subctx->out_memcpys);
|
|
} else {
|
|
for (size_t i = 0; i < height; i++) {
|
|
deferred_memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) staging_buffer->ptr + i * width, width, &subctx->out_memcpys);
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t size, bool sync_staging = false) {
|
|
return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging);
|
|
}
|
|
|
|
void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")");
|
|
|
|
// If the device is not an UMA device the memory is host-accessible through rebar. While writing
|
|
// through PCIe is sufficient fast reading back data from PCIe is slower than going through
|
|
// the HW device to host copy path.
|
|
// AMD UMA: uncached host-visible memory is write-combined, CPU reads are slow
|
|
const bool slow_host_read = src->device->vendor_id == VK_VENDOR_ID_AMD &&
|
|
!(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCached);
|
|
if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma && !slow_host_read) {
|
|
GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);
|
|
|
|
std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
|
|
vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool);
|
|
ggml_vk_ctx_begin(src->device, subctx);
|
|
subctx->s->buffer->buf.pipelineBarrier(
|
|
vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,
|
|
vk::PipelineStageFlagBits::eHost,
|
|
{},
|
|
{ { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,
|
|
vk::AccessFlagBits::eHostRead } },
|
|
{}, {});
|
|
ggml_vk_ctx_end(subctx);
|
|
ggml_vk_submit(subctx, src->device->fence);
|
|
VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),
|
|
"vk_buffer_read_2d uma waitForFences", src->device);
|
|
src->device->device.resetFences({ src->device->fence });
|
|
ggml_vk_queue_command_pools_cleanup(src->device);
|
|
|
|
if (width == spitch && width == dpitch) {
|
|
memcpy(dst, (const uint8_t *) src->ptr + offset, width * height);
|
|
} else {
|
|
for (size_t i = 0; i < height; i++) {
|
|
memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) src->ptr + offset + i * spitch, width);
|
|
}
|
|
}
|
|
} else {
|
|
std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
|
|
|
|
vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);
|
|
ggml_vk_ctx_begin(src->device, subctx);
|
|
bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true);
|
|
GGML_ASSERT(ret);
|
|
ggml_vk_ctx_end(subctx);
|
|
|
|
ggml_vk_submit(subctx, src->device->fence);
|
|
VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);
|
|
src->device->device.resetFences({ src->device->fence });
|
|
ggml_vk_queue_command_pools_cleanup(src->device);
|
|
|
|
for (auto& cpy : subctx->out_memcpys) {
|
|
memcpy(cpy.dst, cpy.src, cpy.n);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_read(" << src->buffer << ", " << offset << ", " << size << ")");
|
|
ggml_vk_buffer_read_2d(src, offset, dst, size, size, size, 1);
|
|
}
|
|
|
|
void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_copy_async(" << size << ")");
|
|
// Make sure both buffers are on same device
|
|
GGML_ASSERT(src->device == dst->device);
|
|
|
|
VkBufferCopy bc{ src_offset, dst_offset, size };
|
|
|
|
vkCmdCopyBuffer(ctx->s->buffer->buf, (VkBuffer)src->buffer, (VkBuffer)dst->buffer, 1, &bc);
|
|
}
|
|
|
|
void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {
|
|
if (src->device == dst->device) {
|
|
std::lock_guard<std::recursive_mutex> guard(src->device->mutex);
|
|
VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")");
|
|
// Copy within the device
|
|
vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);
|
|
ggml_vk_ctx_begin(src->device, subctx);
|
|
ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);
|
|
ggml_vk_ctx_end(subctx);
|
|
ggml_vk_submit(subctx, src->device->fence);
|
|
VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);
|
|
src->device->device.resetFences({ src->device->fence });
|
|
ggml_vk_queue_command_pools_cleanup(src->device);
|
|
} else {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")");
|
|
// Copy device to device
|
|
ggml_vk_ensure_sync_staging_buffer(src->device, size);
|
|
|
|
// Copy to src staging buffer
|
|
ggml_vk_buffer_copy(src->device->sync_staging, 0, src, src_offset, size);
|
|
// Copy to dst buffer
|
|
ggml_vk_buffer_write(dst, dst_offset, src->device->sync_staging->ptr, size);
|
|
}
|
|
}
|
|
|
|
void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_memset_async(" << offset << ", " << c << ", " << size << ")");
|
|
|
|
if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&
|
|
dst->device->uma) {
|
|
deferred_memset((uint8_t*)dst->ptr + offset, c, size, &ctx->memsets);
|
|
return;
|
|
}
|
|
|
|
// Fall back to GPU fillBuffer for non-UMA or non-host-visible buffers
|
|
ctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);
|
|
}
|
|
|
|
void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size) {
|
|
VK_LOG_DEBUG("ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")");
|
|
|
|
if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&
|
|
dst->device->uma) {
|
|
memset((uint8_t*)dst->ptr + offset, c, size);
|
|
return;
|
|
}
|
|
|
|
std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);
|
|
vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);
|
|
ggml_vk_ctx_begin(dst->device, subctx);
|
|
subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);
|
|
ggml_vk_ctx_end(subctx);
|
|
|
|
ggml_vk_submit(subctx, dst->device->fence);
|
|
VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);
|
|
dst->device->device.resetFences({ dst->device->fence });
|
|
ggml_vk_queue_command_pools_cleanup(dst->device);
|
|
}
|
|
|
|
ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
|
|
/* .free_buffer = */ ggml_backend_vk_buffer_free_buffer,
|
|
/* .get_base = */ ggml_backend_vk_buffer_get_base,
|
|
/* .init_tensor = */ ggml_backend_vk_buffer_init_tensor,
|
|
/* .memset_tensor = */ ggml_backend_vk_buffer_memset_tensor,
|
|
/* .set_tensor = */ ggml_backend_vk_buffer_set_tensor,
|
|
/* .get_tensor = */ ggml_backend_vk_buffer_get_tensor,
|
|
/* .set_tensor_2d = */ ggml_backend_vk_buffer_set_tensor_2d,
|
|
/* .get_tensor_2d = */ ggml_backend_vk_buffer_get_tensor_2d,
|
|
/* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,
|
|
/* .clear = */ ggml_backend_vk_buffer_clear,
|
|
/* .reset = */ NULL,
|
|
};
|
|
|
|
vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) {
|
|
if (!device->external_memory_host) {
|
|
return {};
|
|
}
|
|
|
|
uintptr_t uptr = reinterpret_cast<uintptr_t>(ptr);
|
|
if (uptr & (device->min_imported_host_pointer_alignment - 1)) {
|
|
return {};
|
|
}
|
|
if (size & (device->min_imported_host_pointer_alignment - 1)) {
|
|
return {};
|
|
}
|
|
|
|
const vk::MemoryPropertyFlags property_flags = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached;
|
|
|
|
vk_buffer buf {};
|
|
try {
|
|
buf = ggml_vk_create_buffer(device, size, { property_flags }, ptr);
|
|
} catch (vk::SystemError& e) {
|
|
GGML_LOG_WARN("ggml_vulkan: Failed ggml_vk_create_buffer (%s)\n", e.what());
|
|
}
|
|
|
|
return buf;
|
|
}
|
|
|