From 9d340ec341e079d2c28afd50871afce358cdbf48 Mon Sep 17 00:00:00 2001 From: Julius Tischbein Date: Mon, 9 Nov 2026 19:22:08 +0100 Subject: [PATCH] Windows unbuffered model load (#26014) * Windows unbuffered io * Fix read len calculation * Add test and model load timing * chunked fallback read + removing filepointer semantic divergence between linux and windows * tests : remove unused mmap test arguments Assisted-by: Codex * remove timing, set file pointer after chunk read, test refinement * Update tests/test-llama-mmap.cpp Co-authored-by: Oliver Simons * Update src/llama-mmap.cpp Co-authored-by: Oliver Simons * test refinement Assisted-by: Codex * remove duplicate constexpr Assisted-by: Codex * adding label behavior to tests Assisted-by: Codex --------- Co-authored-by: Oliver Simons --- src/llama-mmap.cpp | 304 +++++++++++++++++++++++++++++++++----- tests/CMakeLists.txt | 7 + tests/test-llama-mmap.cpp | 232 +++++++++++++++++++++++++++++ 3 files changed, 506 insertions(+), 37 deletions(-) create mode 100644 tests/test-llama-mmap.cpp diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 4d1cd31e9..9688a7ed4 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -30,6 +30,7 @@ #ifndef NOMINMAX #define NOMINMAX #endif + #include #include #ifndef PATH_MAX #define PATH_MAX MAX_PATH @@ -68,7 +69,10 @@ static std::string llama_format_win_err(DWORD err) { struct llama_file::impl { #if defined(_WIN32) - HANDLE fp_win32; + // Keep the buffered handle as the logical cursor; the direct handle only issues aligned I/O. + HANDLE fp_win32 = INVALID_HANDLE_VALUE; + HANDLE fp_win32_direct = INVALID_HANDLE_VALUE; + std::string GetErrorMessageWin32(DWORD error_code) const { std::string ret; LPSTR lpMsgBuf = NULL; @@ -81,13 +85,93 @@ struct llama_file::impl { LocalFree(lpMsgBuf); } + while (!ret.empty() && (ret.back() == '\r' || ret.back() == '\n')) { + ret.pop_back(); + } + return ret; } - impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) { - fp = ggml_fopen(fname, mode); + impl(const char * fname, const char * mode, const bool use_direct_io = false) : fname(fname) { + init_buffered_io(mode); + if (use_direct_io && std::strcmp(mode, "rb") == 0) { + DWORD error = ERROR_SUCCESS; + if (!init_direct_io(error)) { + LLAMA_LOG_WARN("Failed to open file '%s' with unbuffered I/O: %s. Falling back to buffered I/O\n", + fname, GetErrorMessageWin32(error).c_str()); + } + } + } + + bool init_direct_io(DWORD& error) { + error = ERROR_SUCCESS; + alignment = 1; + + fp_win32_direct = ReOpenFile( + fp_win32, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + FILE_FLAG_NO_BUFFERING); + + if (fp_win32_direct == INVALID_HANDLE_VALUE) { + error = GetLastError(); + return false; + } + + const bool ok = [&]() -> bool { +#if defined(NTDDI_VERSION) && defined(NTDDI_WIN8) && (NTDDI_VERSION >= NTDDI_WIN8) + FILE_STORAGE_INFO storage_info{}; + if (!GetFileInformationByHandleEx( + fp_win32_direct, + FileStorageInfo, + &storage_info, + static_cast(sizeof(storage_info)))) { + error = GetLastError(); + return false; + } + + alignment = std::max({ + static_cast(storage_info.LogicalBytesPerSector), + static_cast(storage_info.PhysicalBytesPerSectorForAtomicity), + static_cast(storage_info.PhysicalBytesPerSectorForPerformance), + static_cast(storage_info.FileSystemEffectivePhysicalBytesPerSectorForAtomicity) + }); +#else + error = ERROR_NOT_SUPPORTED; + return false; +#endif + // Required for bit-mask rounding and aligned allocation schemes. + if (alignment <= 1 || (alignment & (alignment - 1)) != 0) { + error = ERROR_NOT_SUPPORTED; + return false; + } + + // ReOpenFile creates a distinct handle; explicitly start direct I/O at offset 0. + LARGE_INTEGER zero{}; + if (!SetFilePointerEx(fp_win32_direct, zero, nullptr, FILE_BEGIN)) { + error = GetLastError(); + return false; + } + + return true; + }(); + + if (!ok) { + CloseHandle(fp_win32_direct); + fp_win32_direct = INVALID_HANDLE_VALUE; + alignment = 1; + return false; + } + + LLAMA_LOG_INFO("%s: file '%s' opened with unbuffered I/O (alignment: %zu bytes)\n", + __func__, fname.c_str(), alignment); + return true; + } + + void init_buffered_io(const char * mode) { + fp = ggml_fopen(fname.c_str(), mode); if (fp == NULL) { - throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); + throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno))); } fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp)); seek(0, SEEK_END); @@ -95,7 +179,7 @@ struct llama_file::impl { seek(0, SEEK_SET); } - impl(FILE * file) : owns_fp(false) { + impl(FILE * file) : fname("(file*)"), owns_fp(false) { fp = file; fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp)); seek(0, SEEK_END); @@ -108,7 +192,7 @@ struct llama_file::impl { li.QuadPart = 0; BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT); if (!ret) { - throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str())); + throw std::runtime_error(format("tell error: %s", GetErrorMessageWin32(GetLastError()).c_str())); } return li.QuadPart; @@ -123,24 +207,159 @@ struct llama_file::impl { li.QuadPart = offset; BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence); if (!ret) { - throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str())); + throw std::runtime_error(format("seek error: %s", GetErrorMessageWin32(GetLastError()).c_str())); } } - void read_raw(void * ptr, size_t len) { - size_t bytes_read = 0; - while (bytes_read < len) { - size_t chunk_size = std::min(len - bytes_read, 64*1024*1024); - DWORD chunk_read = 0; - BOOL result = ReadFile(fp_win32, reinterpret_cast(ptr) + bytes_read, chunk_size, &chunk_read, NULL); - if (!result) { - throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str())); + void read_raw_unsafe(void * ptr, size_t len) { + if (len == 0) { + return; + } + + const size_t read_offset = tell(); + if (read_offset > size) { + throw std::runtime_error("unexpectedly reached end of file"); + } + + if (!has_direct_io()) { + const size_t read_size = std::min(len, size - read_offset); + size_t bytes_read = 0; + while (bytes_read < read_size) { + const DWORD to_read = (DWORD) std::min(read_size - bytes_read, MAXDWORD); + DWORD chunk_read = 0; + BOOL result = ReadFile(fp_win32, reinterpret_cast(ptr) + bytes_read, + to_read, &chunk_read, NULL); + if (!result) { + throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str())); + } + if (chunk_read < to_read) { + throw std::runtime_error("unexpectedly reached end of file"); + } + + bytes_read += chunk_read; } - if (chunk_read < chunk_size || chunk_read == 0) { - throw std::runtime_error("unexpectedly reached end of file"); + return; + } + + DWORD error = ERROR_SUCCESS; + if ((read_offset & (alignment - 1)) != 0 || + (len & (alignment - 1)) != 0 || + (reinterpret_cast(ptr) & (alignment - 1)) != 0) { + error = ERROR_INVALID_PARAMETER; + } else { + LARGE_INTEGER li; + li.QuadPart = read_offset; + if (!SetFilePointerEx(fp_win32_direct, li, NULL, FILE_BEGIN)) { + error = GetLastError(); + } + } + + size_t bytes_read = 0; + const size_t max_read_size = (size_t) MAXDWORD & ~(alignment - 1); + while (error == ERROR_SUCCESS && bytes_read < len) { + const DWORD to_read = (DWORD) std::min(len - bytes_read, max_read_size); + DWORD chunk_read = 0; + BOOL result = ReadFile(fp_win32_direct, reinterpret_cast(ptr) + bytes_read, + to_read, &chunk_read, NULL); + if (!result) { + error = GetLastError(); + if (error == ERROR_HANDLE_EOF && bytes_read == size - read_offset) { + error = ERROR_SUCCESS; + break; + } + break; } bytes_read += chunk_read; + if (chunk_read < to_read) { + if (bytes_read == size - read_offset) { + break; + } + error = ERROR_HANDLE_EOF; + } + } + + if (error == ERROR_INVALID_PARAMETER || error == ERROR_INVALID_USER_BUFFER || error == ERROR_NOACCESS) { + LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n", + __func__, fname.c_str(), GetErrorMessageWin32(error).c_str()); + CloseHandle(fp_win32_direct); + fp_win32_direct = INVALID_HANDLE_VALUE; + alignment = 1; + seek(read_offset, SEEK_SET); + read_raw_unsafe(ptr, len); + return; + } + if (error != ERROR_SUCCESS) { + throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(error).c_str())); + } + + seek(len > size - read_offset ? size : read_offset + len, SEEK_SET); + } + + void read_aligned_chunk(void * dest, size_t len) { + if (len == 0) { + return; + } + + const size_t offset = tell(); + if (offset > size || len > size - offset) { + throw std::runtime_error("unexpectedly reached end of file"); + } + + const size_t aligned_offset = offset & ~(alignment - 1); + const size_t offset_from_alignment = offset - aligned_offset; + const size_t bytes_to_read = (offset_from_alignment + len + alignment - 1) & ~(alignment - 1); + const size_t buffer_size = std::min(bytes_to_read, LLAMA_DIRECT_IO_BUFFER_SIZE); + + struct aligned_buffer_deleter { + void operator()(void * p) const { _aligned_free(p); } + }; + std::unique_ptr buffer(_aligned_malloc(buffer_size, alignment)); + if (!buffer) { + LLAMA_LOG_WARN("%s: Falling back to buffered I/O due to %s\n", + __func__, GetErrorMessageWin32(ERROR_NOT_ENOUGH_MEMORY).c_str()); + CloseHandle(fp_win32_direct); + fp_win32_direct = INVALID_HANDLE_VALUE; + alignment = 1; + seek(offset, SEEK_SET); + read_raw_unsafe(dest, len); + return; + } + + seek(aligned_offset, SEEK_SET); + + size_t skip = offset_from_alignment; + size_t copied = 0; + for (size_t done = 0; done < bytes_to_read; ) { + const size_t n = std::min(buffer_size, bytes_to_read - done); + read_raw_unsafe(buffer.get(), n); + + const size_t chunk_skip = std::min(skip, n); + const size_t count = std::min(n - chunk_skip, len - copied); + memcpy(reinterpret_cast(dest) + copied, reinterpret_cast(buffer.get()) + chunk_skip, count); + + copied += count; + skip -= chunk_skip; + done += n; + } + + seek(offset + len, SEEK_SET); + } + + void read_raw(void * ptr, size_t len) { + if (len == 0) { + return; + } + + const size_t offset = tell(); + if (offset > size || len > size - offset) { + throw std::runtime_error("unexpectedly reached end of file"); + } + + if (has_direct_io()) { + read_aligned_chunk(ptr, len); + } else { + read_raw_unsafe(ptr, len); } } @@ -153,7 +372,7 @@ struct llama_file::impl { void write_raw(const void * ptr, size_t len) const { size_t bytes_written = 0; while (bytes_written < len) { - size_t chunk_size = std::min(len - bytes_written, 64*1024*1024); + size_t chunk_size = std::min(len - bytes_written, LLAMA_DIRECT_IO_BUFFER_SIZE); DWORD chunk_written = 0; BOOL result = WriteFile(fp_win32, reinterpret_cast(ptr) + bytes_written, chunk_size, &chunk_written, NULL); if (!result) { @@ -172,23 +391,29 @@ struct llama_file::impl { } bool has_direct_io() const { - return true; + return fp_win32_direct != INVALID_HANDLE_VALUE && alignment > 1; } ~impl() { + if (fp_win32_direct != INVALID_HANDLE_VALUE) { + CloseHandle(fp_win32_direct); + } if (fp && owns_fp) { std::fclose(fp); } } + std::string fname; #else impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) { #ifdef __linux__ // Try unbuffered I/O for read only if (use_direct_io && std::strcmp(mode, "rb") == 0) { if (init_fd()) { + LLAMA_LOG_INFO("%s: file '%s' opened with direct I/O (alignment: %zu bytes)\n", + __func__, fname, alignment); return; } - LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O", + LLAMA_LOG_WARN("Failed to open file '%s' with unbuffered I/O: %s. Falling back to buffered I/O\n", fname, strerror(errno)); } #endif @@ -268,8 +493,8 @@ struct llama_file::impl { } errno = 0; if (fd == -1) { - const size_t curr_off = tell(); - const size_t to_read = std::min(len, size - curr_off); + const size_t read_offset = tell(); + const size_t to_read = std::min(len, size - read_offset); std::size_t ret = std::fread(ptr, to_read, 1, fp); if (ferror(fp)) { @@ -286,24 +511,23 @@ struct llama_file::impl { if (ret == -1) { if (errno == EINTR) { - continue; // Interrupted by signal, retry + continue; } - // Fallback to std::fread in case the DMA controller cannot access the buffer if (errno == EFAULT || errno == EINVAL) { - LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno)); - auto curr_off = tell(); + LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n", + __func__, fname.c_str(), strerror(errno)); + const size_t read_offset = tell(); close(fd); fd = -1; alignment = 1; init_fp("rb"); - seek(curr_off, SEEK_SET); + seek(read_offset, SEEK_SET); read_raw_unsafe(ptr, len); return; } throw std::runtime_error(format("read error: %s", strerror(errno))); } if (ret == 0) { - // EOF: allow if this read was only pulling alignment padding past file end off_t pos = lseek(fd, 0, SEEK_CUR); if (pos != -1 && (size_t) pos == size) { std::memset(reinterpret_cast(ptr) + bytes_read, 0, len - bytes_read); @@ -317,11 +541,11 @@ struct llama_file::impl { } } - void read_aligned_chunk(void * dest, size_t size) { + void read_aligned_chunk(void * dest, size_t len) { size_t offset = tell(); off_t aligned_offset = offset & ~(alignment - 1); off_t offset_from_alignment = offset - aligned_offset; - size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1); + size_t bytes_to_read = (offset_from_alignment + len + alignment - 1) & ~(alignment - 1); // stage through a bounded buffer, so that a large tensor is not held in memory twice while it loads const size_t buffer_size = std::min(bytes_to_read, LLAMA_DIRECT_IO_BUFFER_SIZE); @@ -329,7 +553,15 @@ struct llama_file::impl { void * raw_buffer = nullptr; int ret = posix_memalign(&raw_buffer, alignment, buffer_size); if (ret != 0) { - throw std::runtime_error(format("posix_memalign failed with error %d", ret)); + LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n", + __func__, fname.c_str(), strerror(ret)); + close(fd); + fd = -1; + alignment = 1; + init_fp("rb"); + seek(offset, SEEK_SET); + read_raw_unsafe(dest, len); + return; } struct aligned_buffer_deleter { @@ -345,13 +577,15 @@ struct llama_file::impl { const size_t n = std::min(buffer_size, bytes_to_read - done); read_raw_unsafe(buffer.get(), n); - const size_t count = std::min(n - skip, size - copied); + const size_t count = std::min(n - skip, len - copied); memcpy(reinterpret_cast(dest) + copied, reinterpret_cast(buffer.get()) + skip, count); copied += count; skip = 0; done += n; } + + seek(offset + len, SEEK_SET); } void read_raw(void * ptr, size_t len) { @@ -390,7 +624,7 @@ struct llama_file::impl { ~impl() { if (fd != -1) { close(fd); - } else if (owns_fp) { + } else if (fp && owns_fp) { std::fclose(fp); } } @@ -439,11 +673,7 @@ int llama_file::file_id() const { void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); } void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); } -#ifdef _WIN32 -void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); } -#else void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); } -#endif uint32_t llama_file::read_u32() { return pimpl->read_u32(); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4224a15ae..4ba52d3cf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -357,6 +357,13 @@ llama_build(test-backend-ops.cpp LINK ggml Threads::Threads) llama_build_and_test(test-backend-tensor.cpp LINK ggml) +if (BUILD_SHARED_LIBS) + llama_build_and_test(test-llama-mmap.cpp ${PROJECT_SOURCE_DIR}/src/llama-impl.cpp ${PROJECT_SOURCE_DIR}/src/llama-mmap.cpp LABEL behavior) + target_compile_definitions(test-llama-mmap PRIVATE LLAMA_BUILD) +else() + llama_build_and_test(test-llama-mmap.cpp LABEL behavior) +endif() + llama_build_and_test(test-tiled-mulmat.cpp LABEL "main behavior backend" LINK ggml) llama_build_and_test(test-model-load-cancel.cpp LABEL "model behavior") llama_build_and_test(test-autorelease.cpp LABEL "model behavior") diff --git a/tests/test-llama-mmap.cpp b/tests/test-llama-mmap.cpp new file mode 100644 index 000000000..06f0be188 --- /dev/null +++ b/tests/test-llama-mmap.cpp @@ -0,0 +1,232 @@ +#include "testing.h" + +#include "../src/llama-mmap.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#endif + +static constexpr size_t test_file_size = LLAMA_DIRECT_IO_BUFFER_SIZE + 3*1024*1024 + 1; +static constexpr size_t test_io_size = 1024*1024; + +static uint8_t test_byte(size_t offset) { + // Mix high offset bits into each byte to detect repeated chunks. + uint64_t value = offset; + value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9ULL; + value = (value ^ (value >> 27)) * 0x94d049bb133111ebULL; + return static_cast(value ^ (value >> 31)); +} + +static bool matches(const void * data, size_t offset, size_t size) { + const auto * bytes = static_cast(data); + for (size_t i = 0; i < size; ++i) { + if (bytes[i] != test_byte(offset + i)) { + return false; + } + } + return true; +} + +static void check_test_pattern(testing & t) { + uint8_t buffer[256]; + for (size_t i = 0; i < sizeof(buffer); ++i) { + buffer[i] = test_byte(i); + } + t.assert_true("reject data repeated after 512 KiB", !matches(buffer, 512 * 1024, sizeof(buffer))); + t.assert_true("reject data repeated after a direct I/O chunk", !matches(buffer, LLAMA_DIRECT_IO_BUFFER_SIZE, sizeof(buffer))); +} + +struct temporary_file { + std::filesystem::path path; + + temporary_file() { + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + path = std::filesystem::temp_directory_path() / ("test-llama-mmap-" + std::to_string(now) + ".bin"); + } + + ~temporary_file() { + std::error_code error; + std::filesystem::remove(path, error); + } +}; + +struct aligned_buffer { + void * data = nullptr; + + aligned_buffer(size_t alignment, size_t size) { + alignment = std::max(alignment, alignof(std::max_align_t)); +#if defined(_WIN32) + data = _aligned_malloc(size, alignment); +#else + if (posix_memalign(&data, alignment, size) != 0) { + data = nullptr; + } +#endif + } + + ~aligned_buffer() { +#if defined(_WIN32) + _aligned_free(data); +#else + free(data); +#endif + } + + aligned_buffer(const aligned_buffer &) = delete; + aligned_buffer & operator=(const aligned_buffer &) = delete; +}; + +static void write_test_file(const std::filesystem::path & path) { + std::ofstream file(path, std::ios::binary); + if (!file) { + throw std::runtime_error("failed to create test file"); + } + + std::vector buffer(test_io_size); + for (size_t offset = 0; offset < test_file_size; offset += buffer.size()) { + const size_t size = std::min(buffer.size(), test_file_size - offset); + for (size_t i = 0; i < size; ++i) { + buffer[i] = test_byte(offset + i); + } + file.write(reinterpret_cast(buffer.data()), size); + if (!file) { + throw std::runtime_error("failed to write test file"); + } + } +} + +static void check_file(testing & t, const std::filesystem::path & path, bool use_direct_io) { + llama_file file(path.string().c_str(), "rb", use_direct_io); + if (use_direct_io && !file.has_direct_io()) { + t.skip("direct I/O is unavailable for the test file"); + return; + } + t.assert_equal("requested I/O mode", use_direct_io, file.has_direct_io()); + t.assert_equal(test_file_size, file.size()); + t.assert_true("read alignment is nonzero", file.read_alignment() != 0); + + const size_t alignment = file.read_alignment(); + const std::vector> ranges = { + { 0, 1 }, + { 1, 31 }, + { alignment - 1, alignment + 3 }, + { LLAMA_DIRECT_IO_BUFFER_SIZE - 11, 27 }, + { test_file_size - 31, 31 }, + { 1, LLAMA_DIRECT_IO_BUFFER_SIZE + 31 }, + { 1, test_file_size - 1 }, + {0, test_file_size} + }; + for (const auto & range : ranges) { + std::vector buffer(range.second); + file.seek(range.first, SEEK_SET); + file.read_raw(buffer.data(), buffer.size()); + t.assert_equal("I/O mode after read", use_direct_io, file.has_direct_io()); + t.assert_true("read expected bytes", matches(buffer.data(), range.first, buffer.size())); + t.assert_equal(range.first + range.second, file.tell()); + } + + std::vector buffer(test_io_size - 1); + file.seek(0, SEEK_SET); + for (size_t offset = 0; offset < test_file_size; offset += buffer.size()) { + const size_t size = std::min(buffer.size(), test_file_size - offset); + file.read_raw(buffer.data(), size); + t.assert_equal("I/O mode after sequential read", use_direct_io, file.has_direct_io()); + t.assert_true("sequential read expected bytes", matches(buffer.data(), offset, size)); + t.assert_equal(offset + size, file.tell()); + } + + const size_t unsafe_size = ((LLAMA_DIRECT_IO_BUFFER_SIZE + 3 * alignment - 1) / alignment) * alignment; + aligned_buffer buffer_aligned(alignment, unsafe_size); + t.assert_true("allocate aligned buffer", buffer_aligned.data != nullptr); + if (buffer_aligned.data != nullptr) { + const size_t offset = 7 * alignment; + file.seek(offset, SEEK_SET); + file.read_raw_unsafe(buffer_aligned.data, unsafe_size); + t.assert_equal("I/O mode after unsafe read", use_direct_io, file.has_direct_io()); + t.assert_true("unsafe read expected bytes", matches(buffer_aligned.data, offset, unsafe_size)); + t.assert_equal(offset + unsafe_size, file.tell()); + + const size_t final_offset = (test_file_size - 1) & ~(alignment - 1); + file.seek(final_offset, SEEK_SET); + file.read_raw_unsafe(buffer_aligned.data, alignment); + t.assert_equal("I/O mode after unsafe EOF read", use_direct_io, file.has_direct_io()); + t.assert_true("unsafe EOF read expected bytes", matches(buffer_aligned.data, final_offset, test_file_size - final_offset)); + t.assert_equal(test_file_size, file.tell()); + } +} + +static void check_direct_io_fallback(testing & t, const std::filesystem::path & path) { + llama_file file(path.string().c_str(), "rb", true); + if (!file.has_direct_io()) { + t.skip("direct I/O is unavailable for the test file"); + return; + } + t.assert_true("direct I/O is enabled before fallback", file.has_direct_io()); + + const size_t alignment = file.read_alignment(); + aligned_buffer buffer(alignment, alignment + 1); + if (!t.assert_true("allocate aligned buffer", buffer.data != nullptr)) { + return; + } + auto * unaligned = static_cast(buffer.data) + 1; + const size_t offset = 7 * alignment; + file.seek(offset, SEEK_SET); + file.read_raw_unsafe(unaligned, alignment); + +#ifdef _WIN32 + t.assert_true("unaligned read disables direct I/O", !file.has_direct_io()); + t.assert_equal(size_t(1), file.read_alignment()); +#endif + t.assert_true("fallback read expected bytes", matches(unaligned, offset, alignment)); + t.assert_equal(offset + alignment, file.tell()); + + file.read_raw(unaligned, alignment); +#ifdef _WIN32 + t.assert_true("subsequent read stays buffered", !file.has_direct_io()); +#endif + t.assert_true("subsequent read expected bytes", matches(unaligned, offset + alignment, alignment)); + t.assert_equal(offset + 2 * alignment, file.tell()); +} + +static void check_mmap(testing & t, const std::filesystem::path & path) { + if (!llama_mmap::SUPPORTED) { + return; + } + + llama_file file(path.string().c_str(), "rb"); + llama_mmap mapping(&file, 0, false); + t.assert_equal(test_file_size, mapping.size()); + const auto * data = static_cast(mapping.addr()); + t.assert_true("mapping has an address", data != nullptr); + if (data != nullptr) { + t.assert_true("first byte", matches(data, 0, 1)); + t.assert_true("chunk boundary", matches(data + LLAMA_DIRECT_IO_BUFFER_SIZE - 11, LLAMA_DIRECT_IO_BUFFER_SIZE - 11, 27)); + t.assert_true("last bytes", matches(data + test_file_size - 31, test_file_size - 31, 31)); + } +} + +int main() { + testing t; + temporary_file test_file; + t.test("test_pattern", [&] (testing & t) { check_test_pattern(t); }); + t.test("write_test_file", [&] (testing &) { write_test_file(test_file.path); }); + + t.test("llama_file.buffered", [&] (testing & t) { check_file(t, test_file.path, false); }); + t.test("llama_file.direct", [&] (testing & t) { check_file(t, test_file.path, true); }); + t.test("llama_file.direct_fallback", [&] (testing & t) { check_direct_io_fallback(t, test_file.path); }); + t.test("llama_mmap", [&] (testing & t) { check_mmap(t, test_file.path); }); + return t.summary(); +}