#include "llama-memory.h" #include "llama-ext.h" #include "llama-io.h" #include #include #include llama_memory_status llama_memory_status_combine(llama_memory_status s0, llama_memory_status s1) { bool has_update = false; switch (s0) { case LLAMA_MEMORY_STATUS_SUCCESS: { has_update = true; break; } case LLAMA_MEMORY_STATUS_NO_UPDATE: { break; } case LLAMA_MEMORY_STATUS_FAILED_PREPARE: case LLAMA_MEMORY_STATUS_FAILED_COMPUTE: { return s0; } } switch (s1) { case LLAMA_MEMORY_STATUS_SUCCESS: { has_update = true; break; } case LLAMA_MEMORY_STATUS_NO_UPDATE: { break; } case LLAMA_MEMORY_STATUS_FAILED_PREPARE: case LLAMA_MEMORY_STATUS_FAILED_COMPUTE: { return s1; } } // if either status has an update, then the combined status has an update return has_update ? LLAMA_MEMORY_STATUS_SUCCESS : LLAMA_MEMORY_STATUS_NO_UPDATE; } bool llama_memory_status_is_fail(llama_memory_status status) { switch (status) { case LLAMA_MEMORY_STATUS_SUCCESS: case LLAMA_MEMORY_STATUS_NO_UPDATE: { return false; } case LLAMA_MEMORY_STATUS_FAILED_PREPARE: case LLAMA_MEMORY_STATUS_FAILED_COMPUTE: { return true; } } return false; } // collects the buffers of the tensors that a state write visits, without keeping the data struct llama_memory_buffers_writer : public llama_io_write_i { std::set buffers; size_t size = 0; void write(const void *, size_t n) override { size += n; } void write_tensor(ggml_tensor * tensor, size_t, size_t n) override { buffers.insert(tensor->buffer); size += n; } size_t n_bytes() override { return size; } }; std::vector llama_memory_get_buffers(const llama_memory_t mem) { llama_memory_buffers_writer writer; mem->state_write(writer); return std::vector(writer.buffers.begin(), writer.buffers.end()); } ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector & data) { // the buffer base must be aligned to 32 bytes; the bytes before data.data() are never accessed uint8_t * base = (uint8_t *) ((uintptr_t) data.data() & ~(uintptr_t) 31); ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr(base, data.size() + (data.data() - base)); size_t off = 0; for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { if (t->view_src == nullptr) { ggml_backend_tensor_alloc(buf, t, data.data() + off); off += ggml_nbytes(t); } } for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { if (t->view_src != nullptr) { ggml_backend_view_init(t); } } return buf; }