From 994fd757fc489800569e288d40db7ae81f7960a3 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 26 Jul 2026 14:55:50 +0200 Subject: [PATCH] ggml: add ggml_backend_free_scratch to drop Vulkan compute prealloc Add an optional backend interface method free_scratch (with a public ggml_backend_free_scratch wrapper) that frees transient/scratch device memory a backend holds outside of any allocated buffer, keeping the backend usable - the scratch is reallocated lazily on the next compute. Implement it for the Vulkan backend (ggml_backend_vk_free_scratch): free the prealloc_x/y/split_k/add_rms_partials and sync_staging device buffers and reset their sizes, so an idle/cold model does not hold the vision or matmul compute preallocations in VRAM. All other backends leave the hook null (no-op). Assisted-by: Claude --- ggml/include/ggml-backend.h | 5 +++++ ggml/src/ggml-backend-impl.h | 5 +++++ ggml/src/ggml-backend.cpp | 9 +++++++++ ggml/src/ggml-vulkan/ggml-vulkan.cpp | 30 ++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index 2924fdbe9..4ca9c7bcd 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -104,6 +104,11 @@ extern "C" { GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); + // Free transient/scratch device memory the backend holds outside of any allocated buffer + // (compute preallocations, staging buffers). No-op if the backend does not implement it. + // The backend remains usable; scratch is reallocated lazily on the next compute. + GGML_API void ggml_backend_free_scratch(ggml_backend_t backend); + // NOTE: will be removed, use device version instead GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft); diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index 9c56ec30c..7b7b859cb 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -137,6 +137,11 @@ extern "C" { // (optional) sort/optimize the nodes in the graph void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph); + + // (optional) free transient/scratch device memory the backend holds outside of any buffer + // (e.g. compute preallocations and staging buffers). The backend stays usable; the scratch + // is reallocated lazily on the next compute. Used to shrink an idle model's device footprint. + void (*free_scratch) (ggml_backend_t backend); }; struct ggml_backend { diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index f54c87a2a..c8006af15 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -420,6 +420,15 @@ void ggml_backend_synchronize(ggml_backend_t backend) { backend->iface.synchronize(backend); } +void ggml_backend_free_scratch(ggml_backend_t backend) { + GGML_ASSERT(backend); + if (backend->iface.free_scratch == NULL) { + return; + } + + backend->iface.free_scratch(backend); +} + ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) { GGML_ASSERT(backend); GGML_ASSERT(backend->iface.graph_plan_create != NULL); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 853773f13..4bbca288b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -15843,6 +15843,35 @@ static const char * ggml_backend_vk_name(ggml_backend_t backend) { return ctx->name.c_str(); } +// Free the compute-scratch device buffers (prealloc_* and the transfer staging buffer) without +// tearing down the backend (pipelines, command pools, fences and device stay alive). These buffers +// are reallocated lazily by ggml_vk_preallocate_buffers() on the next compute, so this just shrinks +// an idle model's device footprint. Mirrors the buffer-freeing subset of ggml_vk_cleanup(). +static void ggml_backend_vk_free_scratch(ggml_backend_t backend) { + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + VK_LOG_DEBUG("ggml_backend_vk_free_scratch(" << ctx->name << ")"); + + // discard any unsubmitted command buffer and wait for in-flight work before freeing + ctx->compute_ctx.reset(); + ggml_vk_synchronize(ctx); + + ggml_vk_destroy_buffer(ctx->prealloc_x); + ggml_vk_destroy_buffer(ctx->prealloc_y); + ggml_vk_destroy_buffer(ctx->prealloc_split_k); + ggml_vk_destroy_buffer(ctx->prealloc_add_rms_partials); + ggml_vk_destroy_buffer(ctx->sync_staging); + + ctx->prealloc_y_last_pipeline_used = nullptr; + ctx->prealloc_y_last_tensor_used = nullptr; + ctx->prealloc_y_last_decode_vector_staging = false; + + ctx->prealloc_size_x = 0; + ctx->prealloc_size_y = 0; + ctx->prealloc_size_split_k = 0; + ctx->prealloc_size_add_rms_partials = 0; + ctx->prealloc_size_add_rms_partials_offset = 0; +} + static void ggml_backend_vk_free(ggml_backend_t backend) { ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")"); @@ -17378,6 +17407,7 @@ static ggml_backend_i ggml_backend_vk_interface = { /* .event_record = */ ggml_backend_vk_event_record, /* .event_wait = */ ggml_backend_vk_event_wait, /* .graph_optimize = */ ggml_vk_graph_optimize, + /* .free_scratch = */ ggml_backend_vk_free_scratch, }; static ggml_guid_t ggml_backend_vk_guid() {