ggml: add ggml_backend_free_scratch to drop Vulkan compute prealloc

Add an optional backend interface method free_scratch (with a public
ggml_backend_free_scratch wrapper) that frees transient/scratch device
memory a backend holds outside of any allocated buffer, keeping the
backend usable - the scratch is reallocated lazily on the next compute.

Implement it for the Vulkan backend (ggml_backend_vk_free_scratch): free
the prealloc_x/y/split_k/add_rms_partials and sync_staging device buffers
and reset their sizes, so an idle/cold model does not hold the vision or
matmul compute preallocations in VRAM. All other backends leave the hook
null (no-op).

Assisted-by: Claude

ggml/include/ggml-backend.h

@@ -104,6 +104,11 @@ extern "C" {
     GGML_API enum ggml_status ggml_backend_graph_compute      (ggml_backend_t backend, struct ggml_cgraph * cgraph);
     GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
 
+    // Free transient/scratch device memory the backend holds outside of any allocated buffer
+    // (compute preallocations, staging buffers). No-op if the backend does not implement it.
+    // The backend remains usable; scratch is reallocated lazily on the next compute.
+    GGML_API void ggml_backend_free_scratch(ggml_backend_t backend);
+
     // NOTE: will be removed, use device version instead
     GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
     GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);

ggml/src/ggml-backend-impl.h

@@ -137,6 +137,11 @@ extern "C" {
 
         // (optional) sort/optimize the nodes in the graph
         void                      (*graph_optimize)    (ggml_backend_t backend, struct ggml_cgraph * cgraph);
+
+        // (optional) free transient/scratch device memory the backend holds outside of any buffer
+        // (e.g. compute preallocations and staging buffers). The backend stays usable; the scratch
+        // is reallocated lazily on the next compute. Used to shrink an idle model's device footprint.
+        void                      (*free_scratch)       (ggml_backend_t backend);
     };
 
     struct ggml_backend {

ggml/src/ggml-backend.cpp

@@ -420,6 +420,15 @@ void ggml_backend_synchronize(ggml_backend_t backend) {
     backend->iface.synchronize(backend);
 }
 
+void ggml_backend_free_scratch(ggml_backend_t backend) {
+    GGML_ASSERT(backend);
+    if (backend->iface.free_scratch == NULL) {
+        return;
+    }
+
+    backend->iface.free_scratch(backend);
+}
+
 ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
     GGML_ASSERT(backend);
     GGML_ASSERT(backend->iface.graph_plan_create != NULL);

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -15843,6 +15843,35 @@ static const char * ggml_backend_vk_name(ggml_backend_t backend) {
     return ctx->name.c_str();
 }
 
+// Free the compute-scratch device buffers (prealloc_* and the transfer staging buffer) without
+// tearing down the backend (pipelines, command pools, fences and device stay alive). These buffers
+// are reallocated lazily by ggml_vk_preallocate_buffers() on the next compute, so this just shrinks
+// an idle model's device footprint. Mirrors the buffer-freeing subset of ggml_vk_cleanup().
+static void ggml_backend_vk_free_scratch(ggml_backend_t backend) {
+    ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
+    VK_LOG_DEBUG("ggml_backend_vk_free_scratch(" << ctx->name << ")");
+
+    // discard any unsubmitted command buffer and wait for in-flight work before freeing
+    ctx->compute_ctx.reset();
+    ggml_vk_synchronize(ctx);
+
+    ggml_vk_destroy_buffer(ctx->prealloc_x);
+    ggml_vk_destroy_buffer(ctx->prealloc_y);
+    ggml_vk_destroy_buffer(ctx->prealloc_split_k);
+    ggml_vk_destroy_buffer(ctx->prealloc_add_rms_partials);
+    ggml_vk_destroy_buffer(ctx->sync_staging);
+
+    ctx->prealloc_y_last_pipeline_used = nullptr;
+    ctx->prealloc_y_last_tensor_used = nullptr;
+    ctx->prealloc_y_last_decode_vector_staging = false;
+
+    ctx->prealloc_size_x = 0;
+    ctx->prealloc_size_y = 0;
+    ctx->prealloc_size_split_k = 0;
+    ctx->prealloc_size_add_rms_partials = 0;
+    ctx->prealloc_size_add_rms_partials_offset = 0;
+}
+
 static void ggml_backend_vk_free(ggml_backend_t backend) {
     ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
     VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
@@ -17378,6 +17407,7 @@ static ggml_backend_i ggml_backend_vk_interface = {
     /* .event_record            = */ ggml_backend_vk_event_record,
     /* .event_wait              = */ ggml_backend_vk_event_wait,
     /* .graph_optimize          = */ ggml_vk_graph_optimize,
+    /* .free_scratch            = */ ggml_backend_vk_free_scratch,
 };
 
 static ggml_guid_t ggml_backend_vk_guid() {