From d32c33dab480d8c1606dabe23443b7c53519a804 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 26 Jul 2026 14:56:43 +0200 Subject: [PATCH] mtmd: release/restore the encoder weights from VRAM on demand Add clip_release_device/clip_restore_device (and mtmd_release_device/ mtmd_restore_device wrappers over the vision + audio contexts) that free the multimodal encoder's device weight buffer to a read-only host shadow and rebuild it on demand, using the same shadow/free/reallocate pattern as llama_model weights. No-op for a CPU-backed encoder. This lets the server drop the ~hundreds-of-MiB vision encoder from VRAM when it is not encoding an image. Assisted-by: Claude --- tools/mtmd/clip.cpp | 75 +++++++++++++++++++++++++++++++++++++++++++++ tools/mtmd/clip.h | 6 ++++ tools/mtmd/mtmd.cpp | 18 +++++++++++ tools/mtmd/mtmd.h | 6 ++++ 4 files changed, 105 insertions(+) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b88665064..f4d83bed4 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -158,6 +158,13 @@ struct clip_ctx { ggml_backend_t backend_cpu = nullptr; ggml_backend_buffer_ptr buf; + // on-demand device (VRAM) residency: the vision/audio encoder weights are read-only, so a host + // shadow is captured once and the device buffer can be freed while the model is cold, then + // rebuilt on wake (mirrors llama_model::release_device_weights). See clip_release_device(). + ggml_backend_buffer_type_t dev_buft = nullptr; + std::vector dev_shadow; + bool dev_released = false; + int max_nodes = 8192; ggml_backend_sched_ptr sched; @@ -3241,6 +3248,74 @@ void clip_free(clip_ctx * ctx) { delete ctx; } +void clip_release_device(struct clip_ctx * ctx) { + if (ctx == nullptr || ctx->dev_released) { + return; + } + ggml_backend_buffer_t buf = ctx->buf.get(); + if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) { + return; // CPU-backed encoder: nothing in VRAM to free + } + // ensure no encode is in flight before freeing the weights + if (ctx->sched) { + ggml_backend_sched_synchronize(ctx->sched.get()); + } + ctx->dev_buft = ggml_backend_buffer_get_type(buf); + + ggml_context * cd = ctx->ctx_data.get(); + // capture the host shadow once (weights are read-only): compact, stable iteration order, + // skipping view tensors (which alias a base and are restored implicitly) + if (ctx->dev_shadow.empty()) { + size_t total = 0; + for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) { + if (t->view_src == nullptr) { total += ggml_nbytes(t); } + } + ctx->dev_shadow.resize(total); + size_t off = 0; + for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) { + if (t->view_src != nullptr) { continue; } + const size_t n = ggml_nbytes(t); + ggml_backend_tensor_get(t, ctx->dev_shadow.data() + off, 0, n); + off += n; + } + } + + // free the device buffer and clear the now-dangling tensor pointers so restore reallocates cleanly + ctx->buf.reset(); + for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) { + t->buffer = nullptr; + t->data = nullptr; + } + ctx->dev_released = true; +} + +bool clip_restore_device(struct clip_ctx * ctx) { + if (ctx == nullptr || !ctx->dev_released) { + return true; + } + ggml_context * cd = ctx->ctx_data.get(); + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(cd, ctx->dev_buft); + if (buf == nullptr) { + LOG_ERR("%s: failed to reallocate encoder device buffer (out of VRAM?)\n", __func__); + return false; + } + ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + size_t off = 0; + for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) { + if (t->view_src != nullptr) { continue; } + const size_t n = ggml_nbytes(t); + ggml_backend_tensor_set(t, ctx->dev_shadow.data() + off, 0, n); + off += n; + } + ctx->buf.reset(buf); + ctx->dev_released = false; + return true; +} + +bool clip_weights_resident(const struct clip_ctx * ctx) { + return ctx == nullptr || !ctx->dev_released; +} + const char * clip_patch_merge_type(const struct clip_ctx * ctx) { return ctx->model.hparams.mm_patch_merge_type == PATCH_MERGE_SPATIAL_UNPAD ? "spatial_unpad" : "flat"; } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 967093a81..c0eda67c0 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -67,6 +67,12 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params void clip_free(struct clip_ctx * ctx); +// on-demand device (VRAM) residency: free/rebuild the encoder weight buffer to shrink an idle +// (cold) multimodal model's VRAM footprint. No-op for a CPU-backed encoder. See clip.cpp. +void clip_release_device(struct clip_ctx * ctx); +bool clip_restore_device(struct clip_ctx * ctx); +bool clip_weights_resident(const struct clip_ctx * ctx); + // TODO: should be enum, not string const char * clip_patch_merge_type(const struct clip_ctx * ctx); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index e10ccf186..00171a6d3 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -806,6 +806,24 @@ void mtmd_free(mtmd_context * ctx) { delete ctx; } +void mtmd_release_device(mtmd_context * ctx) { + if (ctx == nullptr) { + return; + } + if (ctx->ctx_v) { clip_release_device(ctx->ctx_v); } + if (ctx->ctx_a) { clip_release_device(ctx->ctx_a); } +} + +bool mtmd_restore_device(mtmd_context * ctx) { + if (ctx == nullptr) { + return true; + } + bool ok = true; + if (ctx->ctx_v) { ok = clip_restore_device(ctx->ctx_v) && ok; } + if (ctx->ctx_a) { ok = clip_restore_device(ctx->ctx_a) && ok; } + return ok; +} + struct mtmd_tokenizer { mtmd_context * ctx; diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 3b8c1200b..ba3204c76 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -127,6 +127,12 @@ MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname, MTMD_API void mtmd_free(mtmd_context * ctx); +// on-demand device (VRAM) residency: free / rebuild the vision+audio encoder weight buffers so an +// idle (cold) multimodal model releases its encoder VRAM and reclaims it on wake. No-op for a +// CPU-backed encoder. Restore returns false if reallocation failed (out of VRAM). +MTMD_API void mtmd_release_device(mtmd_context * ctx); +MTMD_API bool mtmd_restore_device(mtmd_context * ctx); + // whether we need to set non-causal mask before llama_decode // if chunk is nullptr, we assume the default case where chunk is an image chunk MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);