mtmd: release/restore the encoder weights from VRAM on demand

Add clip_release_device/clip_restore_device (and mtmd_release_device/
mtmd_restore_device wrappers over the vision + audio contexts) that free
the multimodal encoder's device weight buffer to a read-only host shadow
and rebuild it on demand, using the same shadow/free/reallocate pattern
as llama_model weights. No-op for a CPU-backed encoder. This lets the
server drop the ~hundreds-of-MiB vision encoder from VRAM when it is not
encoding an image.

Assisted-by: Claude
This commit is contained in:
2026-07-26 21:58:43 +02:00
parent 6d476d6ccc
commit da76fcb326
4 changed files with 105 additions and 0 deletions
+18
View File
@@ -814,6 +814,24 @@ void mtmd_free(mtmd_context * ctx) {
delete ctx;
}
void mtmd_release_device(mtmd_context * ctx) {
if (ctx == nullptr) {
return;
}
if (ctx->ctx_v) { clip_release_device(ctx->ctx_v); }
if (ctx->ctx_a) { clip_release_device(ctx->ctx_a); }
}
bool mtmd_restore_device(mtmd_context * ctx) {
if (ctx == nullptr) {
return true;
}
bool ok = true;
if (ctx->ctx_v) { ok = clip_restore_device(ctx->ctx_v) && ok; }
if (ctx->ctx_a) { ok = clip_restore_device(ctx->ctx_a) && ok; }
return ok;
}
struct mtmd_tokenizer {
mtmd_context * ctx;