mtmd: release/restore the encoder weights from VRAM on demand
Add clip_release_device/clip_restore_device (and mtmd_release_device/ mtmd_restore_device wrappers over the vision + audio contexts) that free the multimodal encoder's device weight buffer to a read-only host shadow and rebuild it on demand, using the same shadow/free/reallocate pattern as llama_model weights. No-op for a CPU-backed encoder. This lets the server drop the ~hundreds-of-MiB vision encoder from VRAM when it is not encoding an image. Assisted-by: Claude
This commit is contained in:
@@ -814,6 +814,24 @@ void mtmd_free(mtmd_context * ctx) {
|
||||
delete ctx;
|
||||
}
|
||||
|
||||
void mtmd_release_device(mtmd_context * ctx) {
|
||||
if (ctx == nullptr) {
|
||||
return;
|
||||
}
|
||||
if (ctx->ctx_v) { clip_release_device(ctx->ctx_v); }
|
||||
if (ctx->ctx_a) { clip_release_device(ctx->ctx_a); }
|
||||
}
|
||||
|
||||
bool mtmd_restore_device(mtmd_context * ctx) {
|
||||
if (ctx == nullptr) {
|
||||
return true;
|
||||
}
|
||||
bool ok = true;
|
||||
if (ctx->ctx_v) { ok = clip_restore_device(ctx->ctx_v) && ok; }
|
||||
if (ctx->ctx_a) { ok = clip_restore_device(ctx->ctx_a) && ok; }
|
||||
return ok;
|
||||
}
|
||||
|
||||
struct mtmd_tokenizer {
|
||||
mtmd_context * ctx;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user