mtmd: release/restore the encoder weights from VRAM on demand
Add clip_release_device/clip_restore_device (and mtmd_release_device/ mtmd_restore_device wrappers over the vision + audio contexts) that free the multimodal encoder's device weight buffer to a read-only host shadow and rebuild it on demand, using the same shadow/free/reallocate pattern as llama_model weights. No-op for a CPU-backed encoder. This lets the server drop the ~hundreds-of-MiB vision encoder from VRAM when it is not encoding an image. Assisted-by: Claude
This commit is contained in:
@@ -67,6 +67,12 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
|
||||
|
||||
void clip_free(struct clip_ctx * ctx);
|
||||
|
||||
// on-demand device (VRAM) residency: free/rebuild the encoder weight buffer to shrink an idle
|
||||
// (cold) multimodal model's VRAM footprint. No-op for a CPU-backed encoder. See clip.cpp.
|
||||
void clip_release_device(struct clip_ctx * ctx);
|
||||
bool clip_restore_device(struct clip_ctx * ctx);
|
||||
bool clip_weights_resident(const struct clip_ctx * ctx);
|
||||
|
||||
// TODO: should be enum, not string
|
||||
const char * clip_patch_merge_type(const struct clip_ctx * ctx);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user