mtmd: release/restore the encoder weights from VRAM on demand

Add clip_release_device/clip_restore_device (and mtmd_release_device/
mtmd_restore_device wrappers over the vision + audio contexts) that free
the multimodal encoder's device weight buffer to a read-only host shadow
and rebuild it on demand, using the same shadow/free/reallocate pattern
as llama_model weights. No-op for a CPU-backed encoder. This lets the
server drop the ~hundreds-of-MiB vision encoder from VRAM when it is not
encoding an image.

Assisted-by: Claude
This commit is contained in:
Lumpiasty committed 2026-11-22 20:38:32 +01:00
1 parent 392787dee9
commit f5c6f04b4e
4 files changed
+106

No files matched your search

+61
View File
@@ -4617,6 +4617,7 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params
auto model = std::make_unique<mtmd_model>();
model->hparams.custom_image_min_tokens = params.image_min_tokens;
model->hparams.custom_image_max_tokens = params.image_max_tokens;
model->no_alloc = params.no_alloc;
loader.load_hparams(*model, modality);
loader.load_tensors(*model, buft, params.no_alloc);
const std::string & def = model->hparams.gen_default_voice;
@@ -4645,6 +4646,66 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params
return res;
}
void mtmd_model_release_device(mtmd_model & model) {
if (model.dev_released || model.no_alloc) {
return;
}
ggml_backend_buffer_t buf = model.buf.get();
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
return; // CPU-backed encoder: nothing in VRAM to free
}
model.dev_buft = ggml_backend_buffer_get_type(buf);
ggml_context * cd = model.ctx_data.get();
// capture the host shadow once (weights are read-only): compact, stable iteration order,
// skipping view tensors (which alias a base and are restored implicitly)
if (model.dev_shadow.empty()) {
size_t total = 0;
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
}
model.dev_shadow.resize(total);
size_t off = 0;
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
if (t->view_src != nullptr) { continue; }
const size_t n = ggml_nbytes(t);
ggml_backend_tensor_get(t, model.dev_shadow.data() + off, 0, n);
off += n;
}
}
// free the device buffer and clear the now-dangling tensor pointers so restore reallocates cleanly
model.buf.reset();
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
t->buffer = nullptr;
t->data = nullptr;
}
model.dev_released = true;
}
bool mtmd_model_restore_device(mtmd_model & model) {
if (!model.dev_released) {
return true;
}
ggml_context * cd = model.ctx_data.get();
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(cd, model.dev_buft);
if (buf == nullptr) {
LOG_ERR("%s: failed to reallocate encoder device buffer (out of VRAM?)\n", __func__);
return false;
}
ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
size_t off = 0;
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
if (t->view_src != nullptr) { continue; }
const size_t n = ggml_nbytes(t);
ggml_backend_tensor_set(t, model.dev_shadow.data() + off, 0, n);
off += n;
}
model.buf.reset(buf);
model.dev_released = false;
return true;
}
mtmd_model_caps mtmd_model_get_caps(const char * fname) {
mtmd_model_caps res;
mtmd_model_loader loader(fname, /* skip_tensors= */ true);
+13
View File
@@ -1161,6 +1161,14 @@ struct mtmd_model {
ggml_backend_buffer_ptr buf;
std::map<ggml_backend_dev_t, size_t> mem_weights;
// on-demand device (VRAM) residency: the weights are read-only, so a host shadow is captured once and
// the device buffer can be freed while the model is cold, then rebuilt on wake
// (mirrors llama_model::release_device_weights), see mtmd_model_release_device()
bool no_alloc = false;
ggml_backend_buffer_type_t dev_buft = nullptr;
std::vector<uint8_t> dev_shadow;
bool dev_released = false;
// embeddings
ggml_tensor * class_embedding = nullptr;
ggml_tensor * class_position_embedding = nullptr;
@@ -1614,6 +1622,11 @@ struct mtmd_models {
// the weights are allocated in buft, throws on error
mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft);
// free the device weight buffer to a host shadow and rebuild it on demand, a no-op for a CPU-backed model
// the caller must make sure that no graph using the weights is in flight
void mtmd_model_release_device(mtmd_model & model);
bool mtmd_model_restore_device(mtmd_model & model); // false if the reallocation failed (out of VRAM)
struct mtmd_model_caps {
bool has_vision;
bool has_audio;
+26
View File
@@ -1671,6 +1671,32 @@ std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::
return output;
}
void mtmd_release_device(mtmd_context * ctx) {
if (ctx == nullptr) {
return;
}
std::lock_guard<std::mutex> lock(ctx->compute_mutex);
// ensure no encode is in flight before freeing the weights
if (ctx->backends && ctx->backends->sched) {
ggml_backend_sched_synchronize(ctx->backends->sched.get());
}
for (auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) {
if (m) { mtmd_model_release_device(*m); }
}
}
bool mtmd_restore_device(mtmd_context * ctx) {
if (ctx == nullptr) {
return true;
}
std::lock_guard<std::mutex> lock(ctx->compute_mutex);
bool ok = true;
for (auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) {
if (m) { ok = mtmd_model_restore_device(*m) && ok; }
}
return ok;
}
struct mtmd_tokenizer {
const mtmd_context * ctx;
+6
View File
@@ -147,6 +147,12 @@ MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname,
MTMD_API void mtmd_free(mtmd_context * ctx);
// on-demand device (VRAM) residency: free / rebuild the vision+audio encoder weight buffers so an
// idle (cold) multimodal model releases its encoder VRAM and reclaims it on wake. No-op for a
// CPU-backed encoder. Restore returns false if reallocation failed (out of VRAM).
MTMD_API void mtmd_release_device(mtmd_context * ctx);
MTMD_API bool mtmd_restore_device(mtmd_context * ctx);
// whether we need to set non-causal mask before llama_decode
// if chunk is nullptr, we assume the default case where chunk is an image chunk
MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);