mtmd: release/restore the encoder weights from VRAM on demand
Add clip_release_device/clip_restore_device (and mtmd_release_device/ mtmd_restore_device wrappers over the vision + audio contexts) that free the multimodal encoder's device weight buffer to a read-only host shadow and rebuild it on demand, using the same shadow/free/reallocate pattern as llama_model weights. No-op for a CPU-backed encoder. This lets the server drop the ~hundreds-of-MiB vision encoder from VRAM when it is not encoding an image. Assisted-by: Claude
This commit is contained in:
1 parent
392787dee9
commit
f5c6f04b4e
4 files changed
+106
No files matched your search
@@ -4617,6 +4617,7 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params
|
||||
auto model = std::make_unique<mtmd_model>();
|
||||
model->hparams.custom_image_min_tokens = params.image_min_tokens;
|
||||
model->hparams.custom_image_max_tokens = params.image_max_tokens;
|
||||
model->no_alloc = params.no_alloc;
|
||||
loader.load_hparams(*model, modality);
|
||||
loader.load_tensors(*model, buft, params.no_alloc);
|
||||
const std::string & def = model->hparams.gen_default_voice;
|
||||
@@ -4645,6 +4646,66 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params
|
||||
return res;
|
||||
}
|
||||
|
||||
void mtmd_model_release_device(mtmd_model & model) {
|
||||
if (model.dev_released || model.no_alloc) {
|
||||
return;
|
||||
}
|
||||
ggml_backend_buffer_t buf = model.buf.get();
|
||||
if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) {
|
||||
return; // CPU-backed encoder: nothing in VRAM to free
|
||||
}
|
||||
model.dev_buft = ggml_backend_buffer_get_type(buf);
|
||||
|
||||
ggml_context * cd = model.ctx_data.get();
|
||||
// capture the host shadow once (weights are read-only): compact, stable iteration order,
|
||||
// skipping view tensors (which alias a base and are restored implicitly)
|
||||
if (model.dev_shadow.empty()) {
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||
}
|
||||
model.dev_shadow.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, model.dev_shadow.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
}
|
||||
|
||||
// free the device buffer and clear the now-dangling tensor pointers so restore reallocates cleanly
|
||||
model.buf.reset();
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||
t->buffer = nullptr;
|
||||
t->data = nullptr;
|
||||
}
|
||||
model.dev_released = true;
|
||||
}
|
||||
|
||||
bool mtmd_model_restore_device(mtmd_model & model) {
|
||||
if (!model.dev_released) {
|
||||
return true;
|
||||
}
|
||||
ggml_context * cd = model.ctx_data.get();
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(cd, model.dev_buft);
|
||||
if (buf == nullptr) {
|
||||
LOG_ERR("%s: failed to reallocate encoder device buffer (out of VRAM?)\n", __func__);
|
||||
return false;
|
||||
}
|
||||
ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_set(t, model.dev_shadow.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
model.buf.reset(buf);
|
||||
model.dev_released = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
mtmd_model_caps mtmd_model_get_caps(const char * fname) {
|
||||
mtmd_model_caps res;
|
||||
mtmd_model_loader loader(fname, /* skip_tensors= */ true);
|
||||
|
||||
@@ -1161,6 +1161,14 @@ struct mtmd_model {
|
||||
ggml_backend_buffer_ptr buf;
|
||||
std::map<ggml_backend_dev_t, size_t> mem_weights;
|
||||
|
||||
// on-demand device (VRAM) residency: the weights are read-only, so a host shadow is captured once and
|
||||
// the device buffer can be freed while the model is cold, then rebuilt on wake
|
||||
// (mirrors llama_model::release_device_weights), see mtmd_model_release_device()
|
||||
bool no_alloc = false;
|
||||
ggml_backend_buffer_type_t dev_buft = nullptr;
|
||||
std::vector<uint8_t> dev_shadow;
|
||||
bool dev_released = false;
|
||||
|
||||
// embeddings
|
||||
ggml_tensor * class_embedding = nullptr;
|
||||
ggml_tensor * class_position_embedding = nullptr;
|
||||
@@ -1614,6 +1622,11 @@ struct mtmd_models {
|
||||
// the weights are allocated in buft, throws on error
|
||||
mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft);
|
||||
|
||||
// free the device weight buffer to a host shadow and rebuild it on demand, a no-op for a CPU-backed model
|
||||
// the caller must make sure that no graph using the weights is in flight
|
||||
void mtmd_model_release_device(mtmd_model & model);
|
||||
bool mtmd_model_restore_device(mtmd_model & model); // false if the reallocation failed (out of VRAM)
|
||||
|
||||
struct mtmd_model_caps {
|
||||
bool has_vision;
|
||||
bool has_audio;
|
||||
|
||||
@@ -1671,6 +1671,32 @@ std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::
|
||||
return output;
|
||||
}
|
||||
|
||||
void mtmd_release_device(mtmd_context * ctx) {
|
||||
if (ctx == nullptr) {
|
||||
return;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(ctx->compute_mutex);
|
||||
// ensure no encode is in flight before freeing the weights
|
||||
if (ctx->backends && ctx->backends->sched) {
|
||||
ggml_backend_sched_synchronize(ctx->backends->sched.get());
|
||||
}
|
||||
for (auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) {
|
||||
if (m) { mtmd_model_release_device(*m); }
|
||||
}
|
||||
}
|
||||
|
||||
bool mtmd_restore_device(mtmd_context * ctx) {
|
||||
if (ctx == nullptr) {
|
||||
return true;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(ctx->compute_mutex);
|
||||
bool ok = true;
|
||||
for (auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) {
|
||||
if (m) { ok = mtmd_model_restore_device(*m) && ok; }
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
struct mtmd_tokenizer {
|
||||
const mtmd_context * ctx;
|
||||
|
||||
|
||||
@@ -147,6 +147,12 @@ MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname,
|
||||
|
||||
MTMD_API void mtmd_free(mtmd_context * ctx);
|
||||
|
||||
// on-demand device (VRAM) residency: free / rebuild the vision+audio encoder weight buffers so an
|
||||
// idle (cold) multimodal model releases its encoder VRAM and reclaims it on wake. No-op for a
|
||||
// CPU-backed encoder. Restore returns false if reallocation failed (out of VRAM).
|
||||
MTMD_API void mtmd_release_device(mtmd_context * ctx);
|
||||
MTMD_API bool mtmd_restore_device(mtmd_context * ctx);
|
||||
|
||||
// whether we need to set non-causal mask before llama_decode
|
||||
// if chunk is nullptr, we assume the default case where chunk is an image chunk
|
||||
MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);
|
||||
|
||||
Reference in new issue
Block a user