diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp index 570750b9b..5ab047f44 100644 --- a/tools/mtmd/mtmd-model.cpp +++ b/tools/mtmd/mtmd-model.cpp @@ -4617,6 +4617,7 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params auto model = std::make_unique(); model->hparams.custom_image_min_tokens = params.image_min_tokens; model->hparams.custom_image_max_tokens = params.image_max_tokens; + model->no_alloc = params.no_alloc; loader.load_hparams(*model, modality); loader.load_tensors(*model, buft, params.no_alloc); const std::string & def = model->hparams.gen_default_voice; @@ -4645,6 +4646,66 @@ mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params return res; } +void mtmd_model_release_device(mtmd_model & model) { + if (model.dev_released || model.no_alloc) { + return; + } + ggml_backend_buffer_t buf = model.buf.get(); + if (buf == nullptr || ggml_backend_buffer_is_host(buf) || ggml_backend_buffer_get_size(buf) == 0) { + return; // CPU-backed encoder: nothing in VRAM to free + } + model.dev_buft = ggml_backend_buffer_get_type(buf); + + ggml_context * cd = model.ctx_data.get(); + // capture the host shadow once (weights are read-only): compact, stable iteration order, + // skipping view tensors (which alias a base and are restored implicitly) + if (model.dev_shadow.empty()) { + size_t total = 0; + for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) { + if (t->view_src == nullptr) { total += ggml_nbytes(t); } + } + model.dev_shadow.resize(total); + size_t off = 0; + for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) { + if (t->view_src != nullptr) { continue; } + const size_t n = ggml_nbytes(t); + ggml_backend_tensor_get(t, model.dev_shadow.data() + off, 0, n); + off += n; + } + } + + // free the device buffer and clear the now-dangling tensor pointers so restore reallocates cleanly + model.buf.reset(); + for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) { + t->buffer = nullptr; + t->data = nullptr; + } + model.dev_released = true; +} + +bool mtmd_model_restore_device(mtmd_model & model) { + if (!model.dev_released) { + return true; + } + ggml_context * cd = model.ctx_data.get(); + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(cd, model.dev_buft); + if (buf == nullptr) { + LOG_ERR("%s: failed to reallocate encoder device buffer (out of VRAM?)\n", __func__); + return false; + } + ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + size_t off = 0; + for (ggml_tensor * t = ggml_get_first_tensor(cd); t != nullptr; t = ggml_get_next_tensor(cd, t)) { + if (t->view_src != nullptr) { continue; } + const size_t n = ggml_nbytes(t); + ggml_backend_tensor_set(t, model.dev_shadow.data() + off, 0, n); + off += n; + } + model.buf.reset(buf); + model.dev_released = false; + return true; +} + mtmd_model_caps mtmd_model_get_caps(const char * fname) { mtmd_model_caps res; mtmd_model_loader loader(fname, /* skip_tensors= */ true); diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h index 10c3154e8..c777a12c9 100644 --- a/tools/mtmd/mtmd-model.h +++ b/tools/mtmd/mtmd-model.h @@ -1161,6 +1161,14 @@ struct mtmd_model { ggml_backend_buffer_ptr buf; std::map mem_weights; + // on-demand device (VRAM) residency: the weights are read-only, so a host shadow is captured once and + // the device buffer can be freed while the model is cold, then rebuilt on wake + // (mirrors llama_model::release_device_weights), see mtmd_model_release_device() + bool no_alloc = false; + ggml_backend_buffer_type_t dev_buft = nullptr; + std::vector dev_shadow; + bool dev_released = false; + // embeddings ggml_tensor * class_embedding = nullptr; ggml_tensor * class_position_embedding = nullptr; @@ -1614,6 +1622,11 @@ struct mtmd_models { // the weights are allocated in buft, throws on error mtmd_models mtmd_model_load(const char * fname, const mtmd_model_params & params, ggml_backend_buffer_type_t buft); +// free the device weight buffer to a host shadow and rebuild it on demand, a no-op for a CPU-backed model +// the caller must make sure that no graph using the weights is in flight +void mtmd_model_release_device(mtmd_model & model); +bool mtmd_model_restore_device(mtmd_model & model); // false if the reallocation failed (out of VRAM) + struct mtmd_model_caps { bool has_vision; bool has_audio; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index fdac762ac..eff314316 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1671,6 +1671,32 @@ std::vector> mtmd_group_mergeable_bitmaps(std:: return output; } +void mtmd_release_device(mtmd_context * ctx) { + if (ctx == nullptr) { + return; + } + std::lock_guard lock(ctx->compute_mutex); + // ensure no encode is in flight before freeing the weights + if (ctx->backends && ctx->backends->sched) { + ggml_backend_sched_synchronize(ctx->backends->sched.get()); + } + for (auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) { + if (m) { mtmd_model_release_device(*m); } + } +} + +bool mtmd_restore_device(mtmd_context * ctx) { + if (ctx == nullptr) { + return true; + } + std::lock_guard lock(ctx->compute_mutex); + bool ok = true; + for (auto * m : { ctx->models.v.get(), ctx->models.a.get(), ctx->models.gen_a.get() }) { + if (m) { ok = mtmd_model_restore_device(*m) && ok; } + } + return ok; +} + struct mtmd_tokenizer { const mtmd_context * ctx; diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 09cae0ad8..0bba736e2 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -147,6 +147,12 @@ MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname, MTMD_API void mtmd_free(mtmd_context * ctx); +// on-demand device (VRAM) residency: free / rebuild the vision+audio encoder weight buffers so an +// idle (cold) multimodal model releases its encoder VRAM and reclaims it on wake. No-op for a +// CPU-backed encoder. Restore returns false if reallocation failed (out of VRAM). +MTMD_API void mtmd_release_device(mtmd_context * ctx); +MTMD_API bool mtmd_restore_device(mtmd_context * ctx); + // whether we need to set non-causal mask before llama_decode // if chunk is nullptr, we assume the default case where chunk is an image chunk MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);