mtmd: share compute buffer among subcontext (#31945)

* shared compute buffer

* add load-time alloc for audio gen
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-10 11:45:09 +01:00
1 parent 203b5f404c
commit a591cbe17c
5 files changed
+149 -125

No files matched your search

+61 -69
View File
@@ -32,7 +32,7 @@ static void mtmd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_thr
}
}
std::shared_ptr<ggml_threadpool> mtmd_threadpool_init(int n_threads) {
static std::shared_ptr<ggml_threadpool> mtmd_threadpool_init(int n_threads) {
ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
if (!reg || n_threads < 2) {
@@ -63,14 +63,9 @@ ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device) {
return dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU);
}
mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, const mtmd_subctx_params & params) :
model(model),
threadpool(params.threadpool),
max_nodes(model.max_nodes),
flash_attn_type(params.flash_attn_type) {
if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO && !model.flash_attn_auto) {
flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED;
}
mtmd_backends::mtmd_backends(ggml_backend_dev_t device, int n_threads, int max_nodes,
ggml_backend_sched_eval_callback cb_eval, void * cb_eval_user_data) {
threadpool = mtmd_threadpool_init(n_threads);
backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
if (!backend_cpu) {
@@ -83,57 +78,70 @@ mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, const mtmd_subctx_par
set_threadpool_fn(backend_cpu, threadpool.get());
}
}
if (params.device != nullptr) {
backend = ggml_backend_dev_init(params.device, nullptr);
if (device != nullptr) {
backend = ggml_backend_dev_init(device, nullptr);
if (!backend) {
ggml_backend_free(backend_cpu);
throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",
__func__, ggml_backend_dev_name(params.device)));
__func__, ggml_backend_dev_name(device)));
}
}
if (backend) {
LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));
backend_ptrs.push_back(backend);
backend_buft.push_back(ggml_backend_get_default_buffer_type(backend));
ptrs.push_back(backend);
bufts.push_back(ggml_backend_get_default_buffer_type(backend));
} else {
backend = backend_cpu;
LOG_INF("%s: using CPU backend\n", __func__);
}
backend_ptrs.push_back(backend_cpu);
backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu));
ptrs.push_back(backend_cpu);
bufts.push_back(ggml_backend_get_default_buffer_type(backend_cpu));
sched.reset(
ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, true)
);
sched.reset(ggml_backend_sched_new(ptrs.data(), bufts.data(), ptrs.size(), max_nodes, false, true));
if (cb_eval != nullptr) {
ggml_backend_sched_set_eval_callback(sched.get(), cb_eval, cb_eval_user_data);
}
}
if (params.cb_eval != nullptr) {
ggml_backend_sched_set_eval_callback(sched.get(), params.cb_eval, params.cb_eval_user_data);
mtmd_backends::~mtmd_backends() {
// before the backends it uses
sched.reset();
ggml_backend_free(backend);
if (backend != backend_cpu) {
ggml_backend_free(backend_cpu);
}
}
std::map<ggml_backend_dev_t, size_t> mtmd_backends::get_mem_compute() const {
std::map<ggml_backend_dev_t, size_t> res;
for (ggml_backend_t b : ptrs) {
res[ggml_backend_get_device(b)] += ggml_backend_sched_get_buffer_size(sched.get(), b);
}
return res;
}
mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, mtmd_backends & backends, const mtmd_subctx_params & params) :
model(model),
backends(backends),
max_nodes(model.max_nodes),
flash_attn_type(params.flash_attn_type) {
if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO && !model.flash_attn_auto) {
flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED;
}
debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr;
buf_compute_meta.resize(max_nodes * ggml_tensor_overhead() + ggml_graph_overhead_custom(max_nodes, false));
// TODO: fix warmup for audio gen
auto batch = get_dummy_batch();
if (model.modality != MTMD_MODALITY_GEN_AUDIO) {
// check batching support
auto batch = get_dummy_batch();
support_batch = mtmd_get_graph_builder(this, batch)->support_batch();
if (params.warmup) {
warmup(batch);
}
}
}
mtmd_subcontext::~mtmd_subcontext() {
// the scheduler references the backends
sched.reset();
ggml_backend_free(backend);
if (backend != backend_cpu) {
ggml_backend_free(backend_cpu);
if (params.warmup) {
warmup(batch);
}
}
@@ -146,6 +154,11 @@ mtmd_image_f32_batch mtmd_subcontext::get_dummy_batch() const {
const int sz = hparams.warmup_image_size;
img.set_size({sz, sz}, false, false);
LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz);
} else if (model.modality == MTMD_MODALITY_GEN_AUDIO) {
// TODO: improve this, move to per-model impl
const int n_embd = model.n_mmproj_embd();
img.set_size({n_embd, 1}, false, true);
LOG_INF("%s: warmup with one gen-audio step\n", __func__);
} else {
// GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count,
// so the [1, 256] bound only applies to FFT-based models.
@@ -170,7 +183,7 @@ void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
if (!info.fattn && info.fattn_op) {
auto op = info.fattn_op;
LOG_WRN("%s: *****************************************************************\n", __func__);
LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(backend));
LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(backends.backend));
LOG_WRN("%s: op params: \n", __func__);
static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) {
LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn,
@@ -200,7 +213,7 @@ void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
(flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled");
// print ops that are not supported by the GPU backend (if there is one)
if (backend && backend != backend_cpu) {
if (backends.backend != backends.backend_cpu) {
std::vector<support_info_op> unsupported_ops;
for (const auto & op : info.ops) {
if (!op.is_accel) {
@@ -211,7 +224,7 @@ void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
LOG_WRN("%s: *****************************************************************\n", __func__);
LOG_WRN("%s: WARNING: the mmproj graph uses unsupported operators by the backend\n", __func__);
LOG_WRN("%s: the performance will be suboptimal \n", __func__);
LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(backend));
LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(backends.backend));
for (const auto & op : unsupported_ops) {
LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__,
ggml_op_name(op.op->op),
@@ -229,22 +242,9 @@ void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
mtmd_subcontext::support_info_graph mtmd_subcontext::reserve_compute_meta(const mtmd_image_f32_batch & batch) {
ggml_cgraph * gf = mtmd_get_graph_builder(this, batch)->build();
ggml_backend_sched_reserve(sched.get(), gf);
ggml_backend_sched_reserve(backends.sched.get(), gf);
mem_compute.clear();
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
ggml_backend_buffer_type_t buft = backend_buft[i];
size_t size = ggml_backend_sched_get_buffer_size(sched.get(), backend);
if (size > 1) {
LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__,
ggml_backend_buft_name(buft),
size / 1024.0 / 1024.0);
}
mem_compute[ggml_backend_get_device(backend)] += size;
}
const int n_splits = ggml_backend_sched_get_n_splits(sched.get());
const int n_splits = ggml_backend_sched_get_n_splits(backends.sched.get());
const int n_nodes = ggml_graph_n_nodes(gf);
LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes);
@@ -259,7 +259,7 @@ mtmd_subcontext::support_info_graph mtmd_subcontext::reserve_compute_meta(const
for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
ggml_tensor * node = ggml_graph_node(gf, i);
res.ops.push_back({node, true});
if (!ggml_backend_supports_op(backend, node)) {
if (!ggml_backend_supports_op(backends.backend, node)) {
res.ops.back().is_accel = false;
if (node->op == GGML_OP_FLASH_ATTN_EXT) {
res.fattn = false;
@@ -475,7 +475,7 @@ mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) {
ggml_format_name(t, "gen_state_%s", slot.name.c_str());
st->slots.push_back(t);
}
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend));
if (!st->buf) {
LOG_ERR("%s: failed to allocate the gen state\n", __func__);
return nullptr;
@@ -503,7 +503,7 @@ mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
ggml_format_name(t, "enc_state_%s", slot.name.c_str());
st->slots.push_back(t);
}
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend));
if (!st->buf) {
LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
return nullptr;
@@ -543,7 +543,7 @@ mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
st->k.push_back(k);
st->v.push_back(v);
}
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend));
if (!st->buf) {
LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
return nullptr;
@@ -588,14 +588,14 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
}
// build the inference graph
ggml_backend_sched_reset(sched.get());
ggml_backend_sched_reset(backends.sched.get());
ggml_cgraph * gf = mtmd_get_graph_builder(this, imgs, params)->build();
// the last node is the embedding tensor, code2wav has no out_embd
// read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal)
ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) {
if (!ggml_backend_sched_alloc_graph(backends.sched.get(), gf)) {
LOG_ERR("%s: failed to allocate compute graph\n", __func__);
return false;
}
@@ -2309,9 +2309,9 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
GGML_ABORT("Unknown projector type");
}
mtmd_backend_cpu_set_n_threads(backend_cpu, params->n_threads);
mtmd_backend_cpu_set_n_threads(backends.backend_cpu, params->n_threads);
auto status = ggml_backend_sched_graph_compute(sched.get(), gf);
auto status = ggml_backend_sched_graph_compute(backends.sched.get(), gf);
if (status != GGML_STATUS_SUCCESS) {
LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);
return false;
@@ -2709,11 +2709,3 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
return true;
}
std::map<ggml_backend_dev_t, size_t> mtmd_subcontext::get_mem_usage() const {
std::map<ggml_backend_dev_t, size_t> result = model.mem_weights;
for (auto & [dev, size] : mem_compute) {
result[dev] += size;
}
return result;
}