mtmd: share compute buffer among subcontext (#31945)
* shared compute buffer * add load-time alloc for audio gen
This commit is contained in:
1 parent
203b5f404c
commit
a591cbe17c
5 files changed
+149
-125
No files matched your search
@@ -32,7 +32,7 @@ static void mtmd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_thr
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<ggml_threadpool> mtmd_threadpool_init(int n_threads) {
|
||||
static std::shared_ptr<ggml_threadpool> mtmd_threadpool_init(int n_threads) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
|
||||
ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
|
||||
if (!reg || n_threads < 2) {
|
||||
@@ -63,14 +63,9 @@ ggml_backend_dev_t mtmd_select_device(bool use_gpu, ggml_backend_dev_t device) {
|
||||
return dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU);
|
||||
}
|
||||
|
||||
mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, const mtmd_subctx_params & params) :
|
||||
model(model),
|
||||
threadpool(params.threadpool),
|
||||
max_nodes(model.max_nodes),
|
||||
flash_attn_type(params.flash_attn_type) {
|
||||
if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO && !model.flash_attn_auto) {
|
||||
flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED;
|
||||
}
|
||||
mtmd_backends::mtmd_backends(ggml_backend_dev_t device, int n_threads, int max_nodes,
|
||||
ggml_backend_sched_eval_callback cb_eval, void * cb_eval_user_data) {
|
||||
threadpool = mtmd_threadpool_init(n_threads);
|
||||
|
||||
backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
|
||||
if (!backend_cpu) {
|
||||
@@ -83,57 +78,70 @@ mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, const mtmd_subctx_par
|
||||
set_threadpool_fn(backend_cpu, threadpool.get());
|
||||
}
|
||||
}
|
||||
if (params.device != nullptr) {
|
||||
backend = ggml_backend_dev_init(params.device, nullptr);
|
||||
if (device != nullptr) {
|
||||
backend = ggml_backend_dev_init(device, nullptr);
|
||||
if (!backend) {
|
||||
ggml_backend_free(backend_cpu);
|
||||
throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",
|
||||
__func__, ggml_backend_dev_name(params.device)));
|
||||
__func__, ggml_backend_dev_name(device)));
|
||||
}
|
||||
}
|
||||
|
||||
if (backend) {
|
||||
LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));
|
||||
backend_ptrs.push_back(backend);
|
||||
backend_buft.push_back(ggml_backend_get_default_buffer_type(backend));
|
||||
ptrs.push_back(backend);
|
||||
bufts.push_back(ggml_backend_get_default_buffer_type(backend));
|
||||
} else {
|
||||
backend = backend_cpu;
|
||||
LOG_INF("%s: using CPU backend\n", __func__);
|
||||
}
|
||||
|
||||
backend_ptrs.push_back(backend_cpu);
|
||||
backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu));
|
||||
ptrs.push_back(backend_cpu);
|
||||
bufts.push_back(ggml_backend_get_default_buffer_type(backend_cpu));
|
||||
|
||||
sched.reset(
|
||||
ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, true)
|
||||
);
|
||||
sched.reset(ggml_backend_sched_new(ptrs.data(), bufts.data(), ptrs.size(), max_nodes, false, true));
|
||||
if (cb_eval != nullptr) {
|
||||
ggml_backend_sched_set_eval_callback(sched.get(), cb_eval, cb_eval_user_data);
|
||||
}
|
||||
}
|
||||
|
||||
if (params.cb_eval != nullptr) {
|
||||
ggml_backend_sched_set_eval_callback(sched.get(), params.cb_eval, params.cb_eval_user_data);
|
||||
mtmd_backends::~mtmd_backends() {
|
||||
// before the backends it uses
|
||||
sched.reset();
|
||||
ggml_backend_free(backend);
|
||||
if (backend != backend_cpu) {
|
||||
ggml_backend_free(backend_cpu);
|
||||
}
|
||||
}
|
||||
|
||||
std::map<ggml_backend_dev_t, size_t> mtmd_backends::get_mem_compute() const {
|
||||
std::map<ggml_backend_dev_t, size_t> res;
|
||||
for (ggml_backend_t b : ptrs) {
|
||||
res[ggml_backend_get_device(b)] += ggml_backend_sched_get_buffer_size(sched.get(), b);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
mtmd_subcontext::mtmd_subcontext(const mtmd_model & model, mtmd_backends & backends, const mtmd_subctx_params & params) :
|
||||
model(model),
|
||||
backends(backends),
|
||||
max_nodes(model.max_nodes),
|
||||
flash_attn_type(params.flash_attn_type) {
|
||||
if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_AUTO && !model.flash_attn_auto) {
|
||||
flash_attn_type = MTMD_FLASH_ATTN_TYPE_DISABLED;
|
||||
}
|
||||
|
||||
debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr;
|
||||
|
||||
buf_compute_meta.resize(max_nodes * ggml_tensor_overhead() + ggml_graph_overhead_custom(max_nodes, false));
|
||||
|
||||
// TODO: fix warmup for audio gen
|
||||
auto batch = get_dummy_batch();
|
||||
if (model.modality != MTMD_MODALITY_GEN_AUDIO) {
|
||||
// check batching support
|
||||
auto batch = get_dummy_batch();
|
||||
support_batch = mtmd_get_graph_builder(this, batch)->support_batch();
|
||||
|
||||
if (params.warmup) {
|
||||
warmup(batch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mtmd_subcontext::~mtmd_subcontext() {
|
||||
// the scheduler references the backends
|
||||
sched.reset();
|
||||
ggml_backend_free(backend);
|
||||
if (backend != backend_cpu) {
|
||||
ggml_backend_free(backend_cpu);
|
||||
if (params.warmup) {
|
||||
warmup(batch);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,6 +154,11 @@ mtmd_image_f32_batch mtmd_subcontext::get_dummy_batch() const {
|
||||
const int sz = hparams.warmup_image_size;
|
||||
img.set_size({sz, sz}, false, false);
|
||||
LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz);
|
||||
} else if (model.modality == MTMD_MODALITY_GEN_AUDIO) {
|
||||
// TODO: improve this, move to per-model impl
|
||||
const int n_embd = model.n_mmproj_embd();
|
||||
img.set_size({n_embd, 1}, false, true);
|
||||
LOG_INF("%s: warmup with one gen-audio step\n", __func__);
|
||||
} else {
|
||||
// GEMMA4UA uses n_mel_bins as a raw-waveform frame size (640), not a mel-bin count,
|
||||
// so the [1, 256] bound only applies to FFT-based models.
|
||||
@@ -170,7 +183,7 @@ void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
|
||||
if (!info.fattn && info.fattn_op) {
|
||||
auto op = info.fattn_op;
|
||||
LOG_WRN("%s: *****************************************************************\n", __func__);
|
||||
LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(backend));
|
||||
LOG_WRN("%s: WARNING: flash attention not supported by %s, memory usage will increase\n", __func__, ggml_backend_name(backends.backend));
|
||||
LOG_WRN("%s: op params: \n", __func__);
|
||||
static auto print_shape = [](const char * fn, const char * name, ggml_tensor * t) {
|
||||
LOG_WRN("%s: %s: type = %s, ne = [%d %d %d %d], nb = [%d %d %d %d]\n", fn,
|
||||
@@ -200,7 +213,7 @@ void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
|
||||
(flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) ? "enabled" : "disabled");
|
||||
|
||||
// print ops that are not supported by the GPU backend (if there is one)
|
||||
if (backend && backend != backend_cpu) {
|
||||
if (backends.backend != backends.backend_cpu) {
|
||||
std::vector<support_info_op> unsupported_ops;
|
||||
for (const auto & op : info.ops) {
|
||||
if (!op.is_accel) {
|
||||
@@ -211,7 +224,7 @@ void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
|
||||
LOG_WRN("%s: *****************************************************************\n", __func__);
|
||||
LOG_WRN("%s: WARNING: the mmproj graph uses unsupported operators by the backend\n", __func__);
|
||||
LOG_WRN("%s: the performance will be suboptimal \n", __func__);
|
||||
LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(backend));
|
||||
LOG_WRN("%s: list of unsupported ops (backend=%s):\n", __func__, ggml_backend_name(backends.backend));
|
||||
for (const auto & op : unsupported_ops) {
|
||||
LOG_WRN("%s: %16s: type = %s, ne = [%d %d %d %d]\n", __func__,
|
||||
ggml_op_name(op.op->op),
|
||||
@@ -229,22 +242,9 @@ void mtmd_subcontext::warmup(const mtmd_image_f32_batch & batch) {
|
||||
|
||||
mtmd_subcontext::support_info_graph mtmd_subcontext::reserve_compute_meta(const mtmd_image_f32_batch & batch) {
|
||||
ggml_cgraph * gf = mtmd_get_graph_builder(this, batch)->build();
|
||||
ggml_backend_sched_reserve(sched.get(), gf);
|
||||
ggml_backend_sched_reserve(backends.sched.get(), gf);
|
||||
|
||||
mem_compute.clear();
|
||||
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
||||
ggml_backend_t backend = backend_ptrs[i];
|
||||
ggml_backend_buffer_type_t buft = backend_buft[i];
|
||||
size_t size = ggml_backend_sched_get_buffer_size(sched.get(), backend);
|
||||
if (size > 1) {
|
||||
LOG_INF("%s: %10s compute buffer size = %8.2f MiB\n", __func__,
|
||||
ggml_backend_buft_name(buft),
|
||||
size / 1024.0 / 1024.0);
|
||||
}
|
||||
mem_compute[ggml_backend_get_device(backend)] += size;
|
||||
}
|
||||
|
||||
const int n_splits = ggml_backend_sched_get_n_splits(sched.get());
|
||||
const int n_splits = ggml_backend_sched_get_n_splits(backends.sched.get());
|
||||
const int n_nodes = ggml_graph_n_nodes(gf);
|
||||
|
||||
LOG_INF("%s: graph splits = %d, nodes = %d\n", __func__, n_splits, n_nodes);
|
||||
@@ -259,7 +259,7 @@ mtmd_subcontext::support_info_graph mtmd_subcontext::reserve_compute_meta(const
|
||||
for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
|
||||
ggml_tensor * node = ggml_graph_node(gf, i);
|
||||
res.ops.push_back({node, true});
|
||||
if (!ggml_backend_supports_op(backend, node)) {
|
||||
if (!ggml_backend_supports_op(backends.backend, node)) {
|
||||
res.ops.back().is_accel = false;
|
||||
if (node->op == GGML_OP_FLASH_ATTN_EXT) {
|
||||
res.fattn = false;
|
||||
@@ -475,7 +475,7 @@ mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) {
|
||||
ggml_format_name(t, "gen_state_%s", slot.name.c_str());
|
||||
st->slots.push_back(t);
|
||||
}
|
||||
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
|
||||
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend));
|
||||
if (!st->buf) {
|
||||
LOG_ERR("%s: failed to allocate the gen state\n", __func__);
|
||||
return nullptr;
|
||||
@@ -503,7 +503,7 @@ mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
|
||||
ggml_format_name(t, "enc_state_%s", slot.name.c_str());
|
||||
st->slots.push_back(t);
|
||||
}
|
||||
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
|
||||
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend));
|
||||
if (!st->buf) {
|
||||
LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
|
||||
return nullptr;
|
||||
@@ -543,7 +543,7 @@ mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
|
||||
st->k.push_back(k);
|
||||
st->v.push_back(v);
|
||||
}
|
||||
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backend));
|
||||
st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), backends.backend));
|
||||
if (!st->buf) {
|
||||
LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
|
||||
return nullptr;
|
||||
@@ -588,14 +588,14 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
}
|
||||
|
||||
// build the inference graph
|
||||
ggml_backend_sched_reset(sched.get());
|
||||
ggml_backend_sched_reset(backends.sched.get());
|
||||
ggml_cgraph * gf = mtmd_get_graph_builder(this, imgs, params)->build();
|
||||
|
||||
// the last node is the embedding tensor, code2wav has no out_embd
|
||||
// read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal)
|
||||
ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
|
||||
|
||||
if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) {
|
||||
if (!ggml_backend_sched_alloc_graph(backends.sched.get(), gf)) {
|
||||
LOG_ERR("%s: failed to allocate compute graph\n", __func__);
|
||||
return false;
|
||||
}
|
||||
@@ -2309,9 +2309,9 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
GGML_ABORT("Unknown projector type");
|
||||
}
|
||||
|
||||
mtmd_backend_cpu_set_n_threads(backend_cpu, params->n_threads);
|
||||
mtmd_backend_cpu_set_n_threads(backends.backend_cpu, params->n_threads);
|
||||
|
||||
auto status = ggml_backend_sched_graph_compute(sched.get(), gf);
|
||||
auto status = ggml_backend_sched_graph_compute(backends.sched.get(), gf);
|
||||
if (status != GGML_STATUS_SUCCESS) {
|
||||
LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);
|
||||
return false;
|
||||
@@ -2709,11 +2709,3 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::map<ggml_backend_dev_t, size_t> mtmd_subcontext::get_mem_usage() const {
|
||||
std::map<ggml_backend_dev_t, size_t> result = model.mem_weights;
|
||||
for (auto & [dev, size] : mem_compute) {
|
||||
result[dev] += size;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
Reference in new issue
Block a user