CUDA: Clear CUDA graphs after legacy memory pool flush (#26771)
Assisted-by: Claude Opus 5.5
This commit is contained in:
1 parent
d80619397c
commit
13786fbb3a
2 files changed
+25
-6
No files matched your search
@@ -1551,11 +1551,11 @@ struct ggml_backend_cuda_context {
|
||||
// pool
|
||||
std::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS];
|
||||
|
||||
static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(int device, int stream_no);
|
||||
static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(ggml_backend_cuda_context & ctx, int device, int stream_no);
|
||||
|
||||
ggml_cuda_pool & pool(int device) {
|
||||
if (pools[device][curr_stream_no] == nullptr) {
|
||||
pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no);
|
||||
pools[device][curr_stream_no] = new_pool_for_device(*this, device, curr_stream_no);
|
||||
}
|
||||
return *pools[device][curr_stream_no];
|
||||
}
|
||||
|
||||
@@ -457,6 +457,7 @@ const ggml_cuda_device_info & ggml_cuda_info() {
|
||||
struct ggml_cuda_pool_leg : public ggml_cuda_pool {
|
||||
static const int MAX_BUFFERS = 256;
|
||||
|
||||
ggml_backend_cuda_context & ctx;
|
||||
int device;
|
||||
struct ggml_cuda_buffer {
|
||||
void * ptr = nullptr;
|
||||
@@ -466,7 +467,8 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {
|
||||
ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {};
|
||||
size_t pool_size = 0;
|
||||
|
||||
explicit ggml_cuda_pool_leg(int device) :
|
||||
ggml_cuda_pool_leg(ggml_backend_cuda_context & ctx, int device) :
|
||||
ctx(ctx),
|
||||
device(device) {
|
||||
}
|
||||
|
||||
@@ -488,6 +490,20 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {
|
||||
}
|
||||
}
|
||||
|
||||
// captured CUDA graphs can still use freed buffers, so they must be captured again
|
||||
void invalidate_cuda_graphs() {
|
||||
#ifdef USE_CUDA_GRAPH
|
||||
int n_captured = 0;
|
||||
for (const auto & [key, graph] : ctx.cuda_graphs) {
|
||||
n_captured += graph->instance != nullptr;
|
||||
}
|
||||
if (n_captured > 0) {
|
||||
GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: memory freed, %d captured CUDA graphs must be captured again\n", device, n_captured);
|
||||
}
|
||||
ctx.cuda_graphs.clear();
|
||||
#endif // USE_CUDA_GRAPH
|
||||
}
|
||||
|
||||
void * alloc(size_t size, size_t * actual_size) override {
|
||||
#ifdef DEBUG_CUDA_MALLOC
|
||||
int nnz = 0;
|
||||
@@ -538,6 +554,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {
|
||||
device, look_ahead_size/1024.0/1024.0, cached_bytes/1024.0/1024.0);
|
||||
CUDA_CHECK(cudaDeviceSynchronize());
|
||||
clear_pool();
|
||||
invalidate_cuda_graphs();
|
||||
err = ggml_cuda_device_malloc(&ptr, look_ahead_size, device);
|
||||
if (err == cudaSuccess) {
|
||||
GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: retry succeeded\n", device);
|
||||
@@ -566,6 +583,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {
|
||||
ggml_cuda_set_device(device);
|
||||
CUDA_CHECK(cudaFree(ptr));
|
||||
pool_size -= size;
|
||||
invalidate_cuda_graphs();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -720,14 +738,15 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool {
|
||||
};
|
||||
#endif // defined(GGML_USE_VMM)
|
||||
|
||||
std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(int device,
|
||||
[[maybe_unused]] int stream_no) {
|
||||
std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(ggml_backend_cuda_context & ctx,
|
||||
int device,
|
||||
[[maybe_unused]] int stream_no) {
|
||||
#if defined(GGML_USE_VMM)
|
||||
if (ggml_cuda_info().devices[device].vmm) {
|
||||
return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_vmm(device));
|
||||
}
|
||||
#endif // defined(GGML_USE_VMM)
|
||||
return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(device));
|
||||
return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(ctx, device));
|
||||
}
|
||||
|
||||
// destroying a cuBLAS handle while a graph is being captured in a different thread can result in a CUDA error
|
||||
|
||||
Reference in new issue
Block a user