Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

metal: fix memory unwire if model is freed without any GPU operations (#26082)

Browse Source 
* metal: fix memory leak if model is freed without any GPU operations

* metal: run dummy work only if residency sets are used

* metal: wrap function in #if defined

* metal: measure system-wide wired memory in test

* metal: always build regression test

Co-authored-by: YiChen Lv <63285796+forforever73@users.noreply.github.com>

---------

Co-authored-by: YiChen Lv <63285796+forforever73@users.noreply.github.com>
This commit is contained in:
Niklas Wenzel
2026-07-30 11:11:27 +03:00
committed by GitHub
co-authored by YiChen Lv
parent 21a5f5b7f9
commit d0bfb19812
4 changed files with 94 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-metal/ggml-metal-device.m
+37 -2
View File
@@ -557,7 +557,32 @@ struct ggml_metal_rsets {
dispatch_group_t d_group;
};
ggml_metal_rsets_t ggml_metal_rsets_init(void) {
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
static void ggml_metal_dummy_work(ggml_metal_device_t dev) {
if (dev->mtl_queue == nil) {
return;
}
@autoreleasepool {
// perform a minimal dummy operation on the GPU
id<MTLBuffer> buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate];
id<MTLCommandBuffer> cmd_buf = [dev->mtl_queue commandBuffer];
{
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
[encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0];
[encoder endEncoding];
}
[cmd_buf commit];
[buf release];
}
}
#endif
ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) {
ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets));
res->lock = [[NSLock alloc] init];
@@ -610,6 +635,15 @@ ggml_metal_rsets_t ggml_metal_rsets_init(void) {
#endif
});
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
// workaround for residency set memory not being released if no GPU operation occurs
// https://developer.apple.com/forums/thread/839089
// https://github.com/ggml-org/llama.cpp/issues/25937
ggml_metal_dummy_work(dev);
}
#endif
return res;
}
@@ -864,7 +898,7 @@ ggml_metal_device_t ggml_metal_device_init(int device) {
}
if (dev->props.use_residency_sets) {
dev->rsets = ggml_metal_rsets_init();
dev->rsets = ggml_metal_rsets_init(dev);
} else {
dev->rsets = nil;
}
@@ -1484,6 +1518,7 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {
if (buf->rset) {
[buf->rset endResidency];
[buf->rset removeAllAllocations];
[buf->rset commit];
[buf->rset release];
}
}