server: release the GPU token promptly on shutdown
Two ways a shutting-down server kept the VRAM arbiter's token, stalling every other model: - SIGTERM with a request in flight: terminate() stopped only the task queue. The HTTP handler kept waiting for results the loop would never send, the main thread blocked joining it, and the token was released only in the destructor after that join - i.e. never, until the client went away. Terminate the result queue too (readers now get "server is shutting down": a 503, or an SSE error event), and let wait_until_no_sleep give up once stopping. - SIGTERM while waiting for the token: the loop thread sat in a blocking flock inside the wake callback, holding mutex_tasks, so terminate() waited for the current holder to finish and then for a pointless restore of our weights. Poll the lock non-blocking and abort the wait on shutdown; the aborted wake stays cold, skips the restore, and the loop exits before update_slots. Also release the token right after start_loop() returns rather than in the destructor, and only after the device buffers are actually freed: the old order unlocked before destroy(), so the next model could start uploading into VRAM that was still occupied. Freeing at exit uses a new llama_context_discard_device(), which drops weights, KV, MoE cache and scratch without capturing host shadows first. Off production: SIGTERM mid-generation or mid-wait now exits in ~0.2 s with exit 0 and frees the token within 5-70 ms (previously never / after the holder's whole generation). Handoff latency and greedy output are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
11044b35e3
commit
6c480f7e70
17 files changed
+155
-52
No files matched your search
@@ -600,6 +600,9 @@ extern "C" {
|
||||
// Intended for time-sharing a single GPU between several always-loaded models.
|
||||
LLAMA_API void llama_context_release_device(struct llama_context * ctx, bool evict_kv);
|
||||
LLAMA_API void llama_context_restore_device(struct llama_context * ctx);
|
||||
// Free all device buffers (weights, KV, compute) for good, without host copies, e.g. to hand
|
||||
// the VRAM to another process before exit. The context must not be used for compute afterwards.
|
||||
LLAMA_API void llama_context_discard_device(struct llama_context * ctx);
|
||||
LLAMA_API bool llama_context_weights_resident(const struct llama_context * ctx);
|
||||
LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
|
||||
|
||||
|
||||
@@ -949,7 +949,27 @@ void llama_context::release_device(bool evict_kv) {
|
||||
}
|
||||
}
|
||||
|
||||
void llama_context::discard_device() {
|
||||
// same as release_device(true), minus the host copies of the weights and the KV
|
||||
synchronize();
|
||||
model.release_device_weights(/*keep_shadow =*/ false);
|
||||
if (moe_cache) {
|
||||
moe_cache->release_device();
|
||||
}
|
||||
if (memory) {
|
||||
memory->release_device_buffers(/*keep_shadow =*/ false);
|
||||
}
|
||||
kv_device_evicted = true;
|
||||
device_discarded = true;
|
||||
sched.reset();
|
||||
sched_need_reserve = true;
|
||||
for (auto & backend : backends) {
|
||||
ggml_backend_free_scratch(backend.get());
|
||||
}
|
||||
}
|
||||
|
||||
void llama_context::restore_device() {
|
||||
GGML_ASSERT(!device_discarded && "device buffers were discarded");
|
||||
if (device_resident() && !kv_device_evicted && sched) {
|
||||
return;
|
||||
}
|
||||
@@ -4316,6 +4336,10 @@ void llama_context_restore_device(llama_context * ctx) {
|
||||
ctx->restore_device();
|
||||
}
|
||||
|
||||
void llama_context_discard_device(llama_context * ctx) {
|
||||
ctx->discard_device();
|
||||
}
|
||||
|
||||
bool llama_context_weights_resident(const llama_context * ctx) {
|
||||
return ctx->device_resident();
|
||||
}
|
||||
|
||||
@@ -67,6 +67,9 @@ struct llama_context {
|
||||
// other model) - this adds a D2H/H2D copy of the live KV on each cycle.
|
||||
void release_device(bool evict_kv = false);
|
||||
void restore_device();
|
||||
// free every device buffer for good, without host copies (before exit). The context must not
|
||||
// compute again: restore_device() aborts.
|
||||
void discard_device();
|
||||
|
||||
// the model weights and the MoE cache are on the device
|
||||
bool device_resident() const;
|
||||
@@ -405,6 +408,8 @@ private:
|
||||
|
||||
// on-demand device residency: true while the KV cache device buffers have been evicted to host
|
||||
bool kv_device_evicted = false;
|
||||
// set by discard_device(): the device buffers are gone and cannot be restored
|
||||
bool device_discarded = false;
|
||||
|
||||
ggml_backend_t backend_cpu = nullptr;
|
||||
std::vector<ggml_backend_ptr> backends;
|
||||
|
||||
@@ -264,9 +264,9 @@ void llama_kv_cache_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id,
|
||||
kv_swa ->state_read(io, seq_id, flags);
|
||||
}
|
||||
|
||||
void llama_kv_cache_iswa::release_device_buffers() {
|
||||
kv_base->release_device_buffers();
|
||||
kv_swa->release_device_buffers();
|
||||
void llama_kv_cache_iswa::release_device_buffers(bool keep_shadow) {
|
||||
kv_base->release_device_buffers(keep_shadow);
|
||||
kv_swa->release_device_buffers(keep_shadow);
|
||||
}
|
||||
|
||||
bool llama_kv_cache_iswa::restore_device_buffers() {
|
||||
|
||||
@@ -83,7 +83,7 @@ public:
|
||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
void release_device_buffers() override;
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers() override;
|
||||
|
||||
//
|
||||
|
||||
+14
-12
@@ -1929,7 +1929,7 @@ size_t llama_kv_cache::total_size() const {
|
||||
return size;
|
||||
}
|
||||
|
||||
void llama_kv_cache::release_device_buffers() {
|
||||
void llama_kv_cache::release_device_buffers(bool keep_shadow) {
|
||||
// NOTE: the caller must have synchronized the backend so nothing references these buffers.
|
||||
// The KV cache is read-write, so its host shadow is (re)captured fresh on every release.
|
||||
if (dev_released) {
|
||||
@@ -1948,17 +1948,19 @@ void llama_kv_cache::release_device_buffers() {
|
||||
sh.buft = ggml_backend_buffer_get_type(buf);
|
||||
|
||||
// capture live contents compactly in stable iteration order (skip views, which alias a base)
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||
}
|
||||
sh.data.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, sh.data.data() + off, 0, n);
|
||||
off += n;
|
||||
if (keep_shadow) {
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||
}
|
||||
sh.data.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, sh.data.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
}
|
||||
|
||||
freed += ggml_backend_buffer_get_size(buf);
|
||||
|
||||
@@ -152,7 +152,7 @@ public:
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
// on-demand device (VRAM) residency (see llama_memory_i)
|
||||
void release_device_buffers() override;
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers() override;
|
||||
|
||||
//
|
||||
|
||||
@@ -205,10 +205,10 @@ void llama_memory_hybrid::state_read(llama_io_read_i & io, llama_seq_id seq_id,
|
||||
}
|
||||
}
|
||||
|
||||
void llama_memory_hybrid::release_device_buffers() {
|
||||
void llama_memory_hybrid::release_device_buffers(bool keep_shadow) {
|
||||
// evict both the attention KV (grows with context) and the recurrent/SSM state
|
||||
mem_attn->release_device_buffers();
|
||||
mem_recr->release_device_buffers();
|
||||
mem_attn->release_device_buffers(keep_shadow);
|
||||
mem_recr->release_device_buffers(keep_shadow);
|
||||
}
|
||||
|
||||
bool llama_memory_hybrid::restore_device_buffers() {
|
||||
|
||||
@@ -76,7 +76,7 @@ public:
|
||||
void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
void release_device_buffers() override;
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers() override;
|
||||
|
||||
//
|
||||
|
||||
@@ -754,7 +754,7 @@ size_t llama_memory_recurrent::total_size() const {
|
||||
return size;
|
||||
}
|
||||
|
||||
void llama_memory_recurrent::release_device_buffers() {
|
||||
void llama_memory_recurrent::release_device_buffers(bool keep_shadow) {
|
||||
// Same mechanism as llama_kv_cache: the recurrent (SSM/conv) state is read-write, so its host
|
||||
// shadow is (re)captured on every release. The caller must have synchronized the backend.
|
||||
if (dev_released) {
|
||||
@@ -772,17 +772,19 @@ void llama_memory_recurrent::release_device_buffers() {
|
||||
sh.releasable = true;
|
||||
sh.buft = ggml_backend_buffer_get_type(buf);
|
||||
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||
}
|
||||
sh.data.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, sh.data.data() + off, 0, n);
|
||||
off += n;
|
||||
if (keep_shadow) {
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src == nullptr) { total += ggml_nbytes(t); }
|
||||
}
|
||||
sh.data.resize(total);
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) { continue; }
|
||||
const size_t n = ggml_nbytes(t);
|
||||
ggml_backend_tensor_get(t, sh.data.data() + off, 0, n);
|
||||
off += n;
|
||||
}
|
||||
}
|
||||
|
||||
freed += ggml_backend_buffer_get_size(buf);
|
||||
|
||||
@@ -67,7 +67,7 @@ public:
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
// on-demand device (VRAM) residency (see llama_memory_i)
|
||||
void release_device_buffers() override;
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers() override;
|
||||
|
||||
uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot())
|
||||
|
||||
+2
-1
@@ -133,7 +133,8 @@ struct llama_memory_i {
|
||||
// Free this memory's device (VRAM) buffers to a host shadow and rebuild them on demand, to
|
||||
// hand VRAM to another model while keeping the cached data (no re-prefill). The caller must
|
||||
// have synchronized the backend first. Default: no-op (the memory stays resident).
|
||||
virtual void release_device_buffers() {}
|
||||
// keep_shadow = false frees the buffers for good (before exit): no host copy, no restore.
|
||||
virtual void release_device_buffers(bool /*keep_shadow*/ = true) {}
|
||||
virtual bool restore_device_buffers() { return true; }
|
||||
};
|
||||
|
||||
|
||||
+2
-2
@@ -2435,7 +2435,7 @@ ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM
|
||||
tn, ne, flags);
|
||||
}
|
||||
|
||||
void llama_model::release_device_weights() const {
|
||||
void llama_model::release_device_weights(bool keep_shadow) const {
|
||||
// NOTE: the caller is responsible for synchronizing the backend scheduler first, so that no
|
||||
// compute is in flight referencing these buffers when they are freed.
|
||||
size_t freed = 0;
|
||||
@@ -2452,7 +2452,7 @@ void llama_model::release_device_weights() const {
|
||||
// Store tensor bytes compactly in stable iteration order (independent of buffer layout /
|
||||
// alignment padding) so restore does not depend on the re-allocated offsets matching.
|
||||
// View tensors alias their base, so they are skipped (restored implicitly via their base).
|
||||
if (slot.shadow.empty()) {
|
||||
if (keep_shadow && slot.shadow.empty()) {
|
||||
size_t total = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) continue;
|
||||
|
||||
+1
-1
@@ -896,7 +896,7 @@ struct llama_model {
|
||||
// shadow) and rebuild them on demand. Used to time-share VRAM between models without
|
||||
// reloading or losing the KV cache. release_device_weights() requires the caller to have
|
||||
// synchronized the backend scheduler first.
|
||||
void release_device_weights() const;
|
||||
void release_device_weights(bool keep_shadow = true) const;
|
||||
bool restore_device_weights() const;
|
||||
bool weights_resident() const;
|
||||
|
||||
|
||||
@@ -1080,13 +1080,13 @@ public:
|
||||
}
|
||||
|
||||
~server_context_impl() {
|
||||
// release the VRAM token before tearing anything down
|
||||
vram_share_shutdown();
|
||||
if (!sleeping) {
|
||||
// destroy() is already called when entering sleeping state
|
||||
// we don't call it again here to avoid double free
|
||||
destroy();
|
||||
}
|
||||
// only once our VRAM is freed (normally vram_share_exit() has done both already)
|
||||
vram_share_shutdown();
|
||||
}
|
||||
|
||||
server_metrics get_metrics() const {
|
||||
@@ -1249,7 +1249,8 @@ private:
|
||||
// take the shared VRAM token: register on want.lock, then wait for token.lock. want.lock is
|
||||
// level-triggered state, so the holder still learns of us if it is loading or restoring when we
|
||||
// start to wait. No timeout: the holder can be mid-generation, and decoding without the token
|
||||
// risks an OOM that kills both processes.
|
||||
// risks an OOM that kills both processes. Returns false without the token if the server shuts
|
||||
// down meanwhile (the caller checks queue_tasks.is_stopping()).
|
||||
bool vram_take_token() {
|
||||
#if !defined(_WIN32)
|
||||
if (!vram_flock) {
|
||||
@@ -1263,8 +1264,15 @@ private:
|
||||
SRV_INF("%s", "VRAM arbiter: waiting for the GPU token\n");
|
||||
logged = true;
|
||||
}
|
||||
flock(vram_lock_fd, LOCK_EX); // blocking: the kernel grants it the moment the holder unlocks
|
||||
continue; // re-check with LOCK_NB, which also succeeds on a lock we already own
|
||||
// poll, not a blocking flock: terminate() must be able to stop this wait, and it can
|
||||
// only signal us through an atomic (we hold mutex_tasks here)
|
||||
if (queue_tasks.is_stopping()) {
|
||||
SRV_INF("%s", "VRAM arbiter: shutting down, no longer waiting for the GPU token\n");
|
||||
vram_want_release();
|
||||
return false;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
||||
continue;
|
||||
}
|
||||
if (errno == EINTR) {
|
||||
continue; // our signal handlers run without SA_RESTART
|
||||
@@ -1281,12 +1289,19 @@ private:
|
||||
}
|
||||
|
||||
// acquire the VRAM token (blocking) and bring weights back to the device. Called on the loop
|
||||
// thread right before a decode, so it never races compute.
|
||||
void vram_ensure_warm() {
|
||||
// thread right before a decode, so it never races compute. Returns false, still cold, if the
|
||||
// server shuts down first: the caller must not touch the device then.
|
||||
bool vram_ensure_warm() {
|
||||
if (!vram_only || !vram_cold) {
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if (queue_tasks.is_stopping()) {
|
||||
return false; // no restore just to exit
|
||||
}
|
||||
if (!vram_take_token()) {
|
||||
if (queue_tasks.is_stopping()) {
|
||||
return false;
|
||||
}
|
||||
SRV_WRN("%s", "VRAM arbiter: restoring weights without the GPU token\n");
|
||||
}
|
||||
llama_context_restore_device(ctx_tgt);
|
||||
@@ -1302,6 +1317,25 @@ private:
|
||||
// bring the weights up. We dropped want.lock when we took the token, so the next probe
|
||||
// reports only the waiters that are still there.
|
||||
vram_cold = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// the loop has stopped: free our VRAM and hand the token over now, without waiting for the HTTP
|
||||
// threads and the teardown. Discarded, not released: no host copy, and the context never decodes
|
||||
// again. Runs on the loop thread after the loop returned, so nothing else uses the device.
|
||||
void vram_share_exit() {
|
||||
if (vram_only && !vram_cold && ctx_tgt != nullptr) {
|
||||
llama_context_discard_device(ctx_tgt);
|
||||
if (ctx_dft != nullptr) {
|
||||
llama_context_discard_device(ctx_dft);
|
||||
}
|
||||
if (mctx != nullptr) {
|
||||
mtmd_release_device(mctx);
|
||||
}
|
||||
vram_cold = true;
|
||||
SRV_INF("%s", "VRAM arbiter: loop stopped, device memory freed, releasing the GPU token\n");
|
||||
}
|
||||
vram_share_shutdown();
|
||||
}
|
||||
|
||||
void vram_share_shutdown() {
|
||||
@@ -1418,6 +1452,7 @@ private:
|
||||
// (e.g. SWA checkpoint creation reads it via ggml_backend_tensor_get) before the
|
||||
// decode-time vram_ensure_warm(), so with KV eviction the KV must already be resident
|
||||
// here or those reads hit a freed (null) buffer.
|
||||
// false (still cold) only on shutdown: the loop then exits before update_slots
|
||||
vram_ensure_warm();
|
||||
}
|
||||
sleeping = new_state;
|
||||
@@ -4885,7 +4920,9 @@ private:
|
||||
// VRAM arbiter: make sure our weights are resident (acquiring the shared GPU token first)
|
||||
// before we decode. Runs on the loop thread before entering yield_to_queue(), so it never
|
||||
// races an in-flight decode.
|
||||
vram_ensure_warm();
|
||||
if (!vram_ensure_warm()) {
|
||||
throw std::runtime_error("server is shutting down");
|
||||
}
|
||||
|
||||
// yield to the queue, so we can still handle metrics tasks while decoding
|
||||
// note: the sync is done here too, so that the wait is also covered by the yield
|
||||
@@ -5403,9 +5440,12 @@ bool server_context::load_model(common_params & params) {
|
||||
void server_context::start_loop() {
|
||||
auto & params = impl->params_base;
|
||||
impl->queue_tasks.start_loop(params.sleep_idle_seconds * 1000);
|
||||
impl->vram_share_exit();
|
||||
}
|
||||
|
||||
void server_context::terminate() {
|
||||
// wake the HTTP threads waiting for results first: the loop will not produce them any more
|
||||
impl->queue_results.terminate();
|
||||
impl->queue_tasks.terminate();
|
||||
}
|
||||
|
||||
|
||||
@@ -116,7 +116,8 @@ void server_queue::wait_until_no_sleep() {
|
||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||
// re-ask on every wake: the loop clears req_stop_sleeping on the way into sleep, so a loop that
|
||||
// goes back to sleep before we run would strand us here forever
|
||||
while (sleeping) {
|
||||
// a loop that terminates while asleep never wakes: give up then, the reader reports the shutdown
|
||||
while (sleeping && !stopping) {
|
||||
if (!req_stop_sleeping) {
|
||||
QUE_DBG("%s", "requesting to stop sleeping\n");
|
||||
req_stop_sleeping = true;
|
||||
@@ -128,6 +129,7 @@ void server_queue::wait_until_no_sleep() {
|
||||
}
|
||||
|
||||
void server_queue::terminate() {
|
||||
stopping = true; // before the lock: the loop may hold it while it waits for the VRAM token
|
||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||
running = false;
|
||||
condition_tasks.notify_all();
|
||||
@@ -136,7 +138,7 @@ void server_queue::terminate() {
|
||||
bool server_queue::process_new_tasks(bool is_yielding) {
|
||||
while (true) {
|
||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||
if (!running) {
|
||||
if (!running || stopping) {
|
||||
QUE_DBG("%s", "terminate\n");
|
||||
return true;
|
||||
}
|
||||
@@ -473,11 +475,12 @@ server_task_result_ptr server_response::recv_with_timeout(const std::unordered_s
|
||||
}
|
||||
}
|
||||
|
||||
std::cv_status cr_res = condition_results.wait_until(lock, deadline);
|
||||
if (!running) {
|
||||
RES_DBG("%s : queue result stop\n", __func__);
|
||||
std::terminate(); // we cannot return here since the caller is HTTP code
|
||||
return nullptr; // the caller checks is_running()
|
||||
}
|
||||
|
||||
std::cv_status cr_res = condition_results.wait_until(lock, deadline);
|
||||
if (cr_res == std::cv_status::timeout) {
|
||||
return nullptr;
|
||||
}
|
||||
@@ -518,6 +521,7 @@ void server_response::broadcast(server_task_result_ptr && result) {
|
||||
}
|
||||
|
||||
void server_response::terminate() {
|
||||
std::unique_lock<std::mutex> lock(mutex_results);
|
||||
running = false;
|
||||
condition_results.notify_all();
|
||||
}
|
||||
@@ -565,6 +569,14 @@ server_task_result_ptr server_response_reader::next(const std::function<bool()>
|
||||
while (true) {
|
||||
server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
|
||||
if (result == nullptr) {
|
||||
if (!queue_results.is_running()) {
|
||||
// the server is shutting down: no result will ever come
|
||||
stop();
|
||||
auto res = std::make_unique<server_task_result_error>();
|
||||
res->err_type = ERROR_TYPE_UNAVAILABLE;
|
||||
res->err_msg = "server is shutting down";
|
||||
return res;
|
||||
}
|
||||
// timeout, check stop condition
|
||||
if (should_stop()) {
|
||||
return nullptr;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "server-task.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <deque>
|
||||
#include <exception>
|
||||
@@ -16,6 +17,9 @@ struct server_queue {
|
||||
private:
|
||||
int id = 0;
|
||||
bool running = false;
|
||||
// set by terminate() before it takes mutex_tasks, so a loop blocked under that mutex (e.g. in a
|
||||
// VRAM arbiter token wait) can see the shutdown and bail out
|
||||
std::atomic<bool> stopping{false};
|
||||
bool sleeping = false;
|
||||
bool req_stop_sleeping = false;
|
||||
int64_t time_last_task = 0;
|
||||
@@ -78,6 +82,11 @@ public:
|
||||
// end the start_loop routine
|
||||
void terminate();
|
||||
|
||||
// true once terminate() was called; safe from any thread, including under mutex_tasks
|
||||
bool is_stopping() const {
|
||||
return stopping.load();
|
||||
}
|
||||
|
||||
/**
|
||||
* Main loop consists of these steps:
|
||||
* - Wait until a new task arrives
|
||||
@@ -202,8 +211,13 @@ public:
|
||||
// (used by router mode)
|
||||
void broadcast(server_task_result_ptr && result);
|
||||
|
||||
// terminate the waiting loop
|
||||
// terminate the waiting loop: pending and later readers get an error instead of a result
|
||||
void terminate();
|
||||
|
||||
bool is_running() {
|
||||
std::unique_lock<std::mutex> lock(mutex_results);
|
||||
return running;
|
||||
}
|
||||
};
|
||||
|
||||
// RAII wrapper to make working with server_queue and server_response easier
|
||||
|
||||
Reference in new issue
Block a user