diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 5d9726796..effa5bbda 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5302,12 +5302,19 @@ server_context_meta server_context::get_meta() const { // may have bypass_sleep = true if the task does not use ctx_server struct server_res_generator : server_res_spipe { server_response_reader rd; + server_queue * queue_awake = nullptr; // set while this response keeps the server awake server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false) : rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) { // fast path in case sleeping is disabled bypass_sleep |= sleep_idle_seconds < 0; if (!bypass_sleep) { queue_tasks.wait_until_no_sleep(); + queue_awake = &queue_tasks; + } + } + ~server_res_generator() { + if (queue_awake) { + queue_awake->release_inflight(); } } void ok(const json & response_data) { diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index c4ce90721..75daebb0d 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -114,9 +114,7 @@ void server_queue::pop_deferred_task(int id_slot) { void server_queue::wait_until_no_sleep() { std::unique_lock lock(mutex_tasks); - if (!sleeping) { - return; - } else { + if (sleeping) { if (!req_stop_sleeping) { QUE_DBG("%s", "requesting to stop sleeping\n"); req_stop_sleeping = true; @@ -127,6 +125,13 @@ void server_queue::wait_until_no_sleep() { return !sleeping; }); } + n_inflight++; +} + +void server_queue::release_inflight() { + std::unique_lock lock(mutex_tasks); + GGML_ASSERT(n_inflight > 0); + n_inflight--; } void server_queue::terminate() { @@ -289,7 +294,7 @@ void server_queue::start_loop(int64_t idle_sleep_ms) { constexpr auto max_wait_time = std::chrono::seconds(1); auto should_sleep = [&]() -> bool { // caller must hold mutex_tasks - if (idle_sleep_ms < 0) { + if (idle_sleep_ms < 0 || n_inflight > 0) { return false; } int64_t now = ggml_time_ms(); diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h index 692255e4f..f9d66be64 100644 --- a/tools/server/server-queue.h +++ b/tools/server/server-queue.h @@ -19,6 +19,7 @@ private: bool sleeping = false; bool req_stop_sleeping = false; int64_t time_last_task = 0; + int n_inflight = 0; // requests that keep the server awake, see wait_until_no_sleep() // queues std::deque queue_tasks; @@ -67,8 +68,12 @@ public: // if sleeping, request exiting sleep state and wait until it is done // returns immediately if not sleeping + // the server then stays awake until the caller calls release_inflight() void wait_until_no_sleep(); + // let the server sleep again once the caller no longer uses the server state + void release_inflight(); + bool is_sleeping() { std::unique_lock lock(mutex_tasks); return sleeping;