server: keep the server awake while a request uses its state (#32133)
With --sleep-idle-seconds, a handler passes wait_until_no_sleep() while the server is awake, then keeps using the server state, for example to parse the body and tokenize. If the idle timer expires in that window, the main loop enters sleep and the callbacks destroy the model: the handler reads freed vocab memory, and a task posted after that stays in the queue with nobody awake to process it. Count the requests that passed wait_until_no_sleep() and are not done yet, and do not enter sleep while the count is positive. The response generator releases its count in its destructor, so every return path of a handler gives it back. Co-authored-by: Venkata Vineel <vyalamar@users.noreply.github.com>
This commit is contained in:
1 parent
cfb9d4b754
commit
09452cdedd
3 files changed
+21
-4
No files matched your search
@@ -114,9 +114,7 @@ void server_queue::pop_deferred_task(int id_slot) {
|
||||
|
||||
void server_queue::wait_until_no_sleep() {
|
||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||
if (!sleeping) {
|
||||
return;
|
||||
} else {
|
||||
if (sleeping) {
|
||||
if (!req_stop_sleeping) {
|
||||
QUE_DBG("%s", "requesting to stop sleeping\n");
|
||||
req_stop_sleeping = true;
|
||||
@@ -127,6 +125,13 @@ void server_queue::wait_until_no_sleep() {
|
||||
return !sleeping;
|
||||
});
|
||||
}
|
||||
n_inflight++;
|
||||
}
|
||||
|
||||
void server_queue::release_inflight() {
|
||||
std::unique_lock<std::mutex> lock(mutex_tasks);
|
||||
GGML_ASSERT(n_inflight > 0);
|
||||
n_inflight--;
|
||||
}
|
||||
|
||||
void server_queue::terminate() {
|
||||
@@ -289,7 +294,7 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
|
||||
constexpr auto max_wait_time = std::chrono::seconds(1);
|
||||
auto should_sleep = [&]() -> bool {
|
||||
// caller must hold mutex_tasks
|
||||
if (idle_sleep_ms < 0) {
|
||||
if (idle_sleep_ms < 0 || n_inflight > 0) {
|
||||
return false;
|
||||
}
|
||||
int64_t now = ggml_time_ms();
|
||||
|
||||
Reference in new issue
Block a user