diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 5bdece55b..b7dc26c8a 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -772,7 +772,7 @@ static void process_handler_response(server_http_req_ptr && request, server_http sink.done(); SRV_DBG("%s", "http: stream ended\n"); } - return has_next; + return true; }; const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable { response->on_complete(); diff --git a/tools/server/tests/unit/test_completion.py b/tools/server/tests/unit/test_completion.py index 6afcb1773..0c5beb23e 100644 --- a/tools/server/tests/unit/test_completion.py +++ b/tools/server/tests/unit/test_completion.py @@ -1,3 +1,4 @@ +import http.client import pytest import requests import time @@ -93,6 +94,30 @@ def test_completion_stream_vs_non_stream(): assert content_stream == res_non_stream.body["content"] +def test_completion_connection_reuse(): + server.start() + conn = http.client.HTTPConnection(server.server_host, server.server_port, timeout=30) + try: + conn.connect() + conn.auto_open = 0 + original_socket = conn.sock + conn.request("POST", "/completion", json.dumps({ + "prompt": "Hello", "n_predict": 4, "stream": True, + }), {"Content-Type": "application/json"}) + with conn.getresponse() as response: + assert response.status == 200 + assert response.chunked + assert not response.will_close + assert response.read() + conn.request("GET", "/health") + with conn.getresponse() as response: + assert response.status == 200 + assert json.loads(response.read())["status"] == "ok" + assert conn.sock is original_socket + finally: + conn.close() + + def test_completion_with_openai_library(): global server server.start()