server: (router) add LRU scheduler (#26572)

* add lru_sched * handle coalescing (req leaves waiting queue) * add tests * fix stream case * address review comments
2026-08-07 14:46:53 +02:00
parent e1470ee6a2
commit dff15d4ac9
4 changed files with 487 additions and 42 deletions
@@ -132,7 +132,10 @@ class ServerProcess:
        self.external_server = "DEBUG_EXTERNAL" in os.environ

    def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None:
-        env = {**os.environ}
+        env = {
+            **os.environ,
+            "LLAMA_SERVER_DEBUG_FAKE_TIMING": "1",
+        }
        if "LLAMA_CACHE" not in os.environ:
            env["LLAMA_CACHE"] = "tmp"
        if self.external_server: