perf(proxy): register liveness and core inference routes first (#40687)

Starlette scans the route table in registration order, so a request pays one
regex match per route registered ahead of its own. The proxy registers several
hundred routes and left the liveness probe near position 280 and the lazy
loaded /v1/messages at the very end. Move /health/liveliness, /health/liveness,
/v1/chat/completions, /chat/completions and /v1/messages to the front of the
route table after startup registration and again after a lazy router loads.

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/proxy/_lazy_features.py

@@ -18,6 +18,7 @@ from starlette.routing import BaseRoute, Match
 from starlette.types import Receive, Scope, Send
 
 from litellm._logging import verbose_proxy_logger
+from litellm.proxy.route_priority import hot_routes_first
 
 if TYPE_CHECKING:
     from fastapi import APIRouter, FastAPI
@@ -343,31 +344,40 @@ class LazyFeatureMiddleware:
         await self.app(scope, receive, send)
 
 
-def _lazy_slots(app: "FastAPI") -> Mapping[str, int]:
+def _lazy_slots(app: "FastAPI") -> Mapping[str, BaseRoute | None]:
     return app.state.lazy_slots if hasattr(app.state, "lazy_slots") else MappingProxyType({})
 
 
 def reserve_lazy_slot(app: "FastAPI", name: str, features: tuple[LazyFeature, ...] = LAZY_FEATURES) -> None:
-    """Record the table position the feature's router used to be included at, so its
-    routes are spliced back in there once it loads and keep the same precedence."""
+    """Record the route the feature's router used to be included after, so its routes
+    are spliced back in there once it loads and keep the same precedence. Anchoring on
+    the route rather than its index survives later reordering of the table."""
     feat: Final = next(f for f in features if f.name == name)
-    app.state.lazy_slots = MappingProxyType({**_lazy_slots(app), feat.module_path: len(app.router.routes)})
+    anchor: Final = app.router.routes[-1] if app.router.routes else None
+    app.state.lazy_slots = MappingProxyType({**_lazy_slots(app), feat.module_path: anchor})
+
+
+def _slot_index(routes: Sequence[BaseRoute], anchor: BaseRoute | None) -> int:
+    if anchor is None:
+        return 0
+    return next((i + 1 for i, route in enumerate(routes) if route is anchor), len(routes))
 
 
 def _eager_route_wins(app: "FastAPI", feat: LazyFeature, scope: Scope) -> bool:
     """Routes ahead of a feature's reserved slot beat its routes in Starlette's scan,
     so a request one of them fully matches never needs the feature loaded."""
-    slot: Final = _lazy_slots(app).get(feat.module_path)
-    if slot is None:
+    slots: Final = _lazy_slots(app)
+    if feat.module_path not in slots:
         return False
-    return any(route.matches(scope)[0] is Match.FULL for route in app.router.routes[:slot])
+    ahead: Final = app.router.routes[: _slot_index(app.router.routes, slots[feat.module_path])]
+    return any(route.matches(scope)[0] is Match.FULL for route in ahead)
 
 
 def _in_registry_order(
     routes: Sequence[BaseRoute],
     lazy_routes: Mapping[str, tuple[BaseRoute, ...]],
     features: tuple[LazyFeature, ...],
-    slots: Mapping[str, int],
+    slots: Mapping[str, BaseRoute | None],
 ) -> tuple[BaseRoute, ...]:
     """Lazy routers land in registry order, not first-request order, so overlapping
     paths (/openai/{endpoint:path} vs /openai/v1/realtime/calls) resolve the same
@@ -380,7 +390,7 @@ def _in_registry_order(
     eager: Final = tuple(route for route in routes if id(route) not in lazy_ids)
 
     def slot_of(module_path: str) -> int:
-        return min(slots.get(module_path, len(eager)), len(eager))
+        return _slot_index(eager, slots[module_path]) if module_path in slots else len(eager)
 
     return tuple(
         route
@@ -416,8 +426,8 @@ async def _force_load(app: "FastAPI", feat: LazyFeature, features: tuple[LazyFea
                 {**previous, feat.module_path: tuple(app.router.routes[before:])}
             )
             app.state.lazy_routes = lazy_routes  # rebind-ok: the app owns the record of which routes each feature added
-            app.router.routes[:] = _in_registry_order(  # rebind-ok: the app owns its route table
-                app.router.routes, lazy_routes, features, _lazy_slots(app)
+            app.router.routes[:] = hot_routes_first(  # rebind-ok: the app owns its route table
+                _in_registry_order(app.router.routes, lazy_routes, features, _lazy_slots(app))
             )
             app.state.lazy_loaded.add(feat.module_path)
             app.openapi_schema = None

litellm/proxy/proxy_server.py

@@ -655,6 +655,7 @@ from litellm.proxy.rag_endpoints.endpoints import router as rag_router
 from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router
 from litellm.proxy.response_api_endpoints.endpoints import router as response_router
 from litellm.proxy.route_llm_request import route_request
+from litellm.proxy.route_priority import hot_routes_first
 from litellm.proxy.search_endpoints.endpoints import router as search_router
 from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
 from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
@@ -18739,6 +18740,7 @@ app.include_router(ui_discovery_endpoints_router)
 app.include_router(google_router)
 
 attach_lazy_features(app)
+app.router.routes = hot_routes_first(app.router.routes)
 app.add_middleware(
     RequestSizeLimitMiddleware,
     get_max_request_size_mb=lambda: general_settings.get("max_request_size_mb"),

litellm/proxy/route_priority.py

@@ -0,0 +1,24 @@
+"""Starlette matches routes in registration order, so the routes that take the most traffic go first."""
+
+from collections.abc import Sequence
+from typing import Final
+
+from starlette.routing import BaseRoute, Route
+
+HOT_ROUTE_PATHS: Final[frozenset[str]] = frozenset(
+    (
+        "/health/liveliness",
+        "/health/liveness",
+        "/v1/chat/completions",
+        "/chat/completions",
+        "/v1/messages",
+    )
+)
+
+
+def _is_hot(route: BaseRoute) -> bool:
+    return isinstance(route, Route) and route.path in HOT_ROUTE_PATHS
+
+
+def hot_routes_first(routes: Sequence[BaseRoute]) -> list[BaseRoute]:  # mutable-ok: assigned to Router.routes, a list
+    return sorted(routes, key=lambda route: not _is_hot(route))

tests/test_litellm/proxy/test_route_priority.py

@@ -0,0 +1,171 @@
+import sys
+from types import ModuleType
+
+import httpx
+import pytest
+from fastapi import APIRouter, FastAPI
+from fastapi.testclient import TestClient
+from starlette.routing import Match
+
+from litellm.proxy.route_priority import HOT_ROUTE_PATHS, hot_routes_first
+
+FILLER_COUNT = 300
+
+
+def _routes_scanned_before_dispatch(app: FastAPI, method: str, path: str) -> int:
+    """Number of route.matches() calls Starlette's Router.app makes before it finds a full match."""
+    scope = {"type": "http", "method": method, "path": path, "root_path": "", "headers": [], "query_string": b""}
+    for i, route in enumerate(app.router.routes):
+        match, _ = route.matches(dict(scope))
+        if match == Match.FULL:
+            return i + 1
+    raise AssertionError(f"{method} {path} has no route")
+
+
+def _hot_router() -> APIRouter:
+    router = APIRouter()
+
+    @router.get("/health/liveliness")
+    @router.get("/health/liveness")
+    async def liveliness():
+        return "I'm alive!"
+
+    @router.post("/v1/chat/completions")
+    @router.post("/chat/completions")
+    async def chat():
+        return {"object": "chat.completion"}
+
+    return router
+
+
+def _app_with_filler_then_hot_routes() -> FastAPI:
+    app = FastAPI()
+    for i in range(FILLER_COUNT):
+
+        @app.get(f"/filler/{i}")
+        async def filler(i: int = i):
+            return {"filler": i}
+
+    app.include_router(_hot_router())
+    return app
+
+
+def test_hot_routes_first_puts_hot_routes_ahead_of_everything_else():
+    app = _app_with_filler_then_hot_routes()
+    assert _routes_scanned_before_dispatch(app, "GET", "/health/liveliness") > FILLER_COUNT
+
+    app.router.routes = hot_routes_first(app.router.routes)
+
+    hot_count = sum(1 for r in app.router.routes if getattr(r, "path", None) in HOT_ROUTE_PATHS)
+    assert _routes_scanned_before_dispatch(app, "GET", "/health/liveliness") <= hot_count
+    assert _routes_scanned_before_dispatch(app, "GET", "/health/liveness") <= hot_count
+    assert _routes_scanned_before_dispatch(app, "POST", "/v1/chat/completions") <= hot_count
+    assert _routes_scanned_before_dispatch(app, "POST", "/chat/completions") <= hot_count
+
+
+def test_hot_routes_first_keeps_the_other_routes_in_order_and_dispatching():
+    app = _app_with_filler_then_hot_routes()
+    before = [r.path for r in app.router.routes if getattr(r, "path", "").startswith("/filler/")]
+
+    app.router.routes = hot_routes_first(app.router.routes)
+
+    after = [r.path for r in app.router.routes if getattr(r, "path", "").startswith("/filler/")]
+    assert after == before
+    client = TestClient(app)
+    assert client.get("/health/liveliness").json() == "I'm alive!"
+    assert client.get("/filler/7").json() == {"filler": 7}
+    assert client.post("/v1/chat/completions").json() == {"object": "chat.completion"}
+    assert client.get("/v1/chat/completions").status_code == 405
+    assert client.get("/does/not/exist").status_code == 404
+
+
+def test_hot_routes_first_is_idempotent():
+    app = _app_with_filler_then_hot_routes()
+    once = hot_routes_first(app.router.routes)
+    assert hot_routes_first(once) == once
+
+
+@pytest.mark.asyncio
+async def test_lazy_loaded_hot_route_moves_to_the_front(monkeypatch):
+    from litellm.proxy._lazy_features import LazyFeature, LazyFeatureMiddleware
+
+    messages_router = APIRouter()
+
+    @messages_router.post("/v1/messages")
+    async def messages():
+        return {"type": "message"}
+
+    fake_module = ModuleType("fake_anthropic_endpoints")
+    fake_module.router = messages_router
+    monkeypatch.setitem(sys.modules, fake_module.__name__, fake_module)
+
+    target_app = _app_with_filler_then_hot_routes()
+    target_app.router.routes = hot_routes_first(target_app.router.routes)
+
+    async def downstream(scope, receive, send):
+        await send({"type": "http.response.start", "status": 200, "headers": []})
+        await send({"type": "http.response.body", "body": b""})
+
+    feat = LazyFeature(name="anthropic", module_path=fake_module.__name__, path_prefixes=("/v1/messages",))
+    mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
+
+    async def receive():
+        return {"type": "http.request", "body": b"", "more_body": False}
+
+    async def send(message):
+        pass
+
+    await mw({"type": "http", "path": "/v1/messages", "method": "POST", "headers": []}, receive, send)
+
+    hot_count = sum(1 for r in target_app.router.routes if getattr(r, "path", None) in HOT_ROUTE_PATHS)
+    assert _routes_scanned_before_dispatch(target_app, "POST", "/v1/messages") <= hot_count
+    assert TestClient(target_app).post("/v1/messages").json() == {"type": "message"}
+
+
+@pytest.mark.asyncio
+async def test_hot_routes_first_keeps_reserved_lazy_slot_ahead_of_later_eager_routes():
+    """Liveness is registered after the provider passthrough slot, so pulling it to the
+    front must not shift where the lazily loaded catch-all is spliced back in."""
+    from litellm.proxy._lazy_features import LazyFeature, LazyFeatureMiddleware, reserve_lazy_slot
+
+    def register(app, module):
+        router = APIRouter()
+        router.add_api_route("/mistral/{endpoint:path}", lambda: {"handler": "passthrough"}, methods=["POST"])
+        app.include_router(router)
+
+    passthrough = LazyFeature(
+        name="llm_passthrough", module_path="json", path_prefixes=("/mistral/",), register_fn=register
+    )
+    target_app = FastAPI()
+    target_app.add_api_route("/mistral/v1/files", lambda: {"handler": "files"}, methods=["POST"])
+    target_app.add_api_route("/mistral/v1/batches", lambda: {"handler": "batches"}, methods=["POST"])
+    reserve_lazy_slot(target_app, "llm_passthrough", features=(passthrough,))
+    target_app.include_router(_hot_router())
+    target_app.add_api_route("/{mcp_server_name}/mcp", lambda: {"handler": "mcp"}, methods=["POST"])
+    target_app.router.routes = hot_routes_first(target_app.router.routes)
+    target_app.add_middleware(LazyFeatureMiddleware, fastapi_app=target_app, features=(passthrough,))
+
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=target_app), base_url="http://t") as client:
+        batches_first = (await client.post("/mistral/v1/batches")).json()["handler"]
+        loaded_after_batches = frozenset(target_app.state.lazy_loaded)
+        handlers = [
+            (await client.post(path)).json()["handler"]
+            for path in ("/mistral/mcp", "/mistral/v1/files", "/mistral/v1/batches")
+        ]
+
+    assert (batches_first, loaded_after_batches) == ("batches", frozenset())
+    assert handlers == ["passthrough", "files", "batches"]
+    hot_count = sum(1 for r in target_app.router.routes if getattr(r, "path", None) in HOT_ROUTE_PATHS)
+    assert _routes_scanned_before_dispatch(target_app, "GET", "/health/liveliness") <= hot_count
+
+
+def test_proxy_app_dispatches_liveness_and_chat_completions_before_the_rest():
+    from litellm.proxy.proxy_server import app
+
+    hot_count = sum(1 for r in app.router.routes if getattr(r, "path", None) in HOT_ROUTE_PATHS)
+    assert hot_count >= 4
+    assert len(app.router.routes) > 100
+    assert _routes_scanned_before_dispatch(app, "GET", "/health/liveliness") <= hot_count
+    assert _routes_scanned_before_dispatch(app, "GET", "/health/liveness") <= hot_count
+    assert _routes_scanned_before_dispatch(app, "POST", "/v1/chat/completions") <= hot_count
+    assert _routes_scanned_before_dispatch(app, "POST", "/chat/completions") <= hot_count