From e74261edc17f5f5ecc4c274dbd7320edcd4a5044 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 6 Nov 2026 05:44:44 +0900 Subject: [PATCH] refactor(proxy): type the Rust forwarding ASGI boundary precisely Everything in rust_forward now speaks uvicorn's ASGI3Application, and served() is the one cast from Starlette's ASGIApp, since the two type the same callable with unassignable scope, receive and send types. Co-Authored-By: Claude Opus 5.5 --- litellm/proxy/rust_forward/peer.py | 26 ++++++++++++++++------ tests/unit/proxy/rust_forward/test_peer.py | 5 ++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/rust_forward/peer.py b/litellm/proxy/rust_forward/peer.py index 2d03d6ca35c..cf891495c2b 100644 --- a/litellm/proxy/rust_forward/peer.py +++ b/litellm/proxy/rust_forward/peer.py @@ -1,16 +1,27 @@ -from collections.abc import Awaitable, Callable -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import ( + TYPE_CHECKING, + Final, + TypeAlias, + cast, # noqa: TID251 # the one Starlette-to-uvicorn ASGI conversion in served +) from typing_extensions import assert_never from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor if TYPE_CHECKING: - from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + import starlette.types + from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope PEER_HEADER: Final = b"x-litellm-rust-peer" -ASGIApp: TypeAlias = Callable[..., Awaitable[None]] +ASGIApp: TypeAlias = "ASGI3Application" + + +def served(app: "starlette.types.ASGIApp") -> ASGIApp: + return cast( # cast-ok: Starlette and uvicorn type the same ASGI 3 callable with unassignable scope/receive/send + "ASGI3Application", app + ) def _restore_peer(scope: "WWWScope") -> None: @@ -47,11 +58,12 @@ class RustPeerMiddleware: await self.app(scope, receive, send) -def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp: +def wrap(app: "starlette.types.ASGIApp", mode: RustForwardMode) -> ASGIApp: + converted: Final = served(app) match mode: case Supervised(): - return RustPeerMiddleware(app, mode) + return RustPeerMiddleware(converted, mode) case Disabled() | Supervisor(): - return app + return converted case _: assert_never(mode) diff --git a/tests/unit/proxy/rust_forward/test_peer.py b/tests/unit/proxy/rust_forward/test_peer.py index ee625c4a4be..b0c59641e7f 100644 --- a/tests/unit/proxy/rust_forward/test_peer.py +++ b/tests/unit/proxy/rust_forward/test_peer.py @@ -21,8 +21,7 @@ if TYPE_CHECKING: from litellm.proxy.proxy_server import app as proxy_app from litellm.proxy.rust_forward.mode import InheritedFd, Supervised -from litellm.proxy.rust_forward.peer import wrap - +from litellm.proxy.rust_forward.peer import RustPeerMiddleware Headers = tuple[tuple[bytes, bytes], ...] @@ -47,7 +46,7 @@ async def _seen_by_app( seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe - app: Final = wrap(probe, mode) if behind_rust else direct + app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct scope: Final[HTTPScope] = { "type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"},