refactor(proxy): type the Rust forwarding ASGI boundary precisely

Everything in rust_forward now speaks uvicorn's ASGI3Application, and served()
is the one cast from Starlette's ASGIApp, since the two type the same callable
with unassignable scope, receive and send types.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-11-06 05:44:44 +09:00 • committed by yujonglee
parent 9f6c9d2246
commit e74261edc1
2 changed files with 21 additions and 10 deletions

View file

@ -1,16 +1,27 @@
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final, TypeAlias
from typing import (
TYPE_CHECKING,
Final,
TypeAlias,
cast, # noqa: TID251 # the one Starlette-to-uvicorn ASGI conversion in served
)
from typing_extensions import assert_never
from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor
if TYPE_CHECKING:
from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
import starlette.types
from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
PEER_HEADER: Final = b"x-litellm-rust-peer"
ASGIApp: TypeAlias = Callable[..., Awaitable[None]]
ASGIApp: TypeAlias = "ASGI3Application"
def served(app: "starlette.types.ASGIApp") -> ASGIApp:
return cast( # cast-ok: Starlette and uvicorn type the same ASGI 3 callable with unassignable scope/receive/send
"ASGI3Application", app
)
def _restore_peer(scope: "WWWScope") -> None:
@ -47,11 +58,12 @@ class RustPeerMiddleware:
await self.app(scope, receive, send)
def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp:
def wrap(app: "starlette.types.ASGIApp", mode: RustForwardMode) -> ASGIApp:
converted: Final = served(app)
match mode:
case Supervised():
return RustPeerMiddleware(app, mode)
return RustPeerMiddleware(converted, mode)
case Disabled() | Supervisor():
return app
return converted
case _:
assert_never(mode)

View file

@ -21,8 +21,7 @@ if TYPE_CHECKING:
from litellm.proxy.proxy_server import app as proxy_app
from litellm.proxy.rust_forward.mode import InheritedFd, Supervised
from litellm.proxy.rust_forward.peer import wrap
from litellm.proxy.rust_forward.peer import RustPeerMiddleware
Headers = tuple[tuple[bytes, bytes], ...]
@ -47,7 +46,7 @@ async def _seen_by_app(
seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"])))
direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe
app: Final = wrap(probe, mode) if behind_rust else direct
app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct
scope: Final[HTTPScope] = {
"type": "http",
"asgi": {"version": "3.0", "spec_version": "2.3"},