mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(proxy): type the Rust forwarding ASGI boundary precisely
Everything in rust_forward now speaks uvicorn's ASGI3Application, and served() is the one cast from Starlette's ASGIApp, since the two type the same callable with unassignable scope, receive and send types. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
9f6c9d2246
commit
e74261edc1
2 changed files with 21 additions and 10 deletions
|
|
@ -1,16 +1,27 @@
|
|||
from collections.abc import Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
TypeAlias,
|
||||
cast, # noqa: TID251 # the one Starlette-to-uvicorn ASGI conversion in served
|
||||
)
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
|
||||
import starlette.types
|
||||
from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
|
||||
|
||||
PEER_HEADER: Final = b"x-litellm-rust-peer"
|
||||
|
||||
ASGIApp: TypeAlias = Callable[..., Awaitable[None]]
|
||||
ASGIApp: TypeAlias = "ASGI3Application"
|
||||
|
||||
|
||||
def served(app: "starlette.types.ASGIApp") -> ASGIApp:
|
||||
return cast( # cast-ok: Starlette and uvicorn type the same ASGI 3 callable with unassignable scope/receive/send
|
||||
"ASGI3Application", app
|
||||
)
|
||||
|
||||
|
||||
def _restore_peer(scope: "WWWScope") -> None:
|
||||
|
|
@ -47,11 +58,12 @@ class RustPeerMiddleware:
|
|||
await self.app(scope, receive, send)
|
||||
|
||||
|
||||
def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp:
|
||||
def wrap(app: "starlette.types.ASGIApp", mode: RustForwardMode) -> ASGIApp:
|
||||
converted: Final = served(app)
|
||||
match mode:
|
||||
case Supervised():
|
||||
return RustPeerMiddleware(app, mode)
|
||||
return RustPeerMiddleware(converted, mode)
|
||||
case Disabled() | Supervisor():
|
||||
return app
|
||||
return converted
|
||||
case _:
|
||||
assert_never(mode)
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ if TYPE_CHECKING:
|
|||
|
||||
from litellm.proxy.proxy_server import app as proxy_app
|
||||
from litellm.proxy.rust_forward.mode import InheritedFd, Supervised
|
||||
from litellm.proxy.rust_forward.peer import wrap
|
||||
|
||||
from litellm.proxy.rust_forward.peer import RustPeerMiddleware
|
||||
|
||||
Headers = tuple[tuple[bytes, bytes], ...]
|
||||
|
||||
|
|
@ -47,7 +46,7 @@ async def _seen_by_app(
|
|||
seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"])))
|
||||
|
||||
direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe
|
||||
app: Final = wrap(probe, mode) if behind_rust else direct
|
||||
app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct
|
||||
scope: Final[HTTPScope] = {
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0", "spec_version": "2.3"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue