mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(proxy): run the forwarding supervisor's children on anyio
runtime.py hand-wired an asyncio.Queue, a map of partials and per-child background tasks. The driver loop now takes an Executor protocol, and open_children implements it with an AsyncExitStack, an anyio task group and a memory object stream, so the event plumbing lives in one place and the loop has fast unit tests with a scripted executor Stopping a child is shielded from cancellation, since anyio cancels every await in a cancelled scope and an unshielded wait would leave a child still stopping. A child that cannot be spawned comes back as SpawnFailed instead of an OSError, which an anyio task group would wrap in an ExceptionGroup
This commit is contained in:
parent
5f97ff0e60
commit
6ae722d049
3 changed files with 240 additions and 78 deletions
|
|
@ -4,11 +4,13 @@ import sysconfig
|
|||
from collections.abc import Callable, Sequence
|
||||
from typing import Final
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.proxy.rust_forward.platform import Platform, current_platform
|
||||
from litellm.proxy.rust_forward.server import PythonServer
|
||||
from litellm.proxy.rust_forward.supervisor.lifecycle import FAILURE
|
||||
from litellm.proxy.rust_forward.supervisor.preflight import StartupError, gateway_binary, startup_check
|
||||
from litellm.proxy.rust_forward.supervisor.runtime import exit_like, run
|
||||
from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, exit_like, run
|
||||
|
||||
|
||||
def _nothing_to_prepare() -> None:
|
||||
|
|
@ -34,8 +36,18 @@ def supervise(
|
|||
return checked
|
||||
before_start()
|
||||
try:
|
||||
status: Final = asyncio.run(run(chosen, checked, command, server))
|
||||
outcome: Final = asyncio.run(run(chosen, checked, command, server))
|
||||
except OSError as error:
|
||||
sys.stderr.write(f"Rust gateway startup failed: {error}\n")
|
||||
return FAILURE
|
||||
return exit_like(status)
|
||||
return _startup_failed(error)
|
||||
match outcome:
|
||||
case SpawnFailed(error):
|
||||
return _startup_failed(error)
|
||||
case int():
|
||||
return exit_like(outcome)
|
||||
case _:
|
||||
assert_never(outcome)
|
||||
|
||||
|
||||
def _startup_failed(error: OSError) -> int:
|
||||
sys.stderr.write(f"Rust gateway startup failed: {error}\n")
|
||||
return FAILURE
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import math
|
||||
import os
|
||||
import signal
|
||||
from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping, Sequence
|
||||
from functools import partial
|
||||
from collections.abc import AsyncGenerator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
from typing import Final, Protocol
|
||||
|
||||
import anyio
|
||||
from anyio.abc import TaskGroup
|
||||
from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.proxy.rust_forward.mode import RUST_HOP
|
||||
|
|
@ -28,23 +32,49 @@ from litellm.proxy.rust_forward.supervisor.lifecycle import (
|
|||
|
||||
PROBE_INTERVAL_SECONDS: Final = 0.05
|
||||
|
||||
Launch: TypeAlias = Callable[[asyncio.Queue[Event]], contextlib.AbstractAsyncContextManager[asyncio.subprocess.Process]]
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SpawnFailed:
|
||||
error: OSError
|
||||
|
||||
|
||||
async def run(platform: Platform, binary: str, command: Sequence[str], server: PythonServer) -> int:
|
||||
"""Run Python and Rust under ``lifecycle.step`` until it decides the exit status."""
|
||||
events: Final[asyncio.Queue[Event]] = asyncio.Queue()
|
||||
platform.catch(asyncio.get_running_loop(), lambda signum: events.put_nowait(Caught(signum)))
|
||||
class Executor(Protocol):
|
||||
async def spawn(self, child: Child) -> SpawnFailed | None: ...
|
||||
|
||||
def send(self, child: Child, signum: signal.Signals) -> None: ...
|
||||
|
||||
async def next_event(self) -> Event: ...
|
||||
|
||||
|
||||
async def run(platform: Platform, binary: str, command: Sequence[str], server: PythonServer) -> int | SpawnFailed:
|
||||
with platform.private_listener() as private:
|
||||
launches: Final[Mapping[Child, Launch]] = MappingProxyType(
|
||||
specs: Final = MappingProxyType(
|
||||
{
|
||||
Child.PYTHON: partial(
|
||||
_launch_python, platform, python_spec(platform, command, private, server), private.port
|
||||
),
|
||||
Child.RUST: partial(_launch, platform, Child.RUST, rust_spec(platform, binary, private.port, server)),
|
||||
Child.PYTHON: python_spec(platform, command, private, server),
|
||||
Child.RUST: rust_spec(platform, binary, private.port, server),
|
||||
}
|
||||
)
|
||||
return await _drive(platform, events, launches)
|
||||
async with open_children(platform, specs, private.port) as children:
|
||||
platform.catch(asyncio.get_running_loop(), children.caught)
|
||||
return await drive(children, platform.stop_signals)
|
||||
|
||||
|
||||
async def drive(executor: Executor, stop_signals: frozenset[signal.Signals]) -> int | SpawnFailed:
|
||||
"""Carry out what ``lifecycle.step`` decides until it settles the exit status."""
|
||||
state, actions = boot()
|
||||
while True:
|
||||
for action in actions:
|
||||
match action:
|
||||
case Spawn(child):
|
||||
if (failed := await executor.spawn(child)) is not None:
|
||||
return failed
|
||||
case Send(child, signum):
|
||||
executor.send(child, signum)
|
||||
case _:
|
||||
assert_never(action)
|
||||
if (status := exit_status(state)) is not None:
|
||||
return status
|
||||
state, actions = step(state, await executor.next_event(), stop_signals)
|
||||
|
||||
|
||||
def exit_like(status: int) -> int:
|
||||
|
|
@ -57,33 +87,59 @@ def exit_like(status: int) -> int:
|
|||
return 128 + signum
|
||||
|
||||
|
||||
async def _drive(platform: Platform, events: asyncio.Queue[Event], launches: Mapping[Child, Launch]) -> int:
|
||||
async with contextlib.AsyncExitStack() as stack:
|
||||
children: Final[dict[Child, asyncio.subprocess.Process]] = {} # mutable-ok: indexes the stack's children
|
||||
state, actions = boot()
|
||||
while True:
|
||||
for action in actions:
|
||||
match action:
|
||||
case Spawn(child):
|
||||
children[child] = await stack.enter_async_context(launches[child](events))
|
||||
case Send(child, signum):
|
||||
platform.signal_child(children[child], signum)
|
||||
case _:
|
||||
assert_never(action)
|
||||
if (status := exit_status(state)) is not None:
|
||||
return status
|
||||
state, actions = step(state, await events.get(), platform.stop_signals)
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Children:
|
||||
platform: Platform
|
||||
specs: Mapping[Child, ChildSpec]
|
||||
private_port: int
|
||||
stack: contextlib.AsyncExitStack
|
||||
tasks: TaskGroup
|
||||
events_in: MemoryObjectSendStream[Event]
|
||||
events_out: MemoryObjectReceiveStream[Event]
|
||||
processes: dict[Child, asyncio.subprocess.Process] # mutable-ok: the running children the stack owns
|
||||
|
||||
def caught(self, signum: signal.Signals) -> None:
|
||||
with contextlib.suppress(anyio.ClosedResourceError):
|
||||
self.events_in.send_nowait(Caught(signum))
|
||||
|
||||
async def spawn(self, child: Child) -> SpawnFailed | None:
|
||||
try:
|
||||
process: Final = await self.stack.enter_async_context(_process(self.platform, self.specs[child]))
|
||||
except OSError as error:
|
||||
return SpawnFailed(error)
|
||||
self.processes[child] = process
|
||||
self.tasks.start_soon(_report_exit, child, process, self.events_in.clone())
|
||||
if child is Child.PYTHON:
|
||||
probe: Final = self.tasks.start_soon(_report_once_listening, self.private_port, self.events_in.clone())
|
||||
self.stack.callback(probe.cancel)
|
||||
return None
|
||||
|
||||
def send(self, child: Child, signum: signal.Signals) -> None:
|
||||
self.platform.signal_child(self.processes[child], signum)
|
||||
|
||||
async def next_event(self) -> Event:
|
||||
return await self.events_out.receive()
|
||||
|
||||
|
||||
# The task group is entered before any child, so on the way out every child is stopped, last spawned first, before
|
||||
# the task group waits for the tasks reporting their exits. Signal handlers cannot wait for room, hence the
|
||||
# unbounded event buffer.
|
||||
@contextlib.asynccontextmanager
|
||||
async def open_children(
|
||||
platform: Platform, specs: Mapping[Child, ChildSpec], private_port: int
|
||||
) -> AsyncGenerator[Children]:
|
||||
events_in, events_out = anyio.create_memory_object_stream[Event](math.inf)
|
||||
with events_in, events_out:
|
||||
async with contextlib.AsyncExitStack() as stack:
|
||||
tasks: Final = await stack.enter_async_context(anyio.create_task_group())
|
||||
yield Children(platform, specs, private_port, stack, tasks, events_in, events_out, {})
|
||||
|
||||
|
||||
# A child's stdin is a pipe only the supervisor holds open, so the child sees EOF as soon as the supervisor is gone,
|
||||
# however it went, and shuts down instead of outliving it.
|
||||
# however it went, and shuts down instead of outliving it. Stopping is shielded so a cancelled supervisor still waits
|
||||
# for the child it is stopping.
|
||||
@contextlib.asynccontextmanager
|
||||
async def _launch(
|
||||
platform: Platform,
|
||||
child: Child,
|
||||
spec: ChildSpec,
|
||||
events: asyncio.Queue[Event],
|
||||
) -> AsyncGenerator[asyncio.subprocess.Process]:
|
||||
async def _process(platform: Platform, spec: ChildSpec) -> AsyncGenerator[asyncio.subprocess.Process]:
|
||||
process: Final = await asyncio.create_subprocess_exec(
|
||||
*spec.argv,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
|
|
@ -92,51 +148,36 @@ async def _launch(
|
|||
start_new_session=spec.spawn.start_new_session,
|
||||
creationflags=spec.spawn.creationflags,
|
||||
)
|
||||
waiter: Final = asyncio.create_task(_report_exit(child, process, events))
|
||||
try:
|
||||
yield process
|
||||
finally:
|
||||
platform.stop_group(process)
|
||||
await waiter
|
||||
with anyio.CancelScope(shield=True):
|
||||
platform.stop_group(process)
|
||||
await process.wait()
|
||||
if process.stdin is not None:
|
||||
process.stdin.close()
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _launch_python(
|
||||
platform: Platform, spec: ChildSpec, port: int, events: asyncio.Queue[Event]
|
||||
) -> AsyncGenerator[asyncio.subprocess.Process]:
|
||||
async with (
|
||||
_launch(platform, Child.PYTHON, spec, events) as process,
|
||||
_in_background(_report_once_listening(port, events)),
|
||||
):
|
||||
yield process
|
||||
async def _report_exit(
|
||||
child: Child, process: asyncio.subprocess.Process, events: MemoryObjectSendStream[Event]
|
||||
) -> None:
|
||||
with events:
|
||||
events.send_nowait(ChildExited(child, await process.wait()))
|
||||
|
||||
|
||||
async def _report_exit(child: Child, process: asyncio.subprocess.Process, events: asyncio.Queue[Event]) -> None:
|
||||
events.put_nowait(ChildExited(child, await process.wait()))
|
||||
|
||||
|
||||
async def _report_once_listening(port: int, events: asyncio.Queue[Event]) -> None:
|
||||
while not await _accepts(port):
|
||||
await asyncio.sleep(PROBE_INTERVAL_SECONDS)
|
||||
events.put_nowait(Probe.PYTHON_LISTENING)
|
||||
async def _report_once_listening(port: int, events: MemoryObjectSendStream[Event]) -> None:
|
||||
with events:
|
||||
while not await _accepts(port):
|
||||
await anyio.sleep(PROBE_INTERVAL_SECONDS)
|
||||
events.send_nowait(Probe.PYTHON_LISTENING)
|
||||
|
||||
|
||||
async def _accepts(port: int) -> bool:
|
||||
try:
|
||||
_, writer = await asyncio.wait_for(asyncio.open_connection(RUST_HOP, port), PROBE_INTERVAL_SECONDS)
|
||||
except (OSError, asyncio.TimeoutError):
|
||||
return False
|
||||
writer.close()
|
||||
return True
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _in_background(work: Coroutine[None, None, None]) -> AsyncGenerator[None]:
|
||||
task: Final = asyncio.create_task(work)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
with anyio.move_on_after(PROBE_INTERVAL_SECONDS):
|
||||
try:
|
||||
stream: Final = await anyio.connect_tcp(RUST_HOP, port)
|
||||
except OSError:
|
||||
return False
|
||||
await stream.aclose()
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -1,12 +1,29 @@
|
|||
import signal
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import anyio
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.rust_forward.platform import Platform, current_platform
|
||||
from litellm.proxy.rust_forward.platform.windows import WindowsPlatform
|
||||
from litellm.proxy.rust_forward.supervisor import supervise
|
||||
from litellm.proxy.rust_forward.supervisor.children import ChildSpec
|
||||
from litellm.proxy.rust_forward.supervisor.lifecycle import (
|
||||
FAILURE,
|
||||
Action,
|
||||
Caught,
|
||||
Child,
|
||||
ChildExited,
|
||||
Event,
|
||||
Probe,
|
||||
Send,
|
||||
Spawn,
|
||||
)
|
||||
from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, drive, open_children
|
||||
from tests.unit.proxy.rust_forward.supervisor._support import python_server
|
||||
|
||||
|
||||
|
|
@ -43,3 +60,95 @@ def test_rust_opens_the_public_port_only_once_python_listens(tmp_path: Path, mon
|
|||
command: Final = (sys.executable, "-I", "-c", PYTHON_CHILD)
|
||||
supervise(python_server(), command, scripts_dir=str(tmp_path), platform=_fake_gateway(tmp_path))
|
||||
assert (tmp_path / "rust_saw_python_listening").read_text() == "True"
|
||||
|
||||
|
||||
TERM: Final = signal.SIGTERM
|
||||
POSIX_ONLY: Final = pytest.mark.skipif(sys.platform == "win32", reason="the child traps SIGTERM")
|
||||
STOPS: Final = frozenset({signal.SIGTERM, signal.SIGINT, signal.SIGQUIT})
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScriptedExecutor:
|
||||
"""Replays a fixed event script and records every action the driver carries out."""
|
||||
|
||||
events: list[Event]
|
||||
failing: frozenset[Child] = frozenset()
|
||||
done: list[Action] = field(default_factory=list)
|
||||
|
||||
async def spawn(self, child: Child) -> SpawnFailed | None:
|
||||
self.done.append(Spawn(child))
|
||||
return SpawnFailed(FileNotFoundError(child.value)) if child in self.failing else None
|
||||
|
||||
def send(self, child: Child, signum: signal.Signals) -> None:
|
||||
self.done.append(Send(child, signum))
|
||||
|
||||
async def next_event(self) -> Event:
|
||||
return self.events.pop(0)
|
||||
|
||||
|
||||
async def test_driver_carries_out_each_action_in_order_and_ends_with_pythons_status() -> None:
|
||||
executor: Final = ScriptedExecutor(
|
||||
[Probe.PYTHON_LISTENING, Caught(TERM), ChildExited(Child.RUST, 0), ChildExited(Child.PYTHON, -15)]
|
||||
)
|
||||
assert await drive(executor, STOPS) == -15
|
||||
assert executor.done == [Spawn(Child.PYTHON), Spawn(Child.RUST), Send(Child.RUST, TERM), Send(Child.PYTHON, TERM)]
|
||||
assert executor.events == []
|
||||
|
||||
|
||||
async def test_driver_stops_python_and_fails_when_rust_is_lost() -> None:
|
||||
executor: Final = ScriptedExecutor(
|
||||
[Probe.PYTHON_LISTENING, ChildExited(Child.RUST, 0), ChildExited(Child.PYTHON, -15)]
|
||||
)
|
||||
assert await drive(executor, STOPS) == FAILURE
|
||||
assert executor.done == [Spawn(Child.PYTHON), Spawn(Child.RUST), Send(Child.PYTHON, TERM)]
|
||||
|
||||
|
||||
async def test_driver_asks_for_no_event_once_the_status_is_settled() -> None:
|
||||
executor: Final = ScriptedExecutor([ChildExited(Child.PYTHON, 3), Caught(TERM)])
|
||||
assert await drive(executor, STOPS) == 3
|
||||
assert executor.events == [Caught(TERM)]
|
||||
|
||||
|
||||
async def test_driver_returns_a_failed_spawn_without_waiting_for_events() -> None:
|
||||
executor: Final = ScriptedExecutor([Probe.PYTHON_LISTENING, Caught(TERM)], failing=frozenset({Child.RUST}))
|
||||
outcome: Final = await drive(executor, STOPS)
|
||||
assert isinstance(outcome, SpawnFailed)
|
||||
assert executor.events == [Caught(TERM)]
|
||||
|
||||
|
||||
def _children(argv: tuple[str, ...]) -> MappingProxyType[Child, ChildSpec]:
|
||||
spec: Final = ChildSpec(argv, MappingProxyType({}), current_platform().spawn_options(()))
|
||||
return MappingProxyType({Child.PYTHON: spec, Child.RUST: spec})
|
||||
|
||||
|
||||
SLOW_TO_STOP: Final = (
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import signal, time; signal.signal(signal.SIGTERM, lambda *_: (time.sleep(0.5), exit(0))); time.sleep(30)",
|
||||
)
|
||||
|
||||
|
||||
@POSIX_ONLY
|
||||
async def test_a_cancelled_supervisor_still_waits_for_the_child_it_stops() -> None:
|
||||
with anyio.CancelScope() as scope:
|
||||
async with open_children(current_platform(), _children(SLOW_TO_STOP), 1) as children:
|
||||
assert await children.spawn(Child.RUST) is None
|
||||
process: Final = children.processes[Child.RUST]
|
||||
await anyio.sleep(0.3)
|
||||
scope.cancel()
|
||||
await anyio.sleep(1)
|
||||
assert process.returncode == 0
|
||||
|
||||
|
||||
async def test_exiting_children_are_reported_as_events() -> None:
|
||||
async with open_children(current_platform(), _children((sys.executable, "-c", "exit(7)")), 1) as children:
|
||||
assert await children.spawn(Child.RUST) is None
|
||||
with anyio.fail_after(5):
|
||||
assert await children.next_event() == ChildExited(Child.RUST, 7)
|
||||
|
||||
|
||||
async def test_a_child_that_cannot_start_is_returned_as_a_failure(tmp_path: Path) -> None:
|
||||
async with open_children(current_platform(), _children((str(tmp_path / "missing"),)), 1) as children:
|
||||
outcome: Final = await children.spawn(Child.RUST)
|
||||
assert isinstance(outcome, SpawnFailed)
|
||||
assert isinstance(outcome.error, FileNotFoundError)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue