diff --git a/litellm/proxy/rust_forward/supervisor/__init__.py b/litellm/proxy/rust_forward/supervisor/__init__.py index 2819f837da3..bfb09ddabe2 100644 --- a/litellm/proxy/rust_forward/supervisor/__init__.py +++ b/litellm/proxy/rust_forward/supervisor/__init__.py @@ -4,11 +4,13 @@ import sysconfig from collections.abc import Callable, Sequence from typing import Final +from typing_extensions import assert_never + from litellm.proxy.rust_forward.platform import Platform, current_platform from litellm.proxy.rust_forward.server import PythonServer from litellm.proxy.rust_forward.supervisor.lifecycle import FAILURE from litellm.proxy.rust_forward.supervisor.preflight import StartupError, gateway_binary, startup_check -from litellm.proxy.rust_forward.supervisor.runtime import exit_like, run +from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, exit_like, run def _nothing_to_prepare() -> None: @@ -34,8 +36,18 @@ def supervise( return checked before_start() try: - status: Final = asyncio.run(run(chosen, checked, command, server)) + outcome: Final = asyncio.run(run(chosen, checked, command, server)) except OSError as error: - sys.stderr.write(f"Rust gateway startup failed: {error}\n") - return FAILURE - return exit_like(status) + return _startup_failed(error) + match outcome: + case SpawnFailed(error): + return _startup_failed(error) + case int(): + return exit_like(outcome) + case _: + assert_never(outcome) + + +def _startup_failed(error: OSError) -> int: + sys.stderr.write(f"Rust gateway startup failed: {error}\n") + return FAILURE diff --git a/litellm/proxy/rust_forward/supervisor/runtime.py b/litellm/proxy/rust_forward/supervisor/runtime.py index 1e8aced9077..8fbb2af4133 100644 --- a/litellm/proxy/rust_forward/supervisor/runtime.py +++ b/litellm/proxy/rust_forward/supervisor/runtime.py @@ -1,12 +1,16 @@ import asyncio import contextlib +import math import os import signal -from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping, Sequence -from functools import partial +from collections.abc import AsyncGenerator, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType -from typing import Final, TypeAlias +from typing import Final, Protocol +import anyio +from anyio.abc import TaskGroup +from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream from typing_extensions import assert_never from litellm.proxy.rust_forward.mode import RUST_HOP @@ -28,23 +32,49 @@ from litellm.proxy.rust_forward.supervisor.lifecycle import ( PROBE_INTERVAL_SECONDS: Final = 0.05 -Launch: TypeAlias = Callable[[asyncio.Queue[Event]], contextlib.AbstractAsyncContextManager[asyncio.subprocess.Process]] + +@dataclass(frozen=True, slots=True) +class SpawnFailed: + error: OSError -async def run(platform: Platform, binary: str, command: Sequence[str], server: PythonServer) -> int: - """Run Python and Rust under ``lifecycle.step`` until it decides the exit status.""" - events: Final[asyncio.Queue[Event]] = asyncio.Queue() - platform.catch(asyncio.get_running_loop(), lambda signum: events.put_nowait(Caught(signum))) +class Executor(Protocol): + async def spawn(self, child: Child) -> SpawnFailed | None: ... + + def send(self, child: Child, signum: signal.Signals) -> None: ... + + async def next_event(self) -> Event: ... + + +async def run(platform: Platform, binary: str, command: Sequence[str], server: PythonServer) -> int | SpawnFailed: with platform.private_listener() as private: - launches: Final[Mapping[Child, Launch]] = MappingProxyType( + specs: Final = MappingProxyType( { - Child.PYTHON: partial( - _launch_python, platform, python_spec(platform, command, private, server), private.port - ), - Child.RUST: partial(_launch, platform, Child.RUST, rust_spec(platform, binary, private.port, server)), + Child.PYTHON: python_spec(platform, command, private, server), + Child.RUST: rust_spec(platform, binary, private.port, server), } ) - return await _drive(platform, events, launches) + async with open_children(platform, specs, private.port) as children: + platform.catch(asyncio.get_running_loop(), children.caught) + return await drive(children, platform.stop_signals) + + +async def drive(executor: Executor, stop_signals: frozenset[signal.Signals]) -> int | SpawnFailed: + """Carry out what ``lifecycle.step`` decides until it settles the exit status.""" + state, actions = boot() + while True: + for action in actions: + match action: + case Spawn(child): + if (failed := await executor.spawn(child)) is not None: + return failed + case Send(child, signum): + executor.send(child, signum) + case _: + assert_never(action) + if (status := exit_status(state)) is not None: + return status + state, actions = step(state, await executor.next_event(), stop_signals) def exit_like(status: int) -> int: @@ -57,33 +87,59 @@ def exit_like(status: int) -> int: return 128 + signum -async def _drive(platform: Platform, events: asyncio.Queue[Event], launches: Mapping[Child, Launch]) -> int: - async with contextlib.AsyncExitStack() as stack: - children: Final[dict[Child, asyncio.subprocess.Process]] = {} # mutable-ok: indexes the stack's children - state, actions = boot() - while True: - for action in actions: - match action: - case Spawn(child): - children[child] = await stack.enter_async_context(launches[child](events)) - case Send(child, signum): - platform.signal_child(children[child], signum) - case _: - assert_never(action) - if (status := exit_status(state)) is not None: - return status - state, actions = step(state, await events.get(), platform.stop_signals) +@dataclass(frozen=True, slots=True) +class Children: + platform: Platform + specs: Mapping[Child, ChildSpec] + private_port: int + stack: contextlib.AsyncExitStack + tasks: TaskGroup + events_in: MemoryObjectSendStream[Event] + events_out: MemoryObjectReceiveStream[Event] + processes: dict[Child, asyncio.subprocess.Process] # mutable-ok: the running children the stack owns + + def caught(self, signum: signal.Signals) -> None: + with contextlib.suppress(anyio.ClosedResourceError): + self.events_in.send_nowait(Caught(signum)) + + async def spawn(self, child: Child) -> SpawnFailed | None: + try: + process: Final = await self.stack.enter_async_context(_process(self.platform, self.specs[child])) + except OSError as error: + return SpawnFailed(error) + self.processes[child] = process + self.tasks.start_soon(_report_exit, child, process, self.events_in.clone()) + if child is Child.PYTHON: + probe: Final = self.tasks.start_soon(_report_once_listening, self.private_port, self.events_in.clone()) + self.stack.callback(probe.cancel) + return None + + def send(self, child: Child, signum: signal.Signals) -> None: + self.platform.signal_child(self.processes[child], signum) + + async def next_event(self) -> Event: + return await self.events_out.receive() + + +# The task group is entered before any child, so on the way out every child is stopped, last spawned first, before +# the task group waits for the tasks reporting their exits. Signal handlers cannot wait for room, hence the +# unbounded event buffer. +@contextlib.asynccontextmanager +async def open_children( + platform: Platform, specs: Mapping[Child, ChildSpec], private_port: int +) -> AsyncGenerator[Children]: + events_in, events_out = anyio.create_memory_object_stream[Event](math.inf) + with events_in, events_out: + async with contextlib.AsyncExitStack() as stack: + tasks: Final = await stack.enter_async_context(anyio.create_task_group()) + yield Children(platform, specs, private_port, stack, tasks, events_in, events_out, {}) # A child's stdin is a pipe only the supervisor holds open, so the child sees EOF as soon as the supervisor is gone, -# however it went, and shuts down instead of outliving it. +# however it went, and shuts down instead of outliving it. Stopping is shielded so a cancelled supervisor still waits +# for the child it is stopping. @contextlib.asynccontextmanager -async def _launch( - platform: Platform, - child: Child, - spec: ChildSpec, - events: asyncio.Queue[Event], -) -> AsyncGenerator[asyncio.subprocess.Process]: +async def _process(platform: Platform, spec: ChildSpec) -> AsyncGenerator[asyncio.subprocess.Process]: process: Final = await asyncio.create_subprocess_exec( *spec.argv, stdin=asyncio.subprocess.PIPE, @@ -92,51 +148,36 @@ async def _launch( start_new_session=spec.spawn.start_new_session, creationflags=spec.spawn.creationflags, ) - waiter: Final = asyncio.create_task(_report_exit(child, process, events)) try: yield process finally: - platform.stop_group(process) - await waiter + with anyio.CancelScope(shield=True): + platform.stop_group(process) + await process.wait() if process.stdin is not None: process.stdin.close() -@contextlib.asynccontextmanager -async def _launch_python( - platform: Platform, spec: ChildSpec, port: int, events: asyncio.Queue[Event] -) -> AsyncGenerator[asyncio.subprocess.Process]: - async with ( - _launch(platform, Child.PYTHON, spec, events) as process, - _in_background(_report_once_listening(port, events)), - ): - yield process +async def _report_exit( + child: Child, process: asyncio.subprocess.Process, events: MemoryObjectSendStream[Event] +) -> None: + with events: + events.send_nowait(ChildExited(child, await process.wait())) -async def _report_exit(child: Child, process: asyncio.subprocess.Process, events: asyncio.Queue[Event]) -> None: - events.put_nowait(ChildExited(child, await process.wait())) - - -async def _report_once_listening(port: int, events: asyncio.Queue[Event]) -> None: - while not await _accepts(port): - await asyncio.sleep(PROBE_INTERVAL_SECONDS) - events.put_nowait(Probe.PYTHON_LISTENING) +async def _report_once_listening(port: int, events: MemoryObjectSendStream[Event]) -> None: + with events: + while not await _accepts(port): + await anyio.sleep(PROBE_INTERVAL_SECONDS) + events.send_nowait(Probe.PYTHON_LISTENING) async def _accepts(port: int) -> bool: - try: - _, writer = await asyncio.wait_for(asyncio.open_connection(RUST_HOP, port), PROBE_INTERVAL_SECONDS) - except (OSError, asyncio.TimeoutError): - return False - writer.close() - return True - - -@contextlib.asynccontextmanager -async def _in_background(work: Coroutine[None, None, None]) -> AsyncGenerator[None]: - task: Final = asyncio.create_task(work) - try: - yield - finally: - task.cancel() - await asyncio.gather(task, return_exceptions=True) + with anyio.move_on_after(PROBE_INTERVAL_SECONDS): + try: + stream: Final = await anyio.connect_tcp(RUST_HOP, port) + except OSError: + return False + await stream.aclose() + return True + return False diff --git a/tests/unit/proxy/rust_forward/supervisor/test_runtime.py b/tests/unit/proxy/rust_forward/supervisor/test_runtime.py index ac47b3975ce..824ef9e3945 100644 --- a/tests/unit/proxy/rust_forward/supervisor/test_runtime.py +++ b/tests/unit/proxy/rust_forward/supervisor/test_runtime.py @@ -1,12 +1,29 @@ +import signal import sys +from dataclasses import dataclass, field from pathlib import Path +from types import MappingProxyType from typing import Final +import anyio import pytest from litellm.proxy.rust_forward.platform import Platform, current_platform from litellm.proxy.rust_forward.platform.windows import WindowsPlatform from litellm.proxy.rust_forward.supervisor import supervise +from litellm.proxy.rust_forward.supervisor.children import ChildSpec +from litellm.proxy.rust_forward.supervisor.lifecycle import ( + FAILURE, + Action, + Caught, + Child, + ChildExited, + Event, + Probe, + Send, + Spawn, +) +from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, drive, open_children from tests.unit.proxy.rust_forward.supervisor._support import python_server @@ -43,3 +60,95 @@ def test_rust_opens_the_public_port_only_once_python_listens(tmp_path: Path, mon command: Final = (sys.executable, "-I", "-c", PYTHON_CHILD) supervise(python_server(), command, scripts_dir=str(tmp_path), platform=_fake_gateway(tmp_path)) assert (tmp_path / "rust_saw_python_listening").read_text() == "True" + + +TERM: Final = signal.SIGTERM +POSIX_ONLY: Final = pytest.mark.skipif(sys.platform == "win32", reason="the child traps SIGTERM") +STOPS: Final = frozenset({signal.SIGTERM, signal.SIGINT, signal.SIGQUIT}) + + +@dataclass +class ScriptedExecutor: + """Replays a fixed event script and records every action the driver carries out.""" + + events: list[Event] + failing: frozenset[Child] = frozenset() + done: list[Action] = field(default_factory=list) + + async def spawn(self, child: Child) -> SpawnFailed | None: + self.done.append(Spawn(child)) + return SpawnFailed(FileNotFoundError(child.value)) if child in self.failing else None + + def send(self, child: Child, signum: signal.Signals) -> None: + self.done.append(Send(child, signum)) + + async def next_event(self) -> Event: + return self.events.pop(0) + + +async def test_driver_carries_out_each_action_in_order_and_ends_with_pythons_status() -> None: + executor: Final = ScriptedExecutor( + [Probe.PYTHON_LISTENING, Caught(TERM), ChildExited(Child.RUST, 0), ChildExited(Child.PYTHON, -15)] + ) + assert await drive(executor, STOPS) == -15 + assert executor.done == [Spawn(Child.PYTHON), Spawn(Child.RUST), Send(Child.RUST, TERM), Send(Child.PYTHON, TERM)] + assert executor.events == [] + + +async def test_driver_stops_python_and_fails_when_rust_is_lost() -> None: + executor: Final = ScriptedExecutor( + [Probe.PYTHON_LISTENING, ChildExited(Child.RUST, 0), ChildExited(Child.PYTHON, -15)] + ) + assert await drive(executor, STOPS) == FAILURE + assert executor.done == [Spawn(Child.PYTHON), Spawn(Child.RUST), Send(Child.PYTHON, TERM)] + + +async def test_driver_asks_for_no_event_once_the_status_is_settled() -> None: + executor: Final = ScriptedExecutor([ChildExited(Child.PYTHON, 3), Caught(TERM)]) + assert await drive(executor, STOPS) == 3 + assert executor.events == [Caught(TERM)] + + +async def test_driver_returns_a_failed_spawn_without_waiting_for_events() -> None: + executor: Final = ScriptedExecutor([Probe.PYTHON_LISTENING, Caught(TERM)], failing=frozenset({Child.RUST})) + outcome: Final = await drive(executor, STOPS) + assert isinstance(outcome, SpawnFailed) + assert executor.events == [Caught(TERM)] + + +def _children(argv: tuple[str, ...]) -> MappingProxyType[Child, ChildSpec]: + spec: Final = ChildSpec(argv, MappingProxyType({}), current_platform().spawn_options(())) + return MappingProxyType({Child.PYTHON: spec, Child.RUST: spec}) + + +SLOW_TO_STOP: Final = ( + sys.executable, + "-c", + "import signal, time; signal.signal(signal.SIGTERM, lambda *_: (time.sleep(0.5), exit(0))); time.sleep(30)", +) + + +@POSIX_ONLY +async def test_a_cancelled_supervisor_still_waits_for_the_child_it_stops() -> None: + with anyio.CancelScope() as scope: + async with open_children(current_platform(), _children(SLOW_TO_STOP), 1) as children: + assert await children.spawn(Child.RUST) is None + process: Final = children.processes[Child.RUST] + await anyio.sleep(0.3) + scope.cancel() + await anyio.sleep(1) + assert process.returncode == 0 + + +async def test_exiting_children_are_reported_as_events() -> None: + async with open_children(current_platform(), _children((sys.executable, "-c", "exit(7)")), 1) as children: + assert await children.spawn(Child.RUST) is None + with anyio.fail_after(5): + assert await children.next_event() == ChildExited(Child.RUST, 7) + + +async def test_a_child_that_cannot_start_is_returned_as_a_failure(tmp_path: Path) -> None: + async with open_children(current_platform(), _children((str(tmp_path / "missing"),)), 1) as children: + outcome: Final = await children.spawn(Child.RUST) + assert isinstance(outcome, SpawnFailed) + assert isinstance(outcome.error, FileNotFoundError)