diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 860f01c4ad1..ebab2a118fc 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3046,7 +3046,6 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", - "futures-channel", "futures-core", "futures-util", "h2 0.4.15", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 8634dce92d0..fa2bdb4224c 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -34,7 +34,7 @@ pyo3 = "0.29.2" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" rand = "0.8" -reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls", "http2", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] } rstest = "0.26.1" rstest_reuse = "0.7.0" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } diff --git a/litellm-rust/clippy.toml b/litellm-rust/clippy.toml new file mode 100644 index 00000000000..f7e3293069b --- /dev/null +++ b/litellm-rust/clippy.toml @@ -0,0 +1,10 @@ +# The Tokio runtime is reached only through `host-python/src/execution.rs`, whose fork gate +# must see every entry. Going around it makes a fork-after-use hang instead of raising. +disallowed-methods = [ + { path = "pyo3_async_runtimes::tokio::get_runtime", reason = "use litellm_host_python::run_sync / run_sync_value" }, + { path = "pyo3_async_runtimes::tokio::future_into_py", reason = "use litellm_host_python::run_async / run_async_value" }, + { path = "pyo3_async_runtimes::tokio::future_into_py_with_locals", reason = "use litellm_host_python::run_async / run_async_value" }, + { path = "pyo3_async_runtimes::tokio::local_future_into_py", reason = "use litellm_host_python::run_async / run_async_value" }, + { path = "pyo3_async_runtimes::tokio::run", reason = "use litellm_host_python::run_sync / run_sync_value" }, + { path = "pyo3_async_runtimes::tokio::run_until_complete", reason = "use litellm_host_python::run_sync / run_sync_value" }, +] diff --git a/litellm-rust/crates/host-python/src/execution.rs b/litellm-rust/crates/host-python/src/execution.rs index 45a1183acf5..083c184e37e 100644 --- a/litellm-rust/crates/host-python/src/execution.rs +++ b/litellm-rust/crates/host-python/src/execution.rs @@ -4,6 +4,7 @@ use std::pin::Pin; use std::task::{Context, Poll, Waker}; use std::time::Duration; +use crate::fork_gate::{ForkGate, Refused, RuntimeAlreadyStarted}; use crate::{Pythonized, panic_to_pyerr, release_gil}; use futures_util::FutureExt; use pyo3::exceptions::PyRuntimeError; @@ -12,6 +13,67 @@ use serde::Serialize; use tokio::runtime::{Handle, Runtime}; use tokio::time::{self, MissedTickBehavior}; +pyo3::create_exception!( + _native, + ForkedAfterNativeRuntimeStarted, + PyRuntimeError, + "This process was forked after the native runtime started. Runtime threads do not survive fork(), so native routes cannot run here." +); + +pyo3::create_exception!( + _native, + ProcessReservedForForking, + PyRuntimeError, + "This process was reserved for forking workers, so native routes cannot run here." +); + +static FORK_GATE: ForkGate = ForkGate::new(); + +/// Whether this process has started the Tokio runtime. +pub fn runtime_started() -> bool { + FORK_GATE.started(std::process::id()) +} + +/// Declares that this process exists to fork workers, so it must never start the runtime. +/// Fails if it already has. Workers are unaffected: the reservation is keyed by pid. +pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> { + FORK_GATE.reserve(std::process::id()) +} + +/// The only door to the Tokio runtime: every route reaches it through this module, which is +/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it. +fn enter_runtime() -> PyResult<()> { + FORK_GATE + .enter(std::process::id()) + .map_err(|refused| match refused { + Refused::ReservedForForking => ProcessReservedForForking::new_err( + "this process is reserved for forking workers and cannot run native routes; \ + move the call into a worker, after the fork", + ), + Refused::ForkedAfterStart => ForkedAfterNativeRuntimeStarted::new_err( + "this process was forked after the native runtime started, and runtime threads \ + do not survive fork(); start workers with spawn or forkserver, or fork before \ + the first native call", + ), + }) +} + +#[expect(clippy::disallowed_methods, reason = "this is the gated door")] +fn runtime() -> PyResult<&'static Runtime> { + enter_runtime()?; + Ok(pyo3_async_runtimes::tokio::get_runtime()) +} + +#[expect(clippy::disallowed_methods, reason = "this is the gated door")] +fn future_into_py(py: Python<'_>, future: F) -> PyResult> +where + F: Future> + Send + 'static, + T: for<'py> IntoPyObject<'py> + Send + 'static, +{ + enter_runtime()?; + pyo3_async_runtimes::tokio::future_into_py(py, future) +} + pub fn run_sync( py: Python<'_>, future: F, @@ -22,12 +84,7 @@ where E: Send + 'static, F: Future> + Send + 'static, { - run_sync_on( - py, - pyo3_async_runtimes::tokio::get_runtime(), - future, - map_error, - ) + run_sync_on(py, runtime()?, future, map_error) } pub fn run_sync_value(py: Python<'_>, future: F) -> PyResult @@ -35,7 +92,7 @@ where T: Send + 'static, F: Future> + Send + 'static, { - run_sync_value_on(py, pyo3_async_runtimes::tokio::get_runtime(), future) + run_sync_value_on(py, runtime()?, future) } fn run_sync_value_on(py: Python<'_>, runtime: &Runtime, future: F) -> PyResult @@ -83,7 +140,7 @@ where E: Send + 'static, F: Future> + Send + 'static, { - pyo3_async_runtimes::tokio::future_into_py(py, async move { + future_into_py(py, async move { let result = catch_future_panic(future).await?; let result = map_core_result(result, map_error)?; Ok(Pythonized(result)) @@ -95,7 +152,7 @@ where T: for<'py> IntoPyObject<'py> + Send + 'static, F: Future> + Send + 'static, { - pyo3_async_runtimes::tokio::future_into_py(py, async move { catch_future_panic(future).await? }) + future_into_py(py, async move { catch_future_panic(future).await? }) } pub fn poll_async_value(py: Python<'_>, future: Pin<&mut F>) -> PyResult> @@ -103,8 +160,9 @@ where T: Send, F: Future> + Send, { + let runtime = runtime()?; let result = release_gil(py, || { - let _runtime = pyo3_async_runtimes::tokio::get_runtime().enter(); + let _runtime = runtime.enter(); std::panic::catch_unwind(AssertUnwindSafe(|| { future.poll(&mut Context::from_waker(Waker::noop())) })) @@ -286,27 +344,25 @@ mod tests { } #[pyfunction] - fn runtime_worker_count() -> usize { - pyo3_async_runtimes::tokio::get_runtime() - .metrics() - .num_workers() + fn runtime_worker_count() -> PyResult { + Ok(runtime()?.metrics().num_workers()) } #[pyfunction] - fn runtime_is_responsive(_py: Python<'_>, expected_completions: usize) -> bool { + fn runtime_is_responsive(_py: Python<'_>, expected_completions: usize) -> PyResult { let completion_deadline = Instant::now() + Duration::from_secs(2); while ASYNC_PROBE_COMPLETED.load(Ordering::SeqCst) < expected_completions { if Instant::now() >= completion_deadline { - return false; + return Ok(false); } thread::sleep(Duration::from_millis(1)); } let (heartbeat_tx, heartbeat_rx) = mpsc::sync_channel(1); - pyo3_async_runtimes::tokio::get_runtime().spawn(async move { + runtime()?.spawn(async move { let _ = heartbeat_tx.send(()); }); - heartbeat_rx.recv_timeout(Duration::from_secs(2)).is_ok() + Ok(heartbeat_rx.recv_timeout(Duration::from_secs(2)).is_ok()) } fn extract_bool(py: Python<'_>, result: PyResult>) -> bool { @@ -317,6 +373,16 @@ mod tests { .expect("result should convert") } + #[rstest] + fn reaching_the_runtime_marks_the_process_as_started( + #[from(initialized_python)] python: &InitializedPython, + ) { + python.attach(|py| { + run_sync_value(py, async { Ok(()) }).unwrap(); + assert!(runtime_started()); + }); + } + #[rstest] fn inline_poll_releases_gil_and_enters_runtime( #[from(initialized_python)] python: &InitializedPython, diff --git a/litellm-rust/crates/host-python/src/fork_gate.rs b/litellm-rust/crates/host-python/src/fork_gate.rs new file mode 100644 index 00000000000..62284e978ff --- /dev/null +++ b/litellm-rust/crates/host-python/src/fork_gate.rs @@ -0,0 +1,121 @@ +use std::sync::atomic::{AtomicU32, Ordering}; + +const UNSET: u32 = 0; + +/// Decides which process may use the Tokio runtime. Its worker threads do not survive +/// `fork()`: a child forked after they started hangs on its first native call. The gate turns +/// both halves of that hazard into errors, keyed by pid so a fork needs no hook to be seen: +/// a process reserved for forking can never start the runtime, and a child of a process that +/// did start it is refused instead of hanging. +pub(crate) struct ForkGate { + runtime_pid: AtomicU32, + fork_only_pid: AtomicU32, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Refused { + ReservedForForking, + ForkedAfterStart, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct RuntimeAlreadyStarted; + +impl ForkGate { + pub(crate) const fn new() -> Self { + Self { + runtime_pid: AtomicU32::new(UNSET), + fork_only_pid: AtomicU32::new(UNSET), + } + } + + /// Claims the runtime for `pid`. Claim first, then look for a reservation: `reserve` does + /// the mirror image, so when the two race at least one of them sees the other. + pub(crate) fn enter(&self, pid: u32) -> Result<(), Refused> { + match self + .runtime_pid + .compare_exchange(UNSET, pid, Ordering::SeqCst, Ordering::SeqCst) + { + Err(owner) if owner != pid => return Err(Refused::ForkedAfterStart), + _ => {} + } + + if self.fork_only_pid.load(Ordering::SeqCst) == pid { + // Nothing was started, so the workers forked from here must still find it unclaimed. + let _ = + self.runtime_pid + .compare_exchange(pid, UNSET, Ordering::SeqCst, Ordering::SeqCst); + return Err(Refused::ReservedForForking); + } + + Ok(()) + } + + pub(crate) fn reserve(&self, pid: u32) -> Result<(), RuntimeAlreadyStarted> { + self.fork_only_pid.store(pid, Ordering::SeqCst); + if self.runtime_pid.load(Ordering::SeqCst) == pid { + return Err(RuntimeAlreadyStarted); + } + Ok(()) + } + + pub(crate) fn started(&self, pid: u32) -> bool { + self.runtime_pid.load(Ordering::SeqCst) == pid + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const MASTER: u32 = 100; + const WORKER: u32 = 101; + + #[test] + fn unreserved_process_starts_the_runtime_and_stays_started() { + let gate = ForkGate::new(); + + assert!(!gate.started(MASTER)); + assert_eq!(gate.enter(MASTER), Ok(())); + assert_eq!(gate.enter(MASTER), Ok(())); + assert!(gate.started(MASTER)); + } + + #[test] + fn reserved_process_can_never_start_the_runtime() { + let gate = ForkGate::new(); + + assert_eq!(gate.reserve(MASTER), Ok(())); + assert_eq!(gate.enter(MASTER), Err(Refused::ReservedForForking)); + assert_eq!(gate.enter(MASTER), Err(Refused::ReservedForForking)); + assert!(!gate.started(MASTER)); + } + + #[test] + fn workers_forked_from_a_reserved_process_start_their_own_runtime() { + let gate = ForkGate::new(); + gate.reserve(MASTER).unwrap(); + gate.enter(MASTER).unwrap_err(); + + assert_eq!(gate.enter(WORKER), Ok(())); + assert!(gate.started(WORKER)); + } + + #[test] + fn reserving_after_the_runtime_started_is_refused() { + let gate = ForkGate::new(); + gate.enter(MASTER).unwrap(); + + assert_eq!(gate.reserve(MASTER), Err(RuntimeAlreadyStarted)); + } + + #[test] + fn child_forked_after_the_runtime_started_is_refused_instead_of_hanging() { + let gate = ForkGate::new(); + gate.enter(MASTER).unwrap(); + + assert_eq!(gate.enter(WORKER), Err(Refused::ForkedAfterStart)); + assert!(!gate.started(WORKER)); + assert_eq!(gate.enter(MASTER), Ok(())); + } +} diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 583a4eb91b6..4e6337d916d 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -8,6 +8,7 @@ mod argument; mod callable; mod driver; mod execution; +mod fork_gate; mod gil; mod handle; mod marshal; @@ -18,7 +19,11 @@ pub use adapter::{ pub use argument::lookup; pub use callable::wrap_failure; pub use driver::run_call; -pub use execution::{poll_async_value, run_async, run_async_value, run_sync, run_sync_value}; +pub use execution::{ + ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value, reserve_process_for_forking, run_async, run_async_value, run_sync, + run_sync_value, runtime_started, +}; +pub use fork_gate::RuntimeAlreadyStarted; pub use gil::{release_count, release_gil}; pub use handle::{Execution, ExecutionBody, ExecutionStep}; pub use marshal::{Pythonized, from_py, from_py_argument, panic_to_pyerr, to_py}; diff --git a/litellm-rust/crates/python-bridge/src/diagnostics.rs b/litellm-rust/crates/python-bridge/src/diagnostics.rs index 39fa8bc3596..687a090e768 100644 --- a/litellm-rust/crates/python-bridge/src/diagnostics.rs +++ b/litellm-rust/crates/python-bridge/src/diagnostics.rs @@ -1,5 +1,5 @@ -use litellm_host_python::release_count; -use pyo3::{prelude::*, types::PyDict}; +use litellm_host_python::{release_count, runtime_started}; +use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyDict}; #[pyfunction] pub(crate) fn gil_stats(py: Python<'_>) -> PyResult> { @@ -8,6 +8,20 @@ pub(crate) fn gil_stats(py: Python<'_>) -> PyResult> { Ok(stats.into_any().unbind()) } +/// True once this process has started the native runtime, which does not survive `fork()`. +#[pyfunction] +pub(crate) fn process_state_started() -> bool { + runtime_started() +} + +/// Declares that this process only forks workers: from now on every native route raises here, +/// so the runtime can never start. Raises if it already has. Forked workers are unaffected. +#[pyfunction] +pub(crate) fn reserve_process_for_forking() -> PyResult<()> { + litellm_host_python::reserve_process_for_forking() + .map_err(|_| PyRuntimeError::new_err("the native runtime already started in this process")) +} + #[cfg(feature = "panic-test")] #[pyfunction] pub(crate) fn _panic_for_test() { diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 7eba0d201be..a41e1500f04 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -13,10 +13,12 @@ mod _native { #[pymodule_export] use crate::diagnostics::_panic_for_test; #[pymodule_export] - use crate::diagnostics::gil_stats; + use crate::diagnostics::{gil_stats, process_state_started, reserve_process_for_forking}; #[pymodule_export] use crate::errors::{RustBridgeDeclined, RustUpstreamError}; #[pymodule_export] + use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{ @@ -50,6 +52,8 @@ mod tests { let mut expected = vec![ "RustBridgeDeclined", "RustUpstreamError", + "ForkedAfterNativeRuntimeStarted", + "ProcessReservedForForking", "ocr", "aocr", "transcription", @@ -62,6 +66,8 @@ mod tests { "ResponsesWebSocketConnection", "TokenCounter", "gil_stats", + "process_state_started", + "reserve_process_for_forking", ]; expected.sort_unstable(); diff --git a/litellm-rust/crates/python-bridge/src/routes/responses.rs b/litellm-rust/crates/python-bridge/src/routes/responses.rs index 9c10d58de4f..2e7e8fcbc21 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses.rs @@ -25,7 +25,7 @@ impl ResponsesWebSocketConnection { ) -> PyResult> { let headers = marshal_headers(headers)?; let timeout = optional_timeout(timeout_seconds); - pyo3_async_runtimes::tokio::future_into_py(py, async move { + litellm_host_python::run_async_value(py, async move { let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout) .await .map_err(responses_error_to_pyerr)?; @@ -35,7 +35,7 @@ impl ResponsesWebSocketConnection { fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult> { let inner = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { + litellm_host_python::run_async_value(py, async move { inner .send_text(text) .await @@ -45,14 +45,14 @@ impl ResponsesWebSocketConnection { fn recv_text<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { + litellm_host_python::run_async_value(py, async move { inner.recv_text().await.map_err(responses_error_to_pyerr) }) } fn close<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { + litellm_host_python::run_async_value(py, async move { inner.close().await.map_err(responses_error_to_pyerr) }) } @@ -68,6 +68,10 @@ mod tests { use tokio_tungstenite::{accept_async, tungstenite::Message}; #[test] + #[expect( + clippy::disallowed_methods, + reason = "the test server shares the routes' runtime" + )] fn responses_websocket_connection_round_trips_through_python() { Python::initialize(); let runtime = pyo3_async_runtimes::tokio::get_runtime(); diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 9f2e4c9802e..0477b6c62e9 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -589,6 +589,11 @@ class ProxyInitializationHelpers: gunicorn_options["certfile"] = ssl_certfile_path gunicorn_options["keyfile"] = ssl_keyfile_path + # The master preloads the app and then forks every worker, so native routes are + # forbidden in it: their runtime threads would not survive the fork. + from litellm.rust_bridge.fork_guard import reserve_process_for_forking + + reserve_process_for_forking("the gunicorn master") start_query_engine_reaper() StandaloneApplication(app=app, options=gunicorn_options).run() # Run gunicorn diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 9f959c056de..c0a06364261 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -9,6 +9,8 @@ from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMe class RustBridgeDeclined(Exception): ... class RustUpstreamError(Exception): ... +class ForkedAfterNativeRuntimeStarted(RuntimeError): ... +class ProcessReservedForForking(RuntimeError): ... def ocr( request: LiteLLMOcrRequest, @@ -101,8 +103,12 @@ class TokenCounter: def acount_request(self, body: bytes) -> Future[dict[str, object]]: ... def gil_stats() -> dict[str, int]: ... +def process_state_started() -> bool: ... +def reserve_process_for_forking() -> None: ... __all__ = [ + "ForkedAfterNativeRuntimeStarted", + "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", "RustUpstreamError", @@ -116,5 +122,7 @@ __all__ = [ "gil_stats", "messages", "ocr", + "process_state_started", + "reserve_process_for_forking", "transcription", ] diff --git a/litellm/rust_bridge/fork_guard.py b/litellm/rust_bridge/fork_guard.py new file mode 100644 index 00000000000..c94665fb8db --- /dev/null +++ b/litellm/rust_bridge/fork_guard.py @@ -0,0 +1,47 @@ +"""Fork safety of the Rust extension. + +Its runtime threads do not survive ``fork``, so a child forked after the first native call +cannot run native routes: it raises ``ForkedAfterNativeRuntimeStarted`` instead of hanging. +Fork before the first native call, or start workers with ``spawn`` / ``forkserver``. + +A process whose job is to fork workers (the gunicorn master under ``preload``) reserves itself: +from then on any native route called in it raises ``ProcessReservedForForking`` at the call +site, so the runtime can never start there. Workers forked from it are unaffected. +""" + +from __future__ import annotations + +from typing import Final + +from litellm.rust_bridge.loader import get_native_bridge + + +class NativeStateStartedBeforeFork(RuntimeError): + pass + + +class _NeverRaised(RuntimeError): + """Stands in for a native exception when the extension is unavailable or predates it.""" + + +_native: Final = get_native_bridge() +ForkedAfterNativeRuntimeStarted: Final[type[RuntimeError]] = getattr( + _native, "ForkedAfterNativeRuntimeStarted", _NeverRaised +) +ProcessReservedForForking: Final[type[RuntimeError]] = getattr(_native, "ProcessReservedForForking", _NeverRaised) + + +def reserve_process_for_forking(where: str) -> None: + """Forbid native routes in this process. Raises if one already ran here.""" + native: Final = get_native_bridge() + reserve: Final = getattr(native, "reserve_process_for_forking", None) + if not callable(reserve): + return + try: + reserve() + except RuntimeError as error: + raise NativeStateStartedBeforeFork( + f"The LiteLLM Rust extension already ran a native route in {where}, and its runtime " + "threads do not survive fork(). Move the native call (warm-up, health check, " + "import-time initialization) into the worker, after the fork." + ) from error diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 712c526b244..c806725d594 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -21,6 +21,15 @@ from uvicorn.importer import import_from_string from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +@pytest.fixture(autouse=True) +def fork_reservation(): + """Reserving is irreversible: it would forbid native routes in this pytest worker for good""" + with patch( # test-quality-ok: process-global native state, a real reservation would poison every later test in the worker + "litellm.rust_bridge.fork_guard.reserve_process_for_forking" + ) as reserve: + yield reserve + + @pytest.mark.xdist_group("proxy_cli") class TestProxyInitializationHelpers: @patch("importlib.metadata.version") @@ -1574,6 +1583,32 @@ class TestProxyInitializationHelpers: assert captured["options"]["max_requests"] == 1000 assert captured["options"]["max_requests_jitter"] == 50 + @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") + def test_gunicorn_master_is_reserved_for_forking_before_it_runs(self, fork_reservation): + """preload forks workers from the master, so native routes are forbidden there first""" + pytest.importorskip("gunicorn") + reserved_before_run: list = [] + + def capture_run(self): + reserved_before_run.append(fork_reservation.call_args) + + with ( + patch("gunicorn.app.base.BaseApplication.run", capture_run), + patch( # test-quality-ok: option tests must not start a thread or change the pytest worker's child ownership + "litellm.proxy.proxy_cli.start_query_engine_reaper" + ), + ): + ProxyInitializationHelpers._run_gunicorn_server( + host="127.0.0.1", + port=4012, + app=MagicMock(), + num_workers=2, + ssl_certfile_path=None, + ssl_keyfile_path=None, + ) + + assert [call.args for call in reserved_before_run] == [("the gunicorn master",)] + @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") def test_gunicorn_jitter_without_base_warns(self): """gunicorn path warns when jitter is set without --max_requests_before_restart""" diff --git a/tests/test_litellm/rust_bridge/test_fork_guard.py b/tests/test_litellm/rust_bridge/test_fork_guard.py new file mode 100644 index 00000000000..54bfd54c230 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_fork_guard.py @@ -0,0 +1,36 @@ +from types import SimpleNamespace + +import pytest + +from litellm.rust_bridge import fork_guard + + +def _reserve_with(monkeypatch: pytest.MonkeyPatch, native: object) -> None: + monkeypatch.setattr(fork_guard, "get_native_bridge", lambda: native) + fork_guard.reserve_process_for_forking("the gunicorn master") + + +def test_missing_extension_has_nothing_to_reserve(monkeypatch: pytest.MonkeyPatch) -> None: + _reserve_with(monkeypatch, None) + + +def test_extension_built_before_reservation_existed_passes(monkeypatch: pytest.MonkeyPatch) -> None: + _reserve_with(monkeypatch, SimpleNamespace()) + + +def test_unused_extension_is_reserved(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[None] = [] + + _reserve_with(monkeypatch, SimpleNamespace(reserve_process_for_forking=lambda: calls.append(None))) + + assert calls == [None] + + +def test_used_extension_refuses_and_names_the_place(monkeypatch: pytest.MonkeyPatch) -> None: + def reserve() -> None: + raise RuntimeError("the native runtime already started in this process") + + with pytest.raises(fork_guard.NativeStateStartedBeforeFork, match="the gunicorn master") as raised: + _reserve_with(monkeypatch, SimpleNamespace(reserve_process_for_forking=reserve)) + + assert isinstance(raised.value.__cause__, RuntimeError) diff --git a/tests/test_litellm_rust/test_fork_guard.py b/tests/test_litellm_rust/test_fork_guard.py new file mode 100644 index 00000000000..b92095cbaaa --- /dev/null +++ b/tests/test_litellm_rust/test_fork_guard.py @@ -0,0 +1,150 @@ +import os +import subprocess +import sys +import textwrap + +import pytest + +pytestmark = pytest.mark.requires_rust_extension + +_NATIVE_CONTRACT = textwrap.dedent( + """ + import os + from litellm.rust_bridge import _native + from litellm.rust_bridge.fork_guard import reserve_process_for_forking + + def native_route_error(): + import asyncio + + async def call(): + await _native.ResponsesWebSocketConnection.connect("ws://127.0.0.1:1", {}, 0.2) + + try: + asyncio.run(call()) + except Exception as error: + return f"{type(error).__name__}: {error}" + return "" + + assert _native.process_state_started() is False + reserve_process_for_forking("the test master") + assert native_route_error().startswith("ProcessReservedForForking: ") + assert _native.process_state_started() is False + + pid = os.fork() + if pid == 0: + error = native_route_error() + started = _native.process_state_started() + os._exit(0 if started and "reserved" not in error and "forked" not in error else 1) + assert os.waitpid(pid, 0)[1] == 0 + + pid = os.fork() + if pid == 0: + native_route_error() + grandchild = os.fork() + if grandchild == 0: + os._exit(0 if native_route_error().startswith("ForkedAfterNativeRuntimeStarted: ") else 1) + os._exit(os.waitpid(grandchild, 0)[1]) + assert os.waitpid(pid, 0)[1] == 0 + """ +) + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="fork only") +def test_compiled_extension_forbids_the_master_and_frees_its_workers() -> None: + env = {**os.environ, "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES"} + + result = subprocess.run( + [sys.executable, "-c", _NATIVE_CONTRACT], capture_output=True, text=True, timeout=60, env=env + ) + + assert result.returncode == 0, result.stderr + + +_SDK_CONTRACT = textwrap.dedent( + """ + import asyncio, json, multiprocessing, os, threading + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + import litellm + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + if self.headers.get("User-Agent", "").startswith("python-httpx"): + self.send_response(418) + self.end_headers() + return + body = json.dumps({ + "pages": [{"index": 0, "markdown": "native", "images": [], "dimensions": None}], + "model": "mistral-ocr-latest", + "usage_info": {"pages_processed": 1, "doc_size_bytes": 3}, + }).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + arguments = { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + "api_key": "test-key", + "api_base": f"http://127.0.0.1:{server.server_port}", + "num_retries": 0, + } + litellm.rust(True) + + SERVED, REFUSED, OTHER = 0, 3, 4 + + def outcome(asynchronous): + try: + response = asyncio.run(litellm.aocr(**arguments)) if asynchronous else litellm.ocr(**arguments) + except ForkedAfterNativeRuntimeStarted: + return REFUSED + except Exception: + return OTHER + return SERVED if response.pages[0].markdown == "native" else OTHER + + def forked(asynchronous): + pid = os.fork() + if pid == 0: + os._exit(outcome(asynchronous)) + return os.waitstatus_to_exitcode(os.waitpid(pid, 0)[1]) + + def pooled(asynchronous): + with multiprocessing.get_context("fork").Pool(1) as pool: + return pool.apply(outcome, (asynchronous,)) + + # Forking before the first native call is fine: the child starts its own runtime. + assert [forked(False), forked(True)] == [SERVED, SERVED] + + assert outcome(False) == SERVED + # After it, a forked child is told so instead of hanging on threads that do not exist. + assert [forked(False), forked(True)] == [REFUSED, REFUSED] + assert [pooled(False), pooled(True)] == [REFUSED, REFUSED] + # The parent is not poisoned by any of it. + assert [outcome(False), outcome(True)] == [SERVED, SERVED] + """ +) + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="fork only") +def test_sdk_call_in_a_child_forked_after_native_use_raises_instead_of_hanging() -> None: + env = { + **os.environ, + "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES", + "LITELLM_RUST": "1", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + } + + result = subprocess.run( + [sys.executable, "-c", _SDK_CONTRACT], capture_output=True, text=True, timeout=120, env=env + ) + + assert result.returncode == 0, result.stderr