From d187fe8b2c5ed10c57ac64afe4f2c91af9eca308 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sat, 19 Sep 2026 15:12:53 -0700 Subject: [PATCH] refactor(rust): compose callback lifecycles in a chain and tighten both contracts Compose the legacy and v1 lifecycles behind one PythonLifecycle through LifecycleChain, built once in python-bridge, and simplify the boundary: - drop LifecycleEvent::Started; begin carries the start time and is the first thing a lifecycle hears of a call - give PythonLifecycle pass-through defaults for the threading steps and the stream notices, and document which steps thread and which observe - export one is_cancellation from litellm-host-python - callbacks-v1: private RequestFacts fields, a golden file per EventKind by construction, Python EVENTS and SUPPORTED_SCHEMAS pinned to the goldens - callbacks-v1-python: the snapshot only parses (register is the only place that rejects), Handler owns call/is_async, Pending keeps a subscriber index - callbacks-legacy-python: LegacyCallbacks becomes inherent PythonLogger methods, deployment hooks become free functions, retained headers are traversed and cleared - define the shared terms in host-python/AGENTS.md and each contract's own terms in its crate - remove test_loader.py, whose fork hook was never committed Co-Authored-By: Claude Fable 5.1 --- litellm-rust/Cargo.lock | 2 + .../crates/callbacks-legacy-python/AGENTS.md | 14 +- .../crates/callbacks-legacy-python/Cargo.toml | 4 +- .../callbacks-legacy-python/src/adapter.rs | 33 +- .../callbacks-legacy-python/src/call.rs | 35 +- .../callbacks-legacy-python/src/callbacks.rs | 94 +-- .../callbacks-legacy-python/src/deferred.rs | 7 +- .../crates/callbacks-legacy-python/src/lib.rs | 16 +- .../callbacks-legacy-python/src/logger.rs | 83 ++- .../src/preparation.rs | 5 +- .../callbacks-legacy-python/src/python.rs | 7 +- .../crates/callbacks-v1-python/AGENTS.md | 15 +- .../crates/callbacks-v1-python/src/adapter.rs | 201 +++---- .../crates/callbacks-v1-python/src/call.rs | 41 +- .../crates/callbacks-v1-python/src/lib.rs | 10 +- .../crates/callbacks-v1-python/src/python.rs | 11 +- .../callbacks-v1-python/src/registry.rs | 96 --- .../callbacks-v1-python/src/subscribers.rs | 113 ++++ .../callbacks-v1-python/tests/lifecycle.rs | 22 +- litellm-rust/crates/callbacks-v1/AGENTS.md | 15 +- .../crates/callbacks-v1/src/envelope.rs | 53 +- litellm-rust/crates/callbacks-v1/src/lib.rs | 4 + litellm-rust/crates/callbacks-v1/src/patch.rs | 4 + .../crates/callbacks-v1/src/policy.rs | 3 + .../crates/callbacks-v1/src/redact.rs | 3 + litellm-rust/crates/host-python/AGENTS.md | 20 + .../crates/host-python/src/adapter.rs | 84 ++- litellm-rust/crates/host-python/src/chain.rs | 560 ++++++++++++++++++ litellm-rust/crates/host-python/src/driver.rs | 22 +- litellm-rust/crates/host-python/src/lib.rs | 5 +- litellm-rust/crates/python-bridge/AGENTS.md | 6 +- litellm-rust/crates/python-bridge/Cargo.toml | 2 + .../crates/python-bridge/src/callbacks.rs | 79 +++ litellm-rust/crates/python-bridge/src/lib.rs | 1 + .../python-bridge/src/routes/messages/mod.rs | 12 +- .../python-bridge/src/routes/ocr/mod.rs | 14 +- .../rust_bridge/test_callbacks_v1_python.py | 7 +- tests/test_litellm/rust_bridge/test_loader.py | 27 - .../ocr/test_callbacks_v1.py | 141 +++++ 39 files changed, 1288 insertions(+), 583 deletions(-) delete mode 100644 litellm-rust/crates/callbacks-v1-python/src/registry.rs create mode 100644 litellm-rust/crates/callbacks-v1-python/src/subscribers.rs create mode 100644 litellm-rust/crates/host-python/src/chain.rs create mode 100644 litellm-rust/crates/python-bridge/src/callbacks.rs delete mode 100644 tests/test_litellm/rust_bridge/test_loader.py create mode 100644 tests/test_litellm_rust/ocr/test_callbacks_v1.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 464d7352298..9c34b1b6dcc 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2222,8 +2222,10 @@ dependencies = [ "litellm-auth", "litellm-auth-gcp", "litellm-callbacks-legacy-python", + "litellm-callbacks-v1-python", "litellm-core", "litellm-core-utils", + "litellm-host", "litellm-host-python", "litellm-http", "litellm-llms", diff --git a/litellm-rust/crates/callbacks-legacy-python/AGENTS.md b/litellm-rust/crates/callbacks-legacy-python/AGENTS.md index 9630fb846af..db281db43db 100644 --- a/litellm-rust/crates/callbacks-legacy-python/AGENTS.md +++ b/litellm-rust/crates/callbacks-legacy-python/AGENTS.md @@ -1,10 +1,20 @@ - Target invariants, not completion claims - Keep this crate the legacy `@client` wrapper as the native call sees it, and nothing else: the `Logging` contract (`function_setup`, the deployment hooks, `pre_call`/`post_call`, the sync and async success and failure fan-out, the deferred proxy release, the argument sharing those callbacks rely on) plus the kwargs rewrites the wrapper makes on the way in (credential-name inheritance, the budget and retry-count limits) - The driver in `litellm-host-python`, the routes and core see one `PythonLifecycle`; they never learn which Python objects consume a call -- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`) +- Terms (lifecycle terms are in `host-python/AGENTS.md`) + - `Logging`: litellm's Python class; logger: the one `Logging` instance of this call (`PythonLogger`), reused from `litellm_logging_obj` or built by `function_setup` + - Borrowed function: a litellm Python internal Rust still calls, one variant of `LegacyPython`; a user's own callback is never one + - Fan-out: `Logging` dispatching to every registered callback; family: the sync or the async set of success and failure handlers + - Deployment hooks: the awaited pre-call, post-call and failure hooks on each callback; async calls only, as in `@client` + - Re-aliasing: replacing body values with the caller's equal objects before `pre_call`, so a callback mutates what the caller holds + - Deferred release: the proxy holds back async success logging until it accepts the response (`PendingLogging.release`) + - Internal call: a proxy-internal call; when it is async, the async success dispatch and both failure families are skipped, and the sync success handlers still run + - Pass-through billing (`PassThroughStream`): end-of-stream cost logging from the delivered chunks; a failure after the stream opened bills them as partial usage, and a sync call falls back to the plain failure handler + - Redaction here is `****`, matching what the Python path logs; `[REDACTED]` belongs to the v1 contract +- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`, `Streaming`) - The enum only shrinks: when Rust owns a subsystem, delete its group rather than adding a Rust path beside it - Calling a user's own callback directly is permanent Python surface and gets its own type outside `LegacyPython` - - `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes hand it over through `run_legacy_python_call` and keep no copy + - `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes capture it and the bridge's `run_python_call` hands it to `LegacyPythonLifecycle::new`, keeping no copy - `setup` reuses a `Logging` the caller passed as `litellm_logging_obj` (the proxy and Router are the live cases) and otherwise builds one through `function_setup`, as `@client` does - Either way every phase calls the same `Logging` method the Python path calls; which callbacks run is `Logging`'s decision, never this crate's - Callbacks receive the caller's own objects and may mutate them; this crate alone carries that obligation diff --git a/litellm-rust/crates/callbacks-legacy-python/Cargo.toml b/litellm-rust/crates/callbacks-legacy-python/Cargo.toml index fe19578e04d..1fc81b6012b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/Cargo.toml +++ b/litellm-rust/crates/callbacks-legacy-python/Cargo.toml @@ -9,13 +9,11 @@ autotests = false [dependencies] litellm-host.workspace = true litellm-host-python.workspace = true - pyo3.workspace = true -strum.workspace = true serde_json.workspace = true +strum.workspace = true [dev-dependencies] litellm-auth.workspace = true proptest.workspace = true rstest.workspace = true -serde_json.workspace = true diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index cee1811c2cf..cb28a60a2e3 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -6,10 +6,10 @@ use litellm_host::event::{ FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest, epoch_seconds, }; use litellm_host_python::{ - LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, missing_state, to_py, + LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py, }; use pyo3::{ - exceptions::{PyBaseException, PyException}, + exceptions::PyBaseException, gc::{PyTraverseError, PyVisit}, prelude::*, types::{PyDict, PyList}, @@ -17,7 +17,8 @@ use pyo3::{ use serde_json::Value; use crate::{ - DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger, + PublicCall, PythonLogger, after_deployment_failure, after_deployment_success, + before_deployment_call, deferred::{PendingLogging, PendingSuccess}, finalize, is_internal_call, prepare, python::Streaming, @@ -79,10 +80,6 @@ fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult> { .map(Bound::unbind) } -fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool { - !error.is_instance_of::(py) -} - impl LegacyPythonLifecycle { pub fn new( py: Python<'_>, @@ -151,7 +148,7 @@ impl LegacyPythonLifecycle { fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> { match self.try_dispatch_success(py) { - Err(error) if error.is_instance_of::(py) => { + Err(error) if !is_cancellation(py, &error) => { error.write_unraisable(py, self.logger.as_ref().map(|logger| logger.object(py))); Ok(()) } @@ -210,7 +207,7 @@ impl LegacyPythonLifecycle { ), ); match billed { - Err(error) if error.is_instance_of::(py) => { + Err(error) if !is_cancellation(py, &error) => { error.write_unraisable(py, Some(logger.object(py))); Ok(()) } @@ -285,10 +282,10 @@ impl PythonLifecycle for LegacyPythonLifecycle { &mut self, py: Python<'_>, arguments: Py, - started_at: f64, + start_time: f64, ) -> PyResult { self.call.set_kwargs(arguments); - self.start = datetime(py, started_at)?; + self.start = datetime(py, start_time)?; self.internal = is_internal_call(py)?; let result = setup( py, @@ -302,7 +299,7 @@ impl PythonLifecycle for LegacyPythonLifecycle { self.call.set_kwargs(result.kwargs()?); if self.runs_deployment_hooks() { self.pending = Some(Pending::DeploymentPreCall); - return Ok(LifecycleStep::Await(DeploymentHooks::before_call( + return Ok(LifecycleStep::Await(before_deployment_call( py, self.call.kwargs(), self.surface.call_type, @@ -365,7 +362,7 @@ impl PythonLifecycle for LegacyPythonLifecycle { self.response = Some(response); if self.runs_deployment_hooks() { self.pending = Some(Pending::DeploymentPostCall); - return Ok(LifecycleStep::Await(DeploymentHooks::after_success( + return Ok(LifecycleStep::Await(after_deployment_success( py, self.call.kwargs(), &self.response, @@ -377,7 +374,6 @@ impl PythonLifecycle for LegacyPythonLifecycle { fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { match event { - LifecycleEvent::Started { .. } => Ok(LifecycleStep::Done), LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => { let api_key = self .context @@ -418,7 +414,7 @@ impl PythonLifecycle for LegacyPythonLifecycle { { let error = self.error.as_ref().ok_or_else(missing_state)?; self.pending = Some(Pending::DeploymentFailure); - return Ok(LifecycleStep::Await(DeploymentHooks::after_failure( + return Ok(LifecycleStep::Await(after_deployment_failure( py, self.call.kwargs(), error, @@ -475,7 +471,11 @@ impl PythonLifecycle for LegacyPythonLifecycle { { error.write_unraisable(py, None); } + self.pending = None; + self.response = None; + self.error = None; self.body = None; + self.headers = None; self.context = None; self.stream = None; } @@ -493,7 +493,8 @@ impl PythonLifecycle for LegacyPythonLifecycle { visit.call(&stream.chunks)?; visit.call(&stream.first_chunk)?; } - visit.call(&self.body) + visit.call(&self.body)?; + visit.call(&self.headers) } } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/call.rs b/litellm-rust/crates/callbacks-legacy-python/src/call.rs index 5dedb705974..022f1d5fcf2 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/call.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/call.rs @@ -3,16 +3,13 @@ //! lifetime. No other callback host has that obligation, which is why nothing outside //! this crate holds them. -use litellm_host::{machine::Machine, route::Route}; -use litellm_host_python::{RouteHost, lookup, run_call}; +use litellm_host_python::lookup; use pyo3::{ gc::{PyTraverseError, PyVisit}, prelude::*, types::{PyDict, PyTuple}, }; -use crate::{LegacyPythonLifecycle, LegacyPythonSurface}; - pub struct PublicCall { args: Py, kwargs: Py, @@ -34,6 +31,11 @@ impl PublicCall { }) } + /// The keyword view the call starts from, which the driver hands to `begin`. + pub fn arguments(&self, py: Python<'_>) -> Py { + self.kwargs.clone_ref(py) + } + pub(crate) fn args(&self) -> &Py { &self.args } @@ -63,31 +65,6 @@ impl PublicCall { } } -/// Runs one native call under the legacy `Logging` contract: the route host projects from -/// the keyword view the contract prepares, and the contract observes the call. -pub fn run_legacy_python_call( - py: Python<'_>, - surface: LegacyPythonSurface, - call: PublicCall, - machine: M, - route: H, - asynchronous: bool, -) -> PyResult> -where - H: RouteHost + 'static, - M: Machine::Response> + 'static, -{ - let arguments = call.kwargs.clone_ref(py); - run_call( - py, - machine, - route, - Box::new(LegacyPythonLifecycle::new(py, surface, call, asynchronous)), - arguments, - asynchronous, - ) -} - #[cfg(test)] mod tests { use super::*; diff --git a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs index ccc6d260b29..cba711a6ab4 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs @@ -9,78 +9,10 @@ use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; use crate::logger::PythonLogger; use crate::python::{Logging, Wrapper}; -pub trait LegacyCallbacks { +impl PythonLogger { /// `Logging.update_from_kwargs`: what the logger is told about the request it is /// about to see, with consumed credentials redacted. - fn update_from_kwargs( - &self, - py: Python<'_>, - kwargs: &Py, - wire: &WireRequest, - context: &RequestContext, - ) -> PyResult<()>; - - /// `Logging.pre_call`. - fn pre_call( - &self, - py: Python<'_>, - input: &str, - api_key: Option<&str>, - body: &Bound<'_, PyDict>, - headers: &Bound<'_, PyDict>, - url: &str, - ) -> PyResult<()>; - - /// `Logging.post_call`. - fn post_call( - &self, - py: Python<'_>, - original_response: &str, - api_key: Option<&str>, - body: Option<&Py>, - headers: Option<&Py>, - ) -> PyResult<()>; - - fn defers_async_logging(&self, py: Python<'_>) -> bool; - - fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()>; - - fn sync_success_for_async_call( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()>; - - fn failure( - &self, - py: Python<'_>, - error: &Py, - start: &Py, - end: &Option>, - asynchronous: bool, - ) -> PyResult>>; - - fn submit_success( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()>; - - fn enqueue_success( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()>; -} - -impl LegacyCallbacks for PythonLogger { - fn update_from_kwargs( + pub(crate) fn update_from_kwargs( &self, py: Python<'_>, kwargs: &Py, @@ -128,7 +60,8 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn pre_call( + /// `Logging.pre_call`. + pub(crate) fn pre_call( &self, py: Python<'_>, input: &str, @@ -145,7 +78,8 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn post_call( + /// `Logging.post_call`. + pub(crate) fn post_call( &self, py: Python<'_>, original_response: &str, @@ -163,19 +97,19 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn defers_async_logging(&self, py: Python<'_>) -> bool { + pub(crate) fn defers_async_logging(&self, py: Python<'_>) -> bool { Logging::DefersAsync .call(py, (self.object(py),)) .and_then(|value| value.extract()) .unwrap_or(false) } - fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> { Logging::DeferSuccess.call(py, (self.object(py), pending))?; Ok(()) } - fn sync_success_for_async_call( + pub(crate) fn sync_success_for_async_call( &self, py: Python<'_>, response: &Option>, @@ -186,7 +120,7 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn failure( + pub(crate) fn failure( &self, py: Python<'_>, error: &Py, @@ -199,7 +133,7 @@ impl LegacyCallbacks for PythonLogger { Ok(asynchronous.then(|| value.unbind())) } - fn submit_success( + pub(crate) fn submit_success( &self, py: Python<'_>, response: &Option>, @@ -210,7 +144,7 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn enqueue_success( + pub(crate) fn enqueue_success( &self, py: Python<'_>, response: &Option>, @@ -227,9 +161,7 @@ impl LegacyCallbacks for PythonLogger { } enqueue.map(|_| ()) } -} -impl PythonLogger { pub(crate) fn initialize_failure_context( &self, py: Python<'_>, @@ -305,6 +237,6 @@ fn redact( } /// Proxy-internal calls skip the legacy success fan-out. -pub fn is_internal_call(py: Python<'_>) -> PyResult { +pub(crate) fn is_internal_call(py: Python<'_>) -> PyResult { Wrapper::IsInternalCall.call(py, ())?.extract() } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs b/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs index b18012f926e..6c94e565045 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs @@ -1,9 +1,10 @@ //! The proxy's deferred success release: the async success handler is queued only once //! the proxy accepts the response, and at most once. -use pyo3::{exceptions::PyException, prelude::*}; +use litellm_host_python::is_cancellation; +use pyo3::prelude::*; -use crate::{LegacyCallbacks, PythonLogger}; +use crate::PythonLogger; pub(crate) struct PendingSuccess { pub(crate) logger: PythonLogger, @@ -37,7 +38,7 @@ impl PendingLogging { && success { match pending.asynchronous(py) { - Err(error) if error.is_instance_of::(py) => { + Err(error) if !is_cancellation(py, &error) => { error.write_unraisable(py, Some(pending.logger.object(py))); } result => return result, diff --git a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs index 6467b994338..5aaf7048b71 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs @@ -6,8 +6,8 @@ //! core never learn which Python object is on the other end. //! //! Legacy callbacks receive the caller's own objects and may mutate them. [`PublicCall`] -//! is where those objects live, and [`run_legacy_python_call`] is how a route hands them over -//! without keeping a copy. +//! is where those objects live; the bridge hands it to [`LegacyPythonLifecycle::new`] and +//! keeps no copy. mod adapter; mod call; @@ -20,9 +20,11 @@ mod python; #[path = "../tests/support.rs"] mod test_support; -pub(crate) use adapter::LegacyPythonLifecycle; -pub use adapter::{LegacyPythonSurface, PassThroughStream}; -pub use call::{PublicCall, run_legacy_python_call}; -pub(crate) use callbacks::{LegacyCallbacks, is_internal_call}; -pub(crate) use logger::{DeploymentHooks, PythonLogger, finalize, setup}; +pub use adapter::{LegacyPythonLifecycle, LegacyPythonSurface, PassThroughStream}; +pub use call::PublicCall; +pub(crate) use callbacks::is_internal_call; +pub(crate) use logger::{ + PythonLogger, after_deployment_failure, after_deployment_success, before_deployment_call, + finalize, setup, +}; pub(crate) use preparation::prepare; diff --git a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs index 38f3bf29828..03c2313182a 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs @@ -1,3 +1,6 @@ +//! One call's `Logging` instance (the logger) and the `@client` wrapper functions that +//! create and finish it: `function_setup`, `finalize` and the awaited deployment hooks. + use pyo3::{ exceptions::PyBaseException, gc::{PyTraverseError, PyVisit}, @@ -5,10 +8,10 @@ use pyo3::{ types::{PyDict, PyTuple}, }; -use crate::python::{self, Wrapper}; +use crate::python::{DeploymentHooks, Wrapper}; /// The `Logging` instance one call fans out through. -pub struct PythonLogger { +pub(crate) struct PythonLogger { object: Py, } @@ -21,17 +24,17 @@ impl PythonLogger { self.object.bind(py) } - pub fn clone_ref(&self, py: Python<'_>) -> Self { + pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { Self { object: self.object.clone_ref(py), } } - pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.object) } - pub fn restore_context(&self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn restore_context(&self, py: Python<'_>) -> PyResult<()> { Wrapper::RestoreContext.call(py, (self.object(py),))?; Ok(()) } @@ -45,19 +48,19 @@ impl FromPyObject<'_, '_> for PythonLogger { } } -pub struct SetupResult<'py>(Bound<'py, PyAny>); +pub(crate) struct SetupResult<'py>(Bound<'py, PyAny>); impl SetupResult<'_> { - pub fn logger(&self) -> PyResult { + pub(crate) fn logger(&self) -> PyResult { Ok(PythonLogger::new(self.0.getattr("logger")?.unbind())) } - pub fn kwargs(&self) -> PyResult> { + pub(crate) fn kwargs(&self) -> PyResult> { Ok(self.0.getattr("kwargs")?.extract()?) } } -pub fn setup<'py>( +pub(crate) fn setup<'py>( py: Python<'py>, call_type: &str, args: &Py, @@ -70,7 +73,7 @@ pub fn setup<'py>( .map(SetupResult) } -pub fn finalize( +pub(crate) fn finalize( py: Python<'_>, response: &Option>, logger: &PythonLogger, @@ -82,40 +85,36 @@ pub fn finalize( Ok(()) } -pub struct DeploymentHooks; +pub(crate) fn before_deployment_call( + py: Python<'_>, + kwargs: &Py, + call_type: &str, +) -> PyResult> { + DeploymentHooks::BeforeDeploymentCall + .call(py, (kwargs, call_type)) + .map(Bound::unbind) +} -impl DeploymentHooks { - pub fn before_call( - py: Python<'_>, - kwargs: &Py, - call_type: &str, - ) -> PyResult> { - python::DeploymentHooks::BeforeDeploymentCall - .call(py, (kwargs, call_type)) - .map(Bound::unbind) - } +pub(crate) fn after_deployment_success( + py: Python<'_>, + kwargs: &Py, + response: &Option>, + call_type: &str, +) -> PyResult> { + DeploymentHooks::AfterDeploymentSuccess + .call(py, (kwargs, response, call_type)) + .map(Bound::unbind) +} - pub fn after_success( - py: Python<'_>, - kwargs: &Py, - response: &Option>, - call_type: &str, - ) -> PyResult> { - python::DeploymentHooks::AfterDeploymentSuccess - .call(py, (kwargs, response, call_type)) - .map(Bound::unbind) - } - - pub fn after_failure( - py: Python<'_>, - kwargs: &Py, - error: &Py, - call_type: &str, - ) -> PyResult> { - python::DeploymentHooks::AfterDeploymentFailure - .call(py, (kwargs, error, call_type)) - .map(Bound::unbind) - } +pub(crate) fn after_deployment_failure( + py: Python<'_>, + kwargs: &Py, + error: &Py, + call_type: &str, +) -> PyResult> { + DeploymentHooks::AfterDeploymentFailure + .call(py, (kwargs, error, call_type)) + .map(Bound::unbind) } #[cfg(test)] diff --git a/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs b/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs index aab654c9893..26f0039576a 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs @@ -1,3 +1,6 @@ +//! The kwargs rewrites the `@client` wrapper makes before the call: the logger is attached, +//! a named credential's values are inherited, and the budget and retry-count limits run. + use pyo3::{ prelude::*, types::{PyDict, PyList}, @@ -17,7 +20,7 @@ impl<'py> CredentialEntry<'py> { } } -pub fn prepare<'py>( +pub(crate) fn prepare<'py>( py: Python<'py>, kwargs: &Bound<'py, PyDict>, logger: &crate::PythonLogger, diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index cb609d52878..1b225e03bae 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -1,3 +1,6 @@ +//! The Python contract of this crate: every function of +//! `litellm/rust_bridge/callbacks_legacy_python.py` that Rust calls, by name. + use pyo3::prelude::*; use strum::{IntoStaticStr, VariantArray}; @@ -153,7 +156,7 @@ mod tests { use crate::test_support::PYTHON_CONTRACT; #[test] - fn every_borrowed_function_is_in_the_python_contract() { + fn every_function_is_in_the_python_contract() { let contract: serde_json::Map = serde_json::from_str(PYTHON_CONTRACT).unwrap(); let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect(); @@ -177,7 +180,7 @@ mod tests { ) .map(LegacyPython::name) .collect(); - assert_eq!(called.len(), declared.len(), "a function is borrowed twice"); + assert_eq!(called.len(), declared.len(), "a function is listed twice"); assert_eq!(called.into_iter().collect::>(), declared); } } diff --git a/litellm-rust/crates/callbacks-v1-python/AGENTS.md b/litellm-rust/crates/callbacks-v1-python/AGENTS.md index 5f1f7bf3293..c28c89134d2 100644 --- a/litellm-rust/crates/callbacks-v1-python/AGENTS.md +++ b/litellm-rust/crates/callbacks-v1-python/AGENTS.md @@ -2,17 +2,26 @@ - Keep this crate the Python runtime adapter for `litellm-callbacks-v1` and nothing else: the subscriber snapshot and one `PythonLifecycle` that delivers the core contract - Envelopes, sequencing, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1` - `litellm/rust_bridge/callbacks_v1_python.py` is the only Python module behind it; `python_contract.json` pins its functions on both sides +- Terms (contract terms are in `callbacks-v1/AGENTS.md`, lifecycle terms in `host-python/AGENTS.md`) + - Registry: the process-wide subscriber list; it lives in Python (`register`/`unregister`), never in Rust + - Snapshot: that list read once per call, with one observer and one interceptor handler already selected per subscriber for the call's mode (`subscribers.rs`) + - Projection: the public response turned into JSON by `project_response`; a failed projection becomes `response_error`, never a failed call + - Report: logging an observer's swallowed error through `report` + - Continuation: what a step returns to the driver once every observer of its envelope has run + - "Callback" in the Python module (`Callback` protocol) is the user's object; once registered it is a subscriber - No Python object of the call reaches a callback; a callback influences a call only through a patch Rust validates - - `begin` returns the dict it was given and `after_success` the response it was given, by identity + - `begin` returns the dict it was given by identity; `after_success` and `delivered` stay the trait's pass-through defaults - Every callback gets a freshly built envelope per event; never cache or share a converted one - Contract values cross into Python only through a fresh conversion per callback - Schema v1 is owned by `litellm-callbacks-v1`; this crate only marshals it - - Everything rejectable is rejected at `register`, never during a call + - Everything rejectable is rejected at `register`, never during a call; the snapshot only parses and the lifecycle never re-validates a subscriber - Observers cannot affect a call: ordinary exceptions are reported and swallowed at every event, awaited or not; an interceptor failure fails the call - - A cancellation (anything that is not a `PyException`) is returned at once, with no further dispatch and no report + - A cancellation is returned at once, with no further dispatch and no report - Exactly one terminal envelope, last; none after a cancellation - In a sync call no step suspends: handler selection happens once, at snapshot, from what registration recorded - Every suspension is awaited inline in the caller's task; never `into_future`, never spawn - The subscriber snapshot is taken once per call before the driver starts; later registrations do not reach that call + - `V1PythonLifecycle::subscribed` is the only constructor callers get; it returns `None` for an empty snapshot, so an unobserved call builds no envelope and projects no response + - This lifecycle may run after another one in a `LifecycleChain`; it observes the arguments and the wire as that one left them and never assumes it is alone - `close` is idempotent; `traverse` visits every retained Python edge and calls no Python - This crate has none of `callbacks-legacy-python`'s obligations and must not grow them; shared behaviour belongs to `PythonLifecycle`, not to either adapter diff --git a/litellm-rust/crates/callbacks-v1-python/src/adapter.rs b/litellm-rust/crates/callbacks-v1-python/src/adapter.rs index 5769783ff96..d834d2c69b9 100644 --- a/litellm-rust/crates/callbacks-v1-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-v1-python/src/adapter.rs @@ -1,11 +1,15 @@ +//! The v1 lifecycle: each step builds one envelope, hands every subscribed observer a +//! fresh conversion of it, and, before the request is sent, folds every interceptor's +//! patch into the wire request. + use litellm_callbacks_v1::{ - CallFacts, Envelope, ErrorFacts, Event, RequestFacts, SCHEMA_V1, Sequencer, WirePatch, apply, + CallFacts, Envelope, ErrorFacts, Event, RequestFacts, Sequencer, WirePatch, apply, }; -use litellm_host::event::{MachineEvent, RequestContext, Timing, WireRequest}; +use litellm_host::event::{MachineEvent, RequestContext, WireRequest}; use litellm_host_python::{ - LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, missing_state, to_py, + LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py, }; -use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::exceptions::PyValueError; use pyo3::gc::{PyTraverseError, PyVisit}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyString}; @@ -13,26 +17,26 @@ use serde_json::{Map, Value}; use crate::call::V1PythonSurface; use crate::python::V1Python; -use crate::registry::{Handler, Subscriber}; +use crate::subscribers::Subscriber; -enum Then { +/// What a step hands back to the driver once every observer of its envelope has run. +enum Continuation { Done, Arguments(Py), Wire(Box), } +/// The suspended step; `index` is the subscriber whose awaitable the driver is awaiting. enum Pending { Observe { envelope: Envelope, - next: usize, - then: Then, - awaiting: String, + index: usize, + then: Continuation, }, Intercept { wire: Box, context: RequestContext, - next: usize, - awaiting: String, + index: usize, }, } @@ -40,18 +44,13 @@ pub struct V1PythonLifecycle { surface: V1PythonSurface, subscribers: Vec, asynchronous: bool, - start_time: f64, sequencer: Option, streamed: bool, pending: Option, closed: bool, } -fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool { - !error.is_instance_of::(py) -} - -impl Then { +impl Continuation { fn into_step(self) -> LifecycleStep { match self { Self::Done => LifecycleStep::Done, @@ -69,12 +68,15 @@ impl Then { } impl V1PythonLifecycle { - pub fn new(surface: V1PythonSurface, subscribers: Vec, asynchronous: bool) -> Self { + pub(crate) fn new( + surface: V1PythonSurface, + subscribers: Vec, + asynchronous: bool, + ) -> Self { Self { surface, subscribers, asynchronous, - start_time: 0.0, sequencer: None, streamed: false, pending: None, @@ -92,10 +94,11 @@ impl V1PythonLifecycle { fn report( &self, py: Python<'_>, - name: &str, + index: usize, envelope: &Envelope, error: PyErr, ) -> PyResult<()> { + let name = &self.subscribers[index].name; let event: &'static str = envelope.event.kind().into(); match V1Python::Report.call(py, (name, event, error.value(py))) { Err(report_error) if is_cancellation(py, &report_error) => Err(report_error), @@ -103,59 +106,33 @@ impl V1PythonLifecycle { } } - fn observer_at( - &self, - py: Python<'_>, - index: usize, - envelope: &Envelope, - ) -> Option<(String, bool, Py)> { - let subscriber = &self.subscribers[index]; - if subscriber.schema != envelope.schema - || !subscriber.events.contains(&envelope.event.kind()) - { - return None; - } - subscriber.observe.as_ref().map(|handler| { - let asynchronous = matches!(handler, Handler::Async(_)); - ( - subscriber.name.clone(), - asynchronous, - handler.object().clone_ref(py), - ) - }) - } - fn observe( &mut self, py: Python<'_>, envelope: Envelope, from: usize, - then: Then, + then: Continuation, ) -> PyResult { for index in from..self.subscribers.len() { - let Some((name, asynchronous, handler)) = self.observer_at(py, index, &envelope) else { + let Some(handler) = self.subscribers[index] + .observer(envelope.event.kind()) + .map(|handler| handler.clone_ref(py)) + else { continue; }; - let argument = match to_py(py, &envelope) { - Ok(argument) => argument, - Err(error) => { - self.report(py, &name, &envelope, error)?; - continue; - } - }; - match handler.bind(py).call1((argument,)) { - Ok(awaitable) if asynchronous => { + let outcome = to_py(py, &envelope).and_then(|argument| handler.call(py, argument)); + match outcome { + Ok(awaitable) if handler.is_async() => { self.pending = Some(Pending::Observe { envelope, - next: index + 1, + index, then, - awaiting: name, }); return Ok(LifecycleStep::Await(awaitable.unbind())); } Ok(_) => {} Err(error) if is_cancellation(py, &error) => return Err(error), - Err(error) => self.report(py, &name, &envelope, error)?, + Err(error) => self.report(py, index, &envelope, error)?, } } Ok(then.into_step()) @@ -164,7 +141,7 @@ impl V1PythonLifecycle { fn apply_patch( &self, py: Python<'_>, - name: &str, + index: usize, wire: WireRequest, result: Py, ) -> PyResult { @@ -173,25 +150,11 @@ impl V1PythonLifecycle { } else { from_py::(result.bind(py))? }; + let name = &self.subscribers[index].name; apply(wire, patch) .map_err(|error| PyValueError::new_err(format!("callback {name}: {error}"))) } - fn interceptor_at(&self, py: Python<'_>, index: usize) -> Option<(String, bool, Py)> { - let subscriber = &self.subscribers[index]; - if subscriber.schema != SCHEMA_V1 { - return None; - } - subscriber.intercept.as_ref().map(|handler| { - let asynchronous = matches!(handler, Handler::Async(_)); - ( - subscriber.name.clone(), - asynchronous, - handler.object().clone_ref(py), - ) - }) - } - fn intercept( &mut self, py: Python<'_>, @@ -201,29 +164,28 @@ impl V1PythonLifecycle { ) -> PyResult { let mut current = wire; for index in from..self.subscribers.len() { - let Some((name, asynchronous, handler)) = self.interceptor_at(py, index) else { + let Some(handler) = self.subscribers[index] + .intercept + .as_ref() + .map(|handler| handler.clone_ref(py)) + else { continue; }; let request = to_py(py, &RequestFacts::new(¤t, &context))?; - match handler.bind(py).call1((request,)) { - Ok(awaitable) if asynchronous => { - self.pending = Some(Pending::Intercept { - wire: current, - context, - next: index + 1, - awaiting: name, - }); - return Ok(LifecycleStep::Await(awaitable.unbind())); - } - Ok(result) => { - current = Box::new(self.apply_patch(py, &name, *current, result.unbind())?); - } - Err(error) => return Err(error), + let result = handler.call(py, request)?; + if handler.is_async() { + self.pending = Some(Pending::Intercept { + wire: current, + context, + index, + }); + return Ok(LifecycleStep::Await(result.unbind())); } + current = Box::new(self.apply_patch(py, index, *current, result.unbind())?); } let envelope = self.envelope(Event::RequestSending(RequestFacts::new(¤t, &context)))?; - self.observe(py, envelope, 0, Then::Wire(current)) + self.observe(py, envelope, 0, Continuation::Wire(current)) } fn metadata(arguments: &Bound<'_, PyDict>) -> PyResult<(Map, Vec)> { @@ -280,18 +242,18 @@ impl PythonLifecycle for V1PythonLifecycle { &mut self, py: Python<'_>, arguments: Py, - _started_at: f64, + start_time: f64, ) -> PyResult { let call_id = Self::call_id(py, arguments.bind(py))?; let (metadata, metadata_dropped) = Self::metadata(arguments.bind(py))?; self.sequencer = Some(Sequencer::new(call_id, self.surface.call_type)); let envelope = self.envelope(Event::CallStarted(CallFacts { - start_time: self.start_time, + start_time, asynchronous: self.asynchronous, metadata, metadata_dropped, }))?; - self.observe(py, envelope, 0, Then::Arguments(arguments)) + self.observe(py, envelope, 0, Continuation::Arguments(arguments)) } fn before_send( @@ -303,35 +265,20 @@ impl PythonLifecycle for V1PythonLifecycle { self.intercept(py, wire, context.clone(), 0) } - fn after_success( - &mut self, - _py: Python<'_>, - response: Py, - _timing: Timing, - ) -> PyResult { - Ok(LifecycleStep::Response(response)) - } - fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { match event { - LifecycleEvent::Started { start_time } => { - self.start_time = start_time; - Ok(LifecycleStep::Done) - } LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => { let envelope = self.envelope(Event::ResponseReceived { body: raw.body.clone(), })?; - self.observe(py, envelope, 0, Then::Done) + self.observe(py, envelope, 0, Continuation::Done) } LifecycleEvent::Succeeded { timing, response } => { - let projection = V1Python::ProjectResponse.call(py, (response,)); - let (response, response_error) = match projection { - Ok(value) => match from_py::(&value) { - Ok(response) => (response, None), - Err(error) if is_cancellation(py, &error) => return Err(error), - Err(error) => (Value::Null, Some(error.to_string())), - }, + let projected = V1Python::ProjectResponse + .call(py, (response,)) + .and_then(|value| from_py::(&value)); + let (response, response_error) = match projected { + Ok(response) => (response, None), Err(error) if is_cancellation(py, &error) => return Err(error), Err(error) => (Value::Null, Some(error.to_string())), }; @@ -341,7 +288,7 @@ impl PythonLifecycle for V1PythonLifecycle { response, response_error, })?; - self.observe(py, envelope, 0, Then::Done) + self.observe(py, envelope, 0, Continuation::Done) } LifecycleEvent::Failed { timing, @@ -355,7 +302,7 @@ impl PythonLifecycle for V1PythonLifecycle { origin: origin.into(), error: facts, })?; - self.observe(py, envelope, 0, Then::Done) + self.observe(py, envelope, 0, Continuation::Done) } } } @@ -365,35 +312,28 @@ impl PythonLifecycle for V1PythonLifecycle { Ok(()) } - fn delivered(&mut self, _py: Python<'_>, _chunk: &Py) -> PyResult<()> { - Ok(()) - } - fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult { match self.pending.take().ok_or_else(missing_state)? { Pending::Observe { envelope, - next, + index, then, - awaiting, } => { if let Err(error) = result { if is_cancellation(py, &error) { return Err(error); } - self.report(py, &awaiting, &envelope, error)?; + self.report(py, index, &envelope, error)?; } - self.observe(py, envelope, next, then) + self.observe(py, envelope, index + 1, then) } Pending::Intercept { wire, context, - next, - awaiting, + index, } => { - let value = result?; - let patched = self.apply_patch(py, &awaiting, *wire, value)?; - self.intercept(py, Box::new(patched), context, next) + let patched = self.apply_patch(py, index, *wire, result?)?; + self.intercept(py, Box::new(patched), context, index + 1) } } } @@ -409,12 +349,7 @@ impl PythonLifecycle for V1PythonLifecycle { fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { for subscriber in &self.subscribers { - if let Some(handler) = &subscriber.observe { - handler.traverse(visit)?; - } - if let Some(handler) = &subscriber.intercept { - handler.traverse(visit)?; - } + subscriber.traverse(visit)?; } match &self.pending { Some(Pending::Observe { then, .. }) => then.traverse(visit), diff --git a/litellm-rust/crates/callbacks-v1-python/src/call.rs b/litellm-rust/crates/callbacks-v1-python/src/call.rs index 2b2c49e8f4a..5826b993446 100644 --- a/litellm-rust/crates/callbacks-v1-python/src/call.rs +++ b/litellm-rust/crates/callbacks-v1-python/src/call.rs @@ -1,36 +1,25 @@ -use litellm_host::{machine::Machine, route::Route}; -use litellm_host_python::{RouteHost, run_call}; +//! What a route tells the v1 contract about itself, and the one constructor callers get. + use pyo3::prelude::*; -use pyo3::types::PyDict; use crate::adapter::V1PythonLifecycle; -use crate::registry::snapshot; +use crate::subscribers::snapshot; #[derive(Clone, Copy, Debug)] pub struct V1PythonSurface { pub call_type: &'static str, } -pub fn run_v1_python_call( - py: Python<'_>, - surface: V1PythonSurface, - kwargs: &Bound<'_, PyDict>, - machine: M, - route: H, - asynchronous: bool, -) -> PyResult> -where - H: RouteHost + 'static, - M: Machine::Response> + 'static, -{ - let subscribers = snapshot(py, asynchronous)?; - let arguments = kwargs.clone().unbind(); - run_call( - py, - machine, - route, - Box::new(V1PythonLifecycle::new(surface, subscribers, asynchronous)), - arguments, - asynchronous, - ) +impl V1PythonLifecycle { + /// The lifecycle for one call, over the subscribers registered right now; later + /// registrations do not reach that call. `None` when nothing is subscribed, so an + /// unobserved call builds no envelope at all. + pub fn subscribed( + py: Python<'_>, + surface: V1PythonSurface, + asynchronous: bool, + ) -> PyResult> { + let subscribers = snapshot(py, asynchronous)?; + Ok((!subscribers.is_empty()).then(|| Self::new(surface, subscribers, asynchronous))) + } } diff --git a/litellm-rust/crates/callbacks-v1-python/src/lib.rs b/litellm-rust/crates/callbacks-v1-python/src/lib.rs index 2675136563b..665034a07a5 100644 --- a/litellm-rust/crates/callbacks-v1-python/src/lib.rs +++ b/litellm-rust/crates/callbacks-v1-python/src/lib.rs @@ -1,9 +1,15 @@ +//! The Python adapter of the v1 callback contract: the per-call subscriber snapshot and +//! one [`PythonLifecycle`](litellm_host_python::PythonLifecycle) that marshals +//! `litellm-callbacks-v1` envelopes and patches to Python handlers. The contract itself +//! lives in `litellm-callbacks-v1`. + mod adapter; mod call; mod python; -mod registry; +mod subscribers; -pub use call::{V1PythonSurface, run_v1_python_call}; +pub use adapter::V1PythonLifecycle; +pub use call::V1PythonSurface; #[cfg(test)] #[path = "../tests/lifecycle.rs"] diff --git a/litellm-rust/crates/callbacks-v1-python/src/python.rs b/litellm-rust/crates/callbacks-v1-python/src/python.rs index 49961945fc1..0fd75cab0d2 100644 --- a/litellm-rust/crates/callbacks-v1-python/src/python.rs +++ b/litellm-rust/crates/callbacks-v1-python/src/python.rs @@ -1,3 +1,6 @@ +//! The Python contract of this crate: every function of +//! `litellm/rust_bridge/callbacks_v1_python.py` that Rust calls, by name. + use pyo3::prelude::*; use strum::{IntoStaticStr, VariantArray}; @@ -35,17 +38,19 @@ mod tests { use strum::VariantArray; use super::V1Python; + use crate::test_support::PYTHON_CONTRACT; #[test] fn every_function_is_in_the_python_contract() { let contract: serde_json::Map = - serde_json::from_str(include_str!("../python_contract.json")).unwrap(); + serde_json::from_str(PYTHON_CONTRACT).unwrap(); let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect(); - let called: BTreeSet<&str> = V1Python::VARIANTS + let called: Vec<&str> = V1Python::VARIANTS .iter() .copied() .map(V1Python::name) .collect(); - assert_eq!(called, declared); + assert_eq!(called.len(), declared.len(), "a function is listed twice"); + assert_eq!(called.into_iter().collect::>(), declared); } } diff --git a/litellm-rust/crates/callbacks-v1-python/src/registry.rs b/litellm-rust/crates/callbacks-v1-python/src/registry.rs deleted file mode 100644 index 9e63b42937c..00000000000 --- a/litellm-rust/crates/callbacks-v1-python/src/registry.rs +++ /dev/null @@ -1,96 +0,0 @@ -use std::collections::BTreeSet; - -use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, SCHEMA_V1, select_handler}; -use pyo3::exceptions::PyValueError; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; - -use crate::python::V1Python; - -pub enum Handler { - Sync(Py), - Async(Py), -} - -pub struct Subscriber { - pub name: String, - pub schema: u32, - pub events: BTreeSet, - pub observe: Option, - pub intercept: Option, -} - -fn selected_handler( - subscriber: &Bound<'_, PyAny>, - asynchronous: bool, - sync_name: &str, - async_name: &str, -) -> PyResult> { - let sync = subscriber.getattr(sync_name)?; - let asynchronous_handler = subscriber.getattr(async_name)?; - let selection = select_handler( - if asynchronous { - ExecutionMode::Async - } else { - ExecutionMode::Sync - }, - !sync.is_none(), - !asynchronous_handler.is_none(), - ); - Ok(match selection { - HandlerSelection::Sync => Some(Handler::Sync(sync.unbind())), - HandlerSelection::Async => Some(Handler::Async(asynchronous_handler.unbind())), - HandlerSelection::Skip => None, - }) -} - -pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult> { - V1Python::Snapshot - .call(py, ())? - .try_iter()? - .map(|item| { - let subscriber = item?; - let schema = subscriber.getattr("schema")?.extract::()?; - if schema != SCHEMA_V1 { - return Err(PyValueError::new_err(format!( - "unsupported callback schema: {schema}" - ))); - } - let event_names = subscriber - .getattr("events")? - .extract::>()?; - let events = event_names - .into_iter() - .map(|name| { - name.parse::().map_err(|_| { - PyValueError::new_err(format!("unknown callback event: {name}")) - }) - }) - .collect::>>()?; - Ok(Subscriber { - name: subscriber.getattr("name")?.extract()?, - schema, - events, - observe: selected_handler(&subscriber, asynchronous, "on_event", "async_on_event")?, - intercept: selected_handler( - &subscriber, - asynchronous, - "before_send", - "async_before_send", - )?, - }) - }) - .collect() -} - -impl Handler { - pub fn object(&self) -> &Py { - match self { - Self::Sync(value) | Self::Async(value) => value, - } - } - - pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { - visit.call(self.object()) - } -} diff --git a/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs b/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs new file mode 100644 index 00000000000..a63e3aa842f --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs @@ -0,0 +1,113 @@ +//! The per-call subscriber snapshot: what Python's `register` recorded, read once, with +//! one handler chosen per role for the call's execution mode. `register` has already +//! rejected everything rejectable, so this only parses. + +use std::collections::BTreeSet; + +use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, select_handler}; +use pyo3::exceptions::PyValueError; +use pyo3::gc::{PyTraverseError, PyVisit}; +use pyo3::prelude::*; + +use crate::python::V1Python; + +/// One callable of a subscriber, tagged with whether calling it returns an awaitable. +pub enum Handler { + Sync(Py), + Async(Py), +} + +pub struct Subscriber { + pub name: String, + pub events: BTreeSet, + /// `on_event` or `async_on_event`: cannot affect the call. + pub observe: Option, + /// `before_send` or `async_before_send`: returns a patch; its failure fails the call. + pub intercept: Option, +} + +fn selected_handler( + subscriber: &Bound<'_, PyAny>, + mode: ExecutionMode, + sync_name: &str, + async_name: &str, +) -> PyResult> { + let sync = subscriber.getattr(sync_name)?; + let asynchronous = subscriber.getattr(async_name)?; + Ok( + match select_handler(mode, !sync.is_none(), !asynchronous.is_none()) { + HandlerSelection::Sync => Some(Handler::Sync(sync.unbind())), + HandlerSelection::Async => Some(Handler::Async(asynchronous.unbind())), + HandlerSelection::Skip => None, + }, + ) +} + +pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult> { + let mode = if asynchronous { + ExecutionMode::Async + } else { + ExecutionMode::Sync + }; + V1Python::Snapshot + .call(py, ())? + .try_iter()? + .map(|item| { + let subscriber = item?; + let events = subscriber + .getattr("events")? + .extract::>()? + .into_iter() + .map(|name| { + name.parse::().map_err(|_| { + PyValueError::new_err(format!("unknown callback event: {name}")) + }) + }) + .collect::>>()?; + Ok(Subscriber { + name: subscriber.getattr("name")?.extract()?, + events, + observe: selected_handler(&subscriber, mode, "on_event", "async_on_event")?, + intercept: selected_handler(&subscriber, mode, "before_send", "async_before_send")?, + }) + }) + .collect() +} + +impl Subscriber { + pub fn observer(&self, kind: EventKind) -> Option<&Handler> { + self.observe + .as_ref() + .filter(|_| self.events.contains(&kind)) + } + + pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + for handler in self.observe.iter().chain(&self.intercept) { + visit.call(handler.object())?; + } + Ok(()) + } +} + +impl Handler { + fn object(&self) -> &Py { + match self { + Self::Sync(value) | Self::Async(value) => value, + } + } + + pub fn is_async(&self) -> bool { + matches!(self, Self::Async(_)) + } + + pub fn clone_ref(&self, py: Python<'_>) -> Self { + match self { + Self::Sync(value) => Self::Sync(value.clone_ref(py)), + Self::Async(value) => Self::Async(value.clone_ref(py)), + } + } + + pub fn call<'py>(&self, py: Python<'py>, argument: Py) -> PyResult> { + self.object().bind(py).call1((argument,)) + } +} diff --git a/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs b/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs index e5f97286e91..6d961e4ca9c 100644 --- a/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs +++ b/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs @@ -10,9 +10,9 @@ use serde_json::{Value, json}; use crate::adapter::V1PythonLifecycle; use crate::call::V1PythonSurface; -use crate::registry::{Handler, Subscriber}; +use crate::subscribers::{Handler, Subscriber}; use crate::test_support::{local, namespace}; -use litellm_callbacks_v1::{EventKind, SCHEMA_V1}; +use litellm_callbacks_v1::EventKind; const TIMING: Timing = Timing { start_time: 10.0, @@ -38,7 +38,6 @@ fn subscriber( }; Subscriber { name: name.to_string(), - schema: SCHEMA_V1, events: events.iter().copied().collect::>(), observe: handler(observe), intercept: handler(intercept), @@ -53,15 +52,6 @@ fn lifecycle(subscribers: Vec, asynchronous: bool) -> V1PythonLifecy ) } -fn started(adapter: &mut V1PythonLifecycle, py: Python<'_>) { - assert!(matches!( - adapter - .emit(py, LifecycleEvent::Started { start_time: 10.0 }) - .unwrap(), - LifecycleStep::Done - )); -} - fn context(secret: &str) -> RequestContext { RequestContext { model: "model".to_string(), @@ -111,7 +101,6 @@ response = {'id': 'response'} None, ); let mut adapter = lifecycle(vec![observer], false); - started(&mut adapter, py); let arguments = PyDict::new(py); arguments.set_item("litellm_call_id", "call-123").unwrap(); let LifecycleStep::Arguments(returned) = @@ -202,7 +191,6 @@ def record(event): subscriber(&locals, "recorder", &event, Some(("record", false)), None), ]; let mut adapter = lifecycle(subscribers, false); - started(&mut adapter, py); adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(); let LifecycleStep::Wire(sent) = adapter .before_send(py, Box::new(wire("secret")), &context("secret")) @@ -248,7 +236,6 @@ def record(event): ), ]; let mut adapter = lifecycle(subscribers, false); - started(&mut adapter, py); assert!(matches!( adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(), LifecycleStep::Arguments(_) @@ -299,7 +286,6 @@ def record(event): ), ]; let mut adapter = lifecycle(subscribers, false); - started(&mut adapter, py); let error = match adapter.begin(py, PyDict::new(py).unbind(), 10.0) { Err(error) => error, Ok(_) => panic!("cancellation was swallowed"), @@ -337,7 +323,6 @@ def second(event): ), ]; let mut adapter = lifecycle(subscribers, true); - started(&mut adapter, py); let arguments = PyDict::new(py).unbind(); assert!(matches!( adapter.begin(py, arguments.clone_ref(py), 10.0).unwrap(), @@ -387,7 +372,6 @@ def observe(event): ), ]; let mut adapter = lifecycle(subscribers, false); - started(&mut adapter, py); adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(); let LifecycleStep::Wire(sent) = adapter .before_send(py, Box::new(wire("secret")), &context("secret")) @@ -440,7 +424,6 @@ def observe(event): ), ]; let mut adapter = lifecycle(subscribers, false); - started(&mut adapter, py); adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(); let error = match adapter.before_send(py, Box::new(wire("secret")), &context("secret")) { Err(error) => error, @@ -474,7 +457,6 @@ response = Broken() None, ); let mut adapter = lifecycle(vec![observer], false); - started(&mut adapter, py); adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(); adapter.opened(py).unwrap(); let response = local(&locals, "response").unbind(); diff --git a/litellm-rust/crates/callbacks-v1/AGENTS.md b/litellm-rust/crates/callbacks-v1/AGENTS.md index 1625f4cb018..2b989ffa1e6 100644 --- a/litellm-rust/crates/callbacks-v1/AGENTS.md +++ b/litellm-rust/crates/callbacks-v1/AGENTS.md @@ -1,8 +1,17 @@ - Target invariants, not completion claims -- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction and validated wire patches +- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction, handler selection and validated wire patches - This crate may depend on `litellm-host`, but never on `litellm-host-python`, PyO3 or a runtime registry - - `golden/v1` pins every serialized envelope shape + - `golden/v1` pins every serialized envelope shape; every `EventKind` has a golden file by construction, and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are tested against those files +- Terms (shared ones are in `host-python/AGENTS.md`) + - Subscriber: one registered consumer of the contract: a name, a schema, the event kinds it wants and its handlers + - Handler: one callable of a subscriber; an observer receives envelopes and cannot affect the call; an interceptor receives `RequestFacts` and returns a patch + - Handler selection (`select_handler`): an async call prefers the async handler and falls back to the sync one; a sync call never selects an async handler + - Event: one thing that happened in a call; `EventKind` is its dotted name (`call.started`, `request.sending`, `response.received`, `call.succeeded`, `call.failed`), the serde tag and the subscription name at once + - Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; `Sequencer` numbers a call's envelopes from zero + - Facts (`CallFacts`, `RequestFacts`, `TimingFacts`, `ErrorFacts`): a redacted, serializable projection of runtime state; never a live object + - Patch (`WirePatch`): an interceptor's edit of the wire: header `remove`, then header `set`, optional `body` replacement; `apply` validates it + - Redaction: credential headers by name (`CREDENTIAL_HEADERS`) and optional params by the route's secret fields become `[REDACTED]`; shape never changes - No runtime object of a call reaches a callback contract value -- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction +- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction; its fields are private so no other crate can build one - A patch cannot change the URL or set or remove a credential header - Schema v1 is additive-only; a rename, removal or retype requires a new contract version diff --git a/litellm-rust/crates/callbacks-v1/src/envelope.rs b/litellm-rust/crates/callbacks-v1/src/envelope.rs index f79d4d9027b..7f78b999a32 100644 --- a/litellm-rust/crates/callbacks-v1/src/envelope.rs +++ b/litellm-rust/crates/callbacks-v1/src/envelope.rs @@ -1,3 +1,7 @@ +//! The v1 envelope: what a subscriber receives. An [`Envelope`] is the schema version, the +//! call's identity, a sequence number and one [`Event`]; every `*Facts` type is a +//! redacted, serializable projection that never holds a runtime object of the call. + use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest}; use serde::Serialize; use serde_json::{Map, Value}; @@ -75,14 +79,16 @@ pub struct CallFacts { pub metadata_dropped: Vec, } +/// The request as a subscriber may see it. The fields are private so that +/// [`RequestFacts::new`], which redacts, is the only way to build one. #[derive(Clone, Debug, PartialEq, Serialize)] pub struct RequestFacts { - pub model: String, - pub custom_llm_provider: String, - pub optional_params: Value, - pub url: String, - pub headers: Vec<(String, String)>, - pub body: Value, + model: String, + custom_llm_provider: String, + optional_params: Value, + url: String, + headers: Vec<(String, String)>, + body: Value, } impl RequestFacts { @@ -243,21 +249,26 @@ mod tests { } } - #[rstest::rstest] - #[case(EventKind::CallStarted, 0, include_str!("../golden/v1/call.started.json"))] - #[case(EventKind::RequestSending, 1, include_str!("../golden/v1/request.sending.json"))] - #[case(EventKind::ResponseReceived, 2, include_str!("../golden/v1/response.received.json"))] - #[case(EventKind::CallSucceeded, 3, include_str!("../golden/v1/call.succeeded.json"))] - #[case(EventKind::CallFailed, 4, include_str!("../golden/v1/call.failed.json"))] - fn serialized_v1_envelopes_match_golden_contract( - #[case] kind: EventKind, - #[case] seq: u32, - #[case] expected: &str, - ) { - assert_eq!( - serde_json::to_value(golden(kind, seq)).unwrap(), - serde_json::from_str::(expected).unwrap() - ); + /// Exhaustive on purpose: a new event kind does not compile without a golden file, + /// and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are pinned to the golden files. + fn golden_json(kind: EventKind) -> &'static str { + match kind { + EventKind::CallStarted => include_str!("../golden/v1/call.started.json"), + EventKind::RequestSending => include_str!("../golden/v1/request.sending.json"), + EventKind::ResponseReceived => include_str!("../golden/v1/response.received.json"), + EventKind::CallSucceeded => include_str!("../golden/v1/call.succeeded.json"), + EventKind::CallFailed => include_str!("../golden/v1/call.failed.json"), + } + } + + #[test] + fn serialized_v1_envelopes_match_golden_contract() { + for (seq, &kind) in EventKind::VARIANTS.iter().enumerate() { + assert_eq!( + serde_json::to_value(golden(kind, seq as u32)).unwrap(), + serde_json::from_str::(golden_json(kind)).unwrap() + ); + } } #[test] diff --git a/litellm-rust/crates/callbacks-v1/src/lib.rs b/litellm-rust/crates/callbacks-v1/src/lib.rs index 28f03482693..2d5b27df591 100644 --- a/litellm-rust/crates/callbacks-v1/src/lib.rs +++ b/litellm-rust/crates/callbacks-v1/src/lib.rs @@ -1,3 +1,7 @@ +//! The runtime-neutral v1 callback contract: envelopes and their sequencing, credential +//! redaction, handler selection and validated wire patches. No PyO3 and no registry; a +//! language host marshals these values to its own subscribers. + mod envelope; mod patch; mod policy; diff --git a/litellm-rust/crates/callbacks-v1/src/patch.rs b/litellm-rust/crates/callbacks-v1/src/patch.rs index dd13cb5e7b1..50505536885 100644 --- a/litellm-rust/crates/callbacks-v1/src/patch.rs +++ b/litellm-rust/crates/callbacks-v1/src/patch.rs @@ -1,3 +1,7 @@ +//! The v1 patch: the only way an interceptor changes a request. A [`WirePatch`] removes +//! and sets headers and may replace the body; it cannot name the URL, and [`apply`] +//! rejects any credential header. + use litellm_host::event::WireRequest; use serde::Deserialize; use serde_json::Value; diff --git a/litellm-rust/crates/callbacks-v1/src/policy.rs b/litellm-rust/crates/callbacks-v1/src/policy.rs index 1386facc620..07ff392af05 100644 --- a/litellm-rust/crates/callbacks-v1/src/policy.rs +++ b/litellm-rust/crates/callbacks-v1/src/policy.rs @@ -1,3 +1,6 @@ +//! Which of a subscriber's handlers a call uses: an async call prefers the async handler +//! and falls back to the sync one; a sync call never selects an async handler. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ExecutionMode { Sync, diff --git a/litellm-rust/crates/callbacks-v1/src/redact.rs b/litellm-rust/crates/callbacks-v1/src/redact.rs index f7c1dbaf59b..dacfc304086 100644 --- a/litellm-rust/crates/callbacks-v1/src/redact.rs +++ b/litellm-rust/crates/callbacks-v1/src/redact.rs @@ -1,3 +1,6 @@ +//! The contract's credential redaction: credential headers by name, optional params by +//! the route's secret fields. Redaction replaces values and never changes shape. + use serde_json::{Map, Value}; pub const REDACTED: &str = "[REDACTED]"; diff --git a/litellm-rust/crates/host-python/AGENTS.md b/litellm-rust/crates/host-python/AGENTS.md index 5aca13eeb18..ba2142b7412 100644 --- a/litellm-rust/crates/host-python/AGENTS.md +++ b/litellm-rust/crates/host-python/AGENTS.md @@ -5,6 +5,26 @@ - `RouteHost::invoke` receives the keyword view the adapter's `begin` returned, not the caller's dict; a route host that projects from it inherits that adapter's rewrites (for the legacy adapter: setup, deployment hooks, credential inheritance) - A native failure, including one a host op returns as `InvokeError::Native`, is classified exactly once through the route's `classify`; a Python exception raised inside the call, and a failure in `begin` or `after_success`, is raised as is - A failing `classify` is raised with the native error's text as its `__context__`, never swallowed +- Terms; the callback crates build on these and never redefine them + - Machine: the native call; it never calls out, it yields a `HostOp` (`Route`, `BeforeSend`, `Emit`, `Open`, `Deliver`) and waits for the `HostResult` + - Host: whatever answers those ops; the Python host is two capabilities the driver composes: `RouteHost` (route ops, argument and response projection) and `PythonLifecycle` (callbacks) + - Host language = callback language: a lifecycle speaks `Py<...>` and `PyErr` directly; another host language gets its own crate of this shape and reuses only `litellm-host` and `litellm-callbacks-v1` + - Driver: runs one call; owns the machine, one route host and exactly one lifecycle + - Lifecycle: one implementation of `PythonLifecycle`, one callback contract's view of a call; chain: several lifecycles behind one, in a fixed order + - Step: one driver-to-lifecycle method call, answered with a `LifecycleStep`; a threading step (`begin`, `before_send`, `after_success`) returns the value the call continues with and defaults to returning what it was given; an observing step (`emit`, `opened`, `delivered`) returns nothing to the call + - Suspension: a step answered with `Await`; the driver awaits it in the caller's task and continues the same step through `resume` + - Terminal event: `Succeeded` or `Failed`, exactly once, last + - Cancellation: a raised `BaseException` that is not an `Exception`; `is_cancellation` is the only definition; never reported, never swallowed, never followed by dispatch + - Keyword view: the kwargs dict as rewritten so far; `begin` threads it and the route projects from the result + - Wire: the `WireRequest` (URL, headers, body) about to be sent; `before_send` threads it + - Surface: the static facts a route declares about itself to a contract (`call_type`, ...) + - Python contract: a crate's `python.rs` enum, its `python_contract.json` and the one `litellm/rust_bridge/*.py` module behind them, tested from both sides +- `PythonLifecycle` is the whole callback boundary: the driver holds exactly one and never learns how many contracts are behind it + - `LifecycleChain` composes several in a fixed order: `begin`, `before_send` and `after_success` thread their value forward, events reach each in turn, any error ends the step with no further dispatch + - `begin` carries the call's start time and is the first thing a lifecycle hears of a call; there is no separate started event + - A lifecycle whose `begin` was never reached receives nothing of that call; the terminal event goes to the ones `begin` reached + - A suspension resumes the lifecycle that suspended, then continues the same step with the ones after it; a retained failure is the raised exception object itself + - Ordering policy (which contract runs first) belongs to whoever builds the chain, never to this crate - Use standard PyO3 ownership and conversion APIs - Prefer `Bound<'py, T>` for attached operations/results, `Py` for retention; binding/unbinding does not copy payloads - Use `pythonize` for selected Serde data, never a JSON-text round trip; share conversion with `Pythonized` diff --git a/litellm-rust/crates/host-python/src/adapter.rs b/litellm-rust/crates/host-python/src/adapter.rs index 3a4cb49be4d..7f61e759536 100644 --- a/litellm-rust/crates/host-python/src/adapter.rs +++ b/litellm-rust/crates/host-python/src/adapter.rs @@ -1,15 +1,22 @@ use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest}; use litellm_host::route::Route; -use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::{PyException, PyRuntimeError}; use pyo3::gc::{PyTraverseError, PyVisit}; use pyo3::prelude::*; use pyo3::types::PyDict; +/// Whether an error ends the call outright: anything that is not a `PyException`, such +/// as `CancelledError` or `KeyboardInterrupt`. A cancellation is never reported, never +/// swallowed and never followed by further dispatch. +pub fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool { + !error.is_instance_of::(py) +} + pub fn missing_state() -> PyErr { PyRuntimeError::new_err("missing native call state") } -/// What an adapter step produced: either the value the driver asked for, or a Python +/// What a lifecycle step produced: either the value the driver asked for, or a Python /// awaitable the driver hands back to the caller's task before asking again. pub enum LifecycleStep { Await(Py), @@ -19,12 +26,10 @@ pub enum LifecycleStep { Done, } -/// What a lifecycle observes: the driver's start, the machine's own events, and one -/// terminal event carrying the public value the caller receives. +/// What a lifecycle observes: the machine's own events, and one terminal event +/// carrying the public value the caller receives. +#[derive(Clone, Copy)] pub enum LifecycleEvent<'a> { - Started { - start_time: f64, - }, Machine(&'a MachineEvent), Succeeded { timing: Timing, @@ -37,46 +42,73 @@ pub enum LifecycleEvent<'a> { }, } -/// One consumer of a call's lifecycle on the Python side. The driver calls the steps in -/// order: `begin` before the machine starts, `before_send` and `emit` while it runs, -/// `after_success` and one terminal `emit` after it completes. Whenever a step returns -/// [`LifecycleStep::Await`], the driver awaits it in the caller's task and continues the -/// same step through `resume`. +/// One consumer of a call's lifecycle on the Python side: the callback capability of the +/// Python host, next to [`RouteHost`], the route capability. +/// +/// The steps come in two kinds. A threading step receives a value and returns the value +/// the call continues with, so a lifecycle may rewrite it; each defaults to returning +/// what it was given. An observing step receives an event or a notice and returns +/// nothing to the call. Two steps answer an operation the machine raised +/// ([`HostOp`](litellm_host::host::HostOp)); the rest the driver originates around it: +/// +/// | step | kind | raised by | +/// |---|---|---| +/// | `begin` | threads the keyword view | driver, before the machine starts | +/// | `before_send` | threads the wire request | machine, `HostOp::BeforeSend` | +/// | `emit(Machine)` | observes | machine, `HostOp::Emit` | +/// | `opened`, `delivered` | observe | driver, answering `HostOp::Open` and `Deliver` | +/// | `after_success` | threads the public response | driver, after the machine completes | +/// | `emit(Succeeded \| Failed)` | observes | driver, exactly once, last | +/// +/// Whenever a step returns [`LifecycleStep::Await`], the driver awaits it in the caller's +/// task and continues the same step through `resume`. /// /// A step that fails with an ordinary exception fails the call with that exception, -/// except on a terminal event, where the adapter is expected to report and swallow its -/// own errors. An exception that is not a `PyException`, such as a cancellation, ends -/// the call without further dispatch. +/// except on a terminal event, where the lifecycle is expected to report and swallow its +/// own errors. A cancellation ([`is_cancellation`]) ends the call without further +/// dispatch. pub trait PythonLifecycle: Send + Sync { + /// `start_time` is the call's start in epoch seconds, the same value the terminal + /// event's `Timing` carries. fn begin( &mut self, - py: Python<'_>, + _py: Python<'_>, arguments: Py, - started_at: f64, - ) -> PyResult; + _start_time: f64, + ) -> PyResult { + Ok(LifecycleStep::Arguments(arguments)) + } fn before_send( &mut self, - py: Python<'_>, + _py: Python<'_>, wire: Box, - context: &RequestContext, - ) -> PyResult; + _context: &RequestContext, + ) -> PyResult { + Ok(LifecycleStep::Wire(wire)) + } fn after_success( &mut self, - py: Python<'_>, + _py: Python<'_>, response: Py, - timing: Timing, - ) -> PyResult; + _timing: Timing, + ) -> PyResult { + Ok(LifecycleStep::Response(response)) + } fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult; /// The call streams and its stream was handed to the caller. The caller is not /// inside an await here, so this step and `delivered` cannot suspend. - fn opened(&mut self, py: Python<'_>) -> PyResult<()>; + fn opened(&mut self, _py: Python<'_>) -> PyResult<()> { + Ok(()) + } /// One chunk of an open stream is about to reach the caller. - fn delivered(&mut self, py: Python<'_>, chunk: &Py) -> PyResult<()>; + fn delivered(&mut self, _py: Python<'_>, _chunk: &Py) -> PyResult<()> { + Ok(()) + } fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult; diff --git a/litellm-rust/crates/host-python/src/chain.rs b/litellm-rust/crates/host-python/src/chain.rs new file mode 100644 index 00000000000..cc817ea11eb --- /dev/null +++ b/litellm-rust/crates/host-python/src/chain.rs @@ -0,0 +1,560 @@ +//! Several lifecycles consuming one call as one [`PythonLifecycle`]. The driver keeps a +//! single consumer; which contracts a call runs under is decided where the chain is +//! built, never by the driver and never by a route. + +use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest}; +use pyo3::exceptions::PyBaseException; +use pyo3::gc::{PyTraverseError, PyVisit}; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +use crate::adapter::{LifecycleEvent, LifecycleStep, PythonLifecycle, missing_state}; + +/// An event kept across a suspension, so the lifecycles after the suspended one still +/// receive it. A failure keeps the raised exception object itself. +enum RetainedEvent { + Machine(MachineEvent), + Succeeded { + timing: Timing, + response: Py, + }, + Failed { + timing: Timing, + origin: FailureOrigin, + error: Py, + }, +} + +/// The step a suspended lifecycle was answering, with what the later lifecycles need to +/// answer it too. The value being threaded stays with the suspended lifecycle. +enum Step { + Begin { start_time: f64 }, + BeforeSend { context: RequestContext }, + AfterSuccess { timing: Timing }, + Emit(RetainedEvent), +} + +struct Suspended { + index: usize, + step: Step, +} + +/// Runs every step through its lifecycles in the order they were given. +/// +/// `begin`, `before_send` and `after_success` thread their value: each lifecycle +/// receives what the one before it returned, so a later lifecycle sees an earlier one's +/// rewrites and never the reverse. Events reach each lifecycle in turn. An error from +/// any lifecycle ends the step at once, with no dispatch to the ones after it. +/// +/// A lifecycle observes a call only once its `begin` was reached: when an earlier +/// lifecycle fails the call in `begin`, the later ones never learn of the call, and the +/// terminal event goes to those that did. +pub struct LifecycleChain { + lifecycles: Vec>, + /// How many lifecycles `begin` has reached. + entered: usize, + pending: Option, +} + +impl RetainedEvent { + fn retain(py: Python<'_>, event: LifecycleEvent<'_>) -> Self { + match event { + LifecycleEvent::Machine(event) => Self::Machine(event.clone()), + LifecycleEvent::Succeeded { timing, response } => Self::Succeeded { + timing, + response: response.clone_ref(py), + }, + LifecycleEvent::Failed { + timing, + origin, + error, + } => Self::Failed { + timing, + origin, + error: error.clone_ref(py).into_value(py), + }, + } + } + + fn emit(&self, py: Python<'_>, lifecycle: &mut dyn PythonLifecycle) -> PyResult { + match self { + Self::Machine(event) => lifecycle.emit(py, LifecycleEvent::Machine(event)), + Self::Succeeded { timing, response } => lifecycle.emit( + py, + LifecycleEvent::Succeeded { + timing: *timing, + response, + }, + ), + Self::Failed { + timing, + origin, + error, + } => { + let error = PyErr::from_value(error.bind(py).clone().into_any()); + lifecycle.emit( + py, + LifecycleEvent::Failed { + timing: *timing, + origin: *origin, + error: &error, + }, + ) + } + } + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + match self { + Self::Succeeded { response, .. } => visit.call(response), + Self::Failed { error, .. } => visit.call(error), + Self::Machine(_) => Ok(()), + } + } +} + +impl LifecycleChain { + pub fn new(lifecycles: Vec>) -> Self { + Self { + lifecycles, + entered: 0, + pending: None, + } + } + + /// How many lifecycles a step reaches: `begin` tries all of them, every later step + /// reaches the ones `begin` reached. + fn audience(&self, step: &Step) -> usize { + match step { + Step::Begin { .. } => self.lifecycles.len(), + Step::BeforeSend { .. } | Step::AfterSuccess { .. } | Step::Emit(_) => self.entered, + } + } + + fn call( + &mut self, + py: Python<'_>, + index: usize, + step: &Step, + carried: LifecycleStep, + ) -> PyResult { + let lifecycle = self.lifecycles[index].as_mut(); + match (step, carried) { + (Step::Begin { start_time }, LifecycleStep::Arguments(arguments)) => { + self.entered = index + 1; + lifecycle.begin(py, arguments, *start_time) + } + (Step::BeforeSend { context }, LifecycleStep::Wire(wire)) => { + lifecycle.before_send(py, wire, context) + } + (Step::AfterSuccess { timing }, LifecycleStep::Response(response)) => { + lifecycle.after_success(py, response, *timing) + } + (Step::Emit(event), LifecycleStep::Done) => event.emit(py, lifecycle), + _ => Err(missing_state()), + } + } + + /// Continues `step` from the lifecycle at `next`, handing each one what the previous + /// produced, until one suspends or the last has answered. + fn proceed( + &mut self, + py: Python<'_>, + mut next: usize, + mut carried: LifecycleStep, + step: Step, + ) -> PyResult { + while next < self.audience(&step) { + carried = match self.call(py, next, &step, carried)? { + LifecycleStep::Await(awaitable) => { + self.pending = Some(Suspended { index: next, step }); + return Ok(LifecycleStep::Await(awaitable)); + } + produced => produced, + }; + next += 1; + } + Ok(carried) + } +} + +impl PythonLifecycle for LifecycleChain { + fn begin( + &mut self, + py: Python<'_>, + arguments: Py, + start_time: f64, + ) -> PyResult { + self.proceed( + py, + 0, + LifecycleStep::Arguments(arguments), + Step::Begin { start_time }, + ) + } + + fn before_send( + &mut self, + py: Python<'_>, + wire: Box, + context: &RequestContext, + ) -> PyResult { + self.proceed( + py, + 0, + LifecycleStep::Wire(wire), + Step::BeforeSend { + context: context.clone(), + }, + ) + } + + fn after_success( + &mut self, + py: Python<'_>, + response: Py, + timing: Timing, + ) -> PyResult { + self.proceed( + py, + 0, + LifecycleStep::Response(response), + Step::AfterSuccess { timing }, + ) + } + + fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { + for index in 0..self.entered { + match self.lifecycles[index].emit(py, event)? { + LifecycleStep::Await(awaitable) => { + self.pending = Some(Suspended { + index, + step: Step::Emit(RetainedEvent::retain(py, event)), + }); + return Ok(LifecycleStep::Await(awaitable)); + } + LifecycleStep::Done => {} + _ => return Err(missing_state()), + } + } + Ok(LifecycleStep::Done) + } + + fn opened(&mut self, py: Python<'_>) -> PyResult<()> { + self.lifecycles[..self.entered] + .iter_mut() + .try_for_each(|lifecycle| lifecycle.opened(py)) + } + + fn delivered(&mut self, py: Python<'_>, chunk: &Py) -> PyResult<()> { + self.lifecycles[..self.entered] + .iter_mut() + .try_for_each(|lifecycle| lifecycle.delivered(py, chunk)) + } + + fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult { + let Suspended { index, step } = self.pending.take().ok_or_else(missing_state)?; + match self.lifecycles[index].resume(py, result)? { + LifecycleStep::Await(awaitable) => { + self.pending = Some(Suspended { index, step }); + Ok(LifecycleStep::Await(awaitable)) + } + produced => self.proceed(py, index + 1, produced, step), + } + } + + fn close(&mut self, py: Python<'_>) { + self.pending = None; + for lifecycle in &mut self.lifecycles { + lifecycle.close(py); + } + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + for lifecycle in &self.lifecycles { + lifecycle.traverse(visit)?; + } + match &self.pending { + Some(Suspended { + step: Step::Emit(event), + .. + }) => event.traverse(visit), + _ => Ok(()), + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use pyo3::exceptions::PyValueError; + + use super::*; + + type Log = Arc>>; + + #[derive(Clone, Copy, PartialEq)] + enum Script { + Plain, + FailBegin, + SuspendBegin, + SuspendTerminal, + } + + struct Recorder { + name: &'static str, + script: Script, + log: Log, + suspended: Option, + } + + impl Recorder { + fn record(&self, entry: &str) { + self.log + .lock() + .unwrap() + .push(format!("{}.{entry}", self.name)); + } + + fn suspend(&mut self, py: Python<'_>, then: LifecycleStep) -> PyResult { + self.suspended = Some(then); + Ok(LifecycleStep::Await(py.None())) + } + } + + impl PythonLifecycle for Recorder { + fn begin( + &mut self, + py: Python<'_>, + arguments: Py, + _: f64, + ) -> PyResult { + let seen: Vec = arguments + .bind(py) + .keys() + .iter() + .map(|key| key.extract().unwrap()) + .collect(); + self.record(&format!("begin{seen:?}")); + arguments.bind(py).set_item(self.name, true)?; + match self.script { + Script::FailBegin => Err(PyValueError::new_err("begin failed")), + Script::SuspendBegin => self.suspend(py, LifecycleStep::Arguments(arguments)), + _ => Ok(LifecycleStep::Arguments(arguments)), + } + } + + fn before_send( + &mut self, + _: Python<'_>, + mut wire: Box, + _: &RequestContext, + ) -> PyResult { + self.record(&format!("before_send:{}", wire.url)); + wire.url.push_str(self.name); + Ok(LifecycleStep::Wire(wire)) + } + + fn after_success( + &mut self, + _: Python<'_>, + response: Py, + _: Timing, + ) -> PyResult { + self.record("after_success"); + Ok(LifecycleStep::Response(response)) + } + + fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { + let terminal = match event { + LifecycleEvent::Machine(_) => { + self.record("machine"); + false + } + LifecycleEvent::Succeeded { response, .. } => { + self.record(&format!("succeeded:{}", response.bind(py))); + true + } + LifecycleEvent::Failed { error, .. } => { + self.record(&format!("failed:{}", error.value(py))); + true + } + }; + if terminal && self.script == Script::SuspendTerminal { + return self.suspend(py, LifecycleStep::Done); + } + Ok(LifecycleStep::Done) + } + + fn opened(&mut self, _: Python<'_>) -> PyResult<()> { + self.record("opened"); + Ok(()) + } + + fn delivered(&mut self, _: Python<'_>, _: &Py) -> PyResult<()> { + self.record("delivered"); + Ok(()) + } + + fn resume(&mut self, _: Python<'_>, _: PyResult>) -> PyResult { + self.record("resume"); + self.suspended.take().ok_or_else(missing_state) + } + + fn close(&mut self, _: Python<'_>) { + self.record("close"); + } + + fn traverse(&self, _: &PyVisit<'_>) -> Result<(), PyTraverseError> { + Ok(()) + } + } + + fn chain(scripts: [(&'static str, Script); 2]) -> (LifecycleChain, Log) { + let log = Log::default(); + let lifecycles = scripts + .into_iter() + .map(|(name, script)| { + Box::new(Recorder { + name, + script, + log: Arc::clone(&log), + suspended: None, + }) as Box + }) + .collect(); + (LifecycleChain::new(lifecycles), log) + } + + fn entries(log: &Log) -> Vec { + std::mem::take(&mut *log.lock().unwrap()) + } + + fn timing() -> Timing { + Timing { + start_time: 1.0, + end_time: 2.0, + } + } + + fn context() -> RequestContext { + RequestContext { + model: "model".to_string(), + custom_llm_provider: "provider".to_string(), + optional_params: serde_json::Value::Null, + secret_fields: Vec::new(), + api_key: None, + } + } + + #[test] + fn a_later_lifecycle_receives_what_an_earlier_one_returned() { + crate::initialize_python(); + Python::attach(|py| { + let (mut chain, log) = chain([("first", Script::Plain), ("second", Script::Plain)]); + let arguments = PyDict::new(py).unbind(); + let step = chain.begin(py, arguments.clone_ref(py), 1.0).unwrap(); + assert!(matches!(step, LifecycleStep::Arguments(returned) if returned.is(&arguments))); + let wire = Box::new(WireRequest { + url: "url:".to_string(), + headers: Vec::new(), + body: serde_json::Value::Null, + }); + let step = chain.before_send(py, wire, &context()).unwrap(); + assert!(matches!(step, LifecycleStep::Wire(wire) if wire.url == "url:firstsecond")); + assert_eq!( + entries(&log), + [ + "first.begin[]", + "second.begin[\"first\"]", + "first.before_send:url:", + "second.before_send:url:first", + ] + ); + }); + } + + #[test] + fn a_lifecycle_begin_never_reached_observes_nothing_of_the_call() { + crate::initialize_python(); + Python::attach(|py| { + let (mut chain, log) = chain([("first", Script::FailBegin), ("second", Script::Plain)]); + let error = chain + .begin(py, PyDict::new(py).unbind(), 1.0) + .err() + .expect("begin fails"); + let failed = LifecycleEvent::Failed { + timing: timing(), + origin: FailureOrigin::Host, + error: &error, + }; + assert!(matches!( + chain.emit(py, failed).unwrap(), + LifecycleStep::Done + )); + chain.close(py); + assert_eq!( + entries(&log), + [ + "first.begin[]", + "first.failed:begin failed", + "first.close", + "second.close", + ] + ); + }); + } + + #[test] + fn a_suspended_step_resumes_its_lifecycle_and_then_reaches_the_later_ones() { + crate::initialize_python(); + Python::attach(|py| { + let (mut chain, log) = chain([ + ("first", Script::SuspendBegin), + ("second", Script::SuspendTerminal), + ]); + let step = chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap(); + assert!(matches!(step, LifecycleStep::Await(_))); + assert_eq!(entries(&log), ["first.begin[]"]); + let step = chain.resume(py, Ok(py.None())).unwrap(); + assert!(matches!(step, LifecycleStep::Arguments(_))); + assert_eq!(entries(&log), ["first.resume", "second.begin[\"first\"]"]); + }); + } + + #[test] + fn an_event_kept_across_a_suspension_reaches_the_later_lifecycles_unchanged() { + crate::initialize_python(); + Python::attach(|py| { + let (mut chain, log) = chain([ + ("first", Script::SuspendTerminal), + ("second", Script::Plain), + ]); + chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap(); + entries(&log); + let raised = PyValueError::new_err("provider failed"); + let failed = LifecycleEvent::Failed { + timing: timing(), + origin: FailureOrigin::Call, + error: &raised, + }; + assert!(matches!( + chain.emit(py, failed).unwrap(), + LifecycleStep::Await(_) + )); + assert!(matches!( + chain.resume(py, Ok(py.None())).unwrap(), + LifecycleStep::Done + )); + assert_eq!( + entries(&log), + [ + "first.failed:provider failed", + "first.resume", + "second.failed:provider failed", + ] + ); + }); + } +} diff --git a/litellm-rust/crates/host-python/src/driver.rs b/litellm-rust/crates/host-python/src/driver.rs index 392d36e10f4..e126ac87af9 100644 --- a/litellm-rust/crates/host-python/src/driver.rs +++ b/litellm-rust/crates/host-python/src/driver.rs @@ -46,7 +46,6 @@ enum Stage { #[derive(Clone, Copy)] enum Expect { - Started, Arguments, Wire, Emitted, @@ -159,13 +158,7 @@ where match (self.pending.take(), result) { (None, None) => { self.started_at = epoch_seconds(); - let started = LifecycleEvent::Started { - start_time: self.started_at, - }; - match self.adapter.emit(py, started) { - Ok(step) => self.on_adapter(py, step, Expect::Started), - Err(error) => self.adapter_failed(py, error), - } + self.begin(py) } (Some(Pending::Native), Some(Ok(_))) => { let result = self.take_native_result()?; @@ -201,7 +194,6 @@ where self.pending = Some(Pending::Adapter(expect)); Ok(ExecutionStep::Await(awaitable)) } - (Expect::Started, LifecycleStep::Done) => self.begin(py), (Expect::Arguments, LifecycleStep::Arguments(arguments)) => { self.arguments = Some(arguments); self.stage = Stage::Call; @@ -795,7 +787,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { self.log.push(match event { - LifecycleEvent::Started { .. } => "started".into(), LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => { format!("response:{}", raw.body) } @@ -927,7 +918,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "before_send", @@ -973,7 +963,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "classify:provider exploded", @@ -1007,7 +996,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "classify:op rejected", @@ -1039,7 +1027,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "failed:Call:op failed", @@ -1077,7 +1064,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "classify:provider exploded", @@ -1108,7 +1094,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "failed:Host:begin failed", "adapter.close", @@ -1239,10 +1224,7 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri ) .unwrap_err(); assert!(!error.is_instance_of::(py)); - assert_eq!( - log.entries(), - ["started", "begin", "route", "adapter.close"] - ); + assert_eq!(log.entries(), ["begin", "route", "adapter.close"]); }); } diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 583a4eb91b6..9048925512d 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -6,6 +6,7 @@ mod adapter; mod argument; mod callable; +mod chain; mod driver; mod execution; mod gil; @@ -13,10 +14,12 @@ mod handle; mod marshal; pub use adapter::{ - InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, missing_state, + InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, is_cancellation, + missing_state, }; pub use argument::lookup; pub use callable::wrap_failure; +pub use chain::LifecycleChain; pub use driver::run_call; pub use execution::{poll_async_value, run_async, run_async_value, run_sync, run_sync_value}; pub use gil::{release_count, release_gil}; diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index 2a77179a7f9..d299aca1f4a 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -1,7 +1,11 @@ - Target invariants, not completion claims; these supersede the crate guidance below where they conflict - Keep this crate the product-specific PyO3 consumer of `litellm-host-python` - Own registration, input projection, the route host and the caller callables it answers operations with (file readers, token providers), public response/error construction and the per-call composition of machine, route host and callback contract - - Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `run_legacy_python_call`; the bridge hands the public call over and keeps no copy + - Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `LegacyPythonLifecycle`; the bridge hands the public call over and keeps no copy + - Routes never select a callback contract: `callbacks::run_python_call` is the only way a route starts a call, and `src/callbacks.rs` is the only place that names a contract crate + - Every call runs the legacy `Logging` lifecycle, then the v1 lifecycle when anything is subscribed; with no subscriber the call is exactly the legacy-only call + - A route describes itself once as a `CallSurface`; fields only the legacy contract reads are marked so and go when it goes + - Retiring legacy for a route, or for all of them, is a change to that one composition, never to a route, the driver or an adapter - Value-oriented execution, sync waiting, nested-runtime checks, signal polling and panic containment live in `litellm-host-python`; native async work uses `pyo3-async-runtimes`, Serde output uses `Pythonized` - Core owns typed native state, the route machine, provider preparation/I/O and normalization; the host driver owns terminal events; the legacy adapter in `litellm-callbacks-legacy-python` owns `Logging` dispatch policy - Python, Rust SDK and gateway use one lifecycle-bearing core route entrypoint; provider helpers stay private, never bridge-accessible transport drivers diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 3a4a579efa3..4d0a3f6d349 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -19,10 +19,12 @@ panic-test = [] bytes.workspace = true litellm-auth.workspace = true litellm-callbacks-legacy-python.workspace = true +litellm-callbacks-v1-python.workspace = true litellm-core.workspace = true litellm-core-utils.workspace = true litellm-auth-gcp.workspace = true litellm-http.workspace = true +litellm-host.workspace = true litellm-llms.workspace = true litellm-types.workspace = true litellm-host-python.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/callbacks.rs b/litellm-rust/crates/python-bridge/src/callbacks.rs new file mode 100644 index 00000000000..ef5f9fa898d --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/callbacks.rs @@ -0,0 +1,79 @@ +//! The callback contracts a Python call runs under, composed here once so that no route +//! selects one. Every call runs the legacy `Logging` contract, for as long as Python's +//! `Logging` exists, and then the v1 contract whenever anything is subscribed to it. +//! Both are a [`PythonLifecycle`]; that trait is the whole boundary the driver knows. + +use litellm_callbacks_legacy_python::{LegacyPythonLifecycle, LegacyPythonSurface}; +pub(crate) use litellm_callbacks_legacy_python::{PassThroughStream, PublicCall}; +use litellm_callbacks_v1_python::{V1PythonLifecycle, V1PythonSurface}; +use litellm_host::{machine::Machine, route::Route}; +use litellm_host_python::{LifecycleChain, PythonLifecycle, RouteHost, run_call}; +use pyo3::prelude::*; + +/// What a route tells the callback contracts about itself. +#[derive(Clone, Copy, Debug)] +pub(crate) struct CallSurface { + pub call_type: &'static str, + /// Legacy only: what `Logging.pre_call` is told the input was. + pub input_description: &'static str, + /// Legacy only: how a streamed response is billed; `None` for a route that never + /// streams. + pub stream: Option, +} + +impl CallSurface { + fn legacy(self) -> LegacyPythonSurface { + LegacyPythonSurface { + call_type: self.call_type, + input_description: self.input_description, + stream: self.stream, + } + } + + fn v1(self) -> V1PythonSurface { + V1PythonSurface { + call_type: self.call_type, + } + } +} + +/// Legacy first: v1 subscribers observe the keyword view and the wire request as the +/// legacy contract left them, and share the `litellm_call_id` its setup assigned. +fn lifecycle( + py: Python<'_>, + surface: CallSurface, + call: PublicCall, + asynchronous: bool, +) -> PyResult> { + let legacy = Box::new(LegacyPythonLifecycle::new( + py, + surface.legacy(), + call, + asynchronous, + )); + Ok( + match V1PythonLifecycle::subscribed(py, surface.v1(), asynchronous)? { + Some(v1) => Box::new(LifecycleChain::new(vec![legacy, Box::new(v1)])), + None => legacy, + }, + ) +} + +/// Runs one native call for a Python caller. The route host projects from the keyword +/// view the lifecycle's `begin` prepares. +pub(crate) fn run_python_call( + py: Python<'_>, + surface: CallSurface, + call: PublicCall, + machine: M, + route: H, + asynchronous: bool, +) -> PyResult> +where + H: RouteHost + 'static, + M: Machine::Response> + 'static, +{ + let arguments = call.arguments(py); + let lifecycle = lifecycle(py, surface, call, asynchronous)?; + run_call(py, machine, route, lifecycle, arguments, asynchronous) +} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 7eba0d201be..6bbf3555f17 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,3 +1,4 @@ +mod callbacks; mod credentials; mod diagnostics; mod errors; diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index 294ba549f11..7f6c1981467 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -1,18 +1,18 @@ mod host; use host::MessagesRouteHost; -use litellm_callbacks_legacy_python::{ - LegacyPythonSurface, PassThroughStream, PublicCall, run_legacy_python_call, -}; use litellm_core::messages::route::{messages_machine, supports}; use pyo3::{ prelude::*, types::{PyDict, PyTuple}, }; -use crate::errors::RustBridgeDeclined; +use crate::{ + callbacks::{CallSurface, PassThroughStream, PublicCall, run_python_call}, + errors::RustBridgeDeclined, +}; -const SURFACE: LegacyPythonSurface = LegacyPythonSurface { +const SURFACE: CallSurface = CallSurface { call_type: "anthropic_messages", input_description: "Messages", stream: Some(PassThroughStream { @@ -39,7 +39,7 @@ fn run_messages( "the Rust Messages route does not serve this provider", )); } - run_legacy_python_call( + run_python_call( py, SURFACE, PublicCall::capture(&request, &args, &kwargs)?, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 05b28c239d3..ca49dc27784 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -7,7 +7,6 @@ use std::sync::{Arc, LazyLock}; use host::OcrRouteHost; use litellm_auth_gcp::VertexAuth; -use litellm_callbacks_legacy_python::{LegacyPythonSurface, PublicCall, run_legacy_python_call}; use litellm_core::ocr::route::ocr_machine; use litellm_core_utils::settings::ProcessEnvironment; use litellm_llms::base_llm::ocr::{ @@ -19,15 +18,20 @@ use pyo3::{ types::{PyDict, PyTuple}, }; -use crate::{errors::RustBridgeDeclined, http, python_settings::PythonSettings}; +use crate::{ + callbacks::{CallSurface, PublicCall, run_python_call}, + errors::RustBridgeDeclined, + http, + python_settings::PythonSettings, +}; -const SURFACE: LegacyPythonSurface = LegacyPythonSurface { +const SURFACE: CallSurface = CallSurface { call_type: "ocr", input_description: "OCR document processing", stream: None, }; -const ASYNC_SURFACE: LegacyPythonSurface = LegacyPythonSurface { +const ASYNC_SURFACE: CallSurface = CallSurface { call_type: "aocr", ..SURFACE }; @@ -52,7 +56,7 @@ fn run_ocr( secrets, ) .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?; - run_legacy_python_call( + run_python_call( py, if asynchronous { ASYNC_SURFACE } else { SURFACE }, PublicCall::capture(&request, &args, &kwargs)?, diff --git a/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py b/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py index 1fa05579f47..f854ea4b121 100644 --- a/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py +++ b/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py @@ -35,10 +35,9 @@ def test_v1_golden_envelopes_match_the_typed_contract() -> None: with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning) adapter: Final = TypeAdapter(callbacks.EnvelopeV1) - events: Final = { - adapter.validate_json(path.read_text(), strict=True)["event"]["type"] for path in GOLDEN_PATH.glob("*.json") - } - assert events == callbacks.EVENTS + envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")] + assert {envelope["event"]["type"] for envelope in envelopes} == callbacks.EVENTS + assert {envelope["schema"] for envelope in envelopes} == callbacks.SUPPORTED_SCHEMAS class ValidCallback: diff --git a/tests/test_litellm/rust_bridge/test_loader.py b/tests/test_litellm/rust_bridge/test_loader.py deleted file mode 100644 index a5bd56d0a6a..00000000000 --- a/tests/test_litellm/rust_bridge/test_loader.py +++ /dev/null @@ -1,27 +0,0 @@ -from types import ModuleType - -import pytest - -from litellm.rust_bridge import loader - - -@pytest.fixture(autouse=True) -def _reset_loader() -> None: - loader.reset_native_bridge_cache() - yield - loader.reset_native_bridge_cache() - - -def test_child_forked_after_the_extension_loaded_reports_it_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(loader, "_cached_bridge", ModuleType("_native")) - - loader._disable_in_forked_child() - - assert loader.get_native_bridge() is None - assert loader.native_bridge_available() is False - - -def test_child_forked_before_the_extension_loaded_keeps_it() -> None: - loader._disable_in_forked_child() - - assert loader._forked_after_load is False diff --git a/tests/test_litellm_rust/ocr/test_callbacks_v1.py b/tests/test_litellm_rust/ocr/test_callbacks_v1.py new file mode 100644 index 00000000000..3da44a6c394 --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_callbacks_v1.py @@ -0,0 +1,141 @@ +from collections.abc import Generator +from typing import Final + +import pytest + +from litellm.integrations.custom_logger import CustomLogger +from litellm.rust_bridge import callbacks_v1_python +from litellm.rust_bridge.callbacks_v1_python import EVENTS, EnvelopeV1, RequestFactsV1, WirePatchV1 +from tests.test_litellm_rust.support.callback_recorder import drain_logging +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import ( + OCR_RESPONSE, + call_native_aocr, + call_native_ocr, + request_body, +) + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +class Observer: + name: Final = "observer" + schema: Final = 1 + events: Final = EVENTS + + def __init__(self) -> None: + self.envelopes: Final[list[EnvelopeV1]] = [] + + def on_event(self, event: EnvelopeV1) -> None: + self.envelopes.append(event) + + +class Interceptor: + name: Final = "interceptor" + schema: Final = 1 + events: Final[frozenset[str]] = frozenset() + + def __init__(self) -> None: + self.bodies: Final[list[object]] = [] + + def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: + self.bodies.append(request["body"]) + return {"headers": {"set": [("x-v1-interceptor", "seen")]}} + + +@pytest.fixture +def observer() -> Generator[Observer]: + subscriber: Final = Observer() + callbacks_v1_python.register(subscriber) + try: + yield subscriber + finally: + callbacks_v1_python.unregister(subscriber) + + +@pytest.fixture +def interceptor() -> Generator[Interceptor]: + subscriber: Final = Interceptor() + callbacks_v1_python.register(subscriber) + try: + yield subscriber + finally: + callbacks_v1_python.unregister(subscriber) + + +class LegacyCallId(CustomLogger): + def __init__(self) -> None: + self.call_ids: Final[list[str]] = [] + + def log_pre_api_call(self, model, messages, kwargs): + self.call_ids.append(kwargs["litellm_call_id"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_ocr_runs_legacy_and_v1_callbacks_on_the_same_call( + ocr_server: RecordingServer, observer: Observer, asynchronous: bool +) -> None: + legacy: Final = LegacyCallId() + + if asynchronous: + await call_native_aocr(ocr_server, callbacks=[legacy]) + else: + call_native_ocr(ocr_server, callbacks=[legacy]) + await drain_logging() + + assert [envelope["event"]["type"] for envelope in observer.envelopes] == [ + "call.started", + "request.sending", + "response.received", + "call.succeeded", + ] + assert [envelope["seq"] for envelope in observer.envelopes] == sorted( + envelope["seq"] for envelope in observer.envelopes + ) + assert len(legacy.call_ids) == 1 + assert {envelope["call_id"] for envelope in observer.envelopes} == set(legacy.call_ids) + + +def test_native_ocr_v1_interceptor_sees_the_wire_as_the_legacy_callback_left_it( + ocr_server: RecordingServer, interceptor: Interceptor +) -> None: + class Edit(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + request_body(kwargs)["include_image_base64"] = True + + call_native_ocr(ocr_server, callbacks=[Edit()]) + + assert len(interceptor.bodies) == 1 + body: Final = interceptor.bodies[0] + assert isinstance(body, dict) + assert body["include_image_base64"] is True + request: Final = ocr_server.requests[-1] + assert request.headers["x-v1-interceptor"] == "seen" + + +def test_native_ocr_failure_reaches_both_contracts_once( + recording_server: RecordingServer, observer: Observer +) -> None: + recording_server.default_response = ResponseSpec(status=400, body={"message": "bad document"}) + failures: Final[list[object]] = [] + + class Failure(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + failures.append(kwargs["exception"]) + + with pytest.raises(Exception) as raised: # noqa: PT011 # the mapped provider error class is covered elsewhere + call_native_ocr(recording_server, callbacks=[Failure()]) + + assert failures == [raised.value] + terminal: Final = [ + envelope for envelope in observer.envelopes if envelope["event"]["type"] in {"call.succeeded", "call.failed"} + ] + assert [envelope["event"]["type"] for envelope in terminal] == ["call.failed"] + assert observer.envelopes[-1] is terminal[0]