diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 2720cf01f2e..a02d9803528 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2043,6 +2043,33 @@ dependencies = [ "strum", ] +[[package]] +name = "litellm-callbacks-v1" +version = "0.1.0" +dependencies = [ + "litellm-auth", + "litellm-host", + "proptest", + "rstest", + "serde", + "serde_json", + "strum", + "thiserror 2.0.19", +] + +[[package]] +name = "litellm-callbacks-v1-python" +version = "0.1.0" +dependencies = [ + "litellm-auth", + "litellm-callbacks-v1", + "litellm-host", + "litellm-host-python", + "pyo3", + "serde_json", + "strum", +] + [[package]] name = "litellm-core" version = "0.1.0" @@ -2085,6 +2112,7 @@ version = "0.1.0" dependencies = [ "fancy-regex", "litellm-types", + "percent-encoding", "rstest", "serde", "serde_json", @@ -2170,6 +2198,7 @@ dependencies = [ "litellm-host", "litellm-http", "litellm-types", + "percent-encoding", "reqwest 0.12.28", "rstest", "serde", @@ -2193,8 +2222,10 @@ dependencies = [ "litellm-auth", "litellm-auth-gcp", "litellm-callbacks-legacy-python", + "litellm-callbacks-v1-python", "litellm-core", "litellm-core-utils", + "litellm-host", "litellm-host-python", "litellm-http", "litellm-llms", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index a6185632871..bfc84b3dcc9 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -12,6 +12,8 @@ repository = "https://github.com/BerriAI/litellm" litellm-core = { path = "crates/core" } litellm-host = { path = "crates/host" } litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" } +litellm-callbacks-v1 = { path = "crates/callbacks-v1" } +litellm-callbacks-v1-python = { path = "crates/callbacks-v1-python" } litellm-framing = { path = "crates/framer" } litellm-auth = { path = "crates/auth" } litellm-auth-aws = { path = "crates/auth-aws" } @@ -52,6 +54,7 @@ base64 = "0.22" moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" +percent-encoding = "2.3.2" webpki-roots = "1" time = { version = "0.3.53", features = ["parsing"] } criterion = "0.8.2" diff --git a/litellm-rust/crates/callbacks-legacy-python/AGENTS.md b/litellm-rust/crates/callbacks-legacy-python/AGENTS.md index 8b2e1c15f6e..db281db43db 100644 --- a/litellm-rust/crates/callbacks-legacy-python/AGENTS.md +++ b/litellm-rust/crates/callbacks-legacy-python/AGENTS.md @@ -1,10 +1,20 @@ - Target invariants, not completion claims - Keep this crate the legacy `@client` wrapper as the native call sees it, and nothing else: the `Logging` contract (`function_setup`, the deployment hooks, `pre_call`/`post_call`, the sync and async success and failure fan-out, the deferred proxy release, the argument sharing those callbacks rely on) plus the kwargs rewrites the wrapper makes on the way in (credential-name inheritance, the budget and retry-count limits) - The driver in `litellm-host-python`, the routes and core see one `PythonLifecycle`; they never learn which Python objects consume a call -- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`) +- Terms (lifecycle terms are in `host-python/AGENTS.md`) + - `Logging`: litellm's Python class; logger: the one `Logging` instance of this call (`PythonLogger`), reused from `litellm_logging_obj` or built by `function_setup` + - Borrowed function: a litellm Python internal Rust still calls, one variant of `LegacyPython`; a user's own callback is never one + - Fan-out: `Logging` dispatching to every registered callback; family: the sync or the async set of success and failure handlers + - Deployment hooks: the awaited pre-call, post-call and failure hooks on each callback; async calls only, as in `@client` + - Re-aliasing: replacing body values with the caller's equal objects before `pre_call`, so a callback mutates what the caller holds + - Deferred release: the proxy holds back async success logging until it accepts the response (`PendingLogging.release`) + - Internal call: a proxy-internal call; when it is async, the async success dispatch and both failure families are skipped, and the sync success handlers still run + - Pass-through billing (`PassThroughStream`): end-of-stream cost logging from the delivered chunks; a failure after the stream opened bills them as partial usage, and a sync call falls back to the plain failure handler + - Redaction here is `****`, matching what the Python path logs; `[REDACTED]` belongs to the v1 contract +- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`, `Streaming`) - The enum only shrinks: when Rust owns a subsystem, delete its group rather than adding a Rust path beside it - Calling a user's own callback directly is permanent Python surface and gets its own type outside `LegacyPython` - - `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes hand it over through `run_legacy_call` and keep no copy + - `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes capture it and the bridge's `run_python_call` hands it to `LegacyPythonLifecycle::new`, keeping no copy - `setup` reuses a `Logging` the caller passed as `litellm_logging_obj` (the proxy and Router are the live cases) and otherwise builds one through `function_setup`, as `@client` does - Either way every phase calls the same `Logging` method the Python path calls; which callbacks run is `Logging`'s decision, never this crate's - Callbacks receive the caller's own objects and may mutate them; this crate alone carries that obligation diff --git a/litellm-rust/crates/callbacks-legacy-python/Cargo.toml b/litellm-rust/crates/callbacks-legacy-python/Cargo.toml index fe19578e04d..1fc81b6012b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/Cargo.toml +++ b/litellm-rust/crates/callbacks-legacy-python/Cargo.toml @@ -9,13 +9,11 @@ autotests = false [dependencies] litellm-host.workspace = true litellm-host-python.workspace = true - pyo3.workspace = true -strum.workspace = true serde_json.workspace = true +strum.workspace = true [dev-dependencies] litellm-auth.workspace = true proptest.workspace = true rstest.workspace = true -serde_json.workspace = true diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index e1742190205..b77f9305aa9 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -6,10 +6,10 @@ use litellm_host::event::{ FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest, epoch_seconds, }; use litellm_host_python::{ - LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, missing_state, to_py, + LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py, }; use pyo3::{ - exceptions::{PyBaseException, PyException}, + exceptions::PyBaseException, gc::{PyTraverseError, PyVisit}, prelude::*, types::{PyDateTime, PyDict, PyList}, @@ -17,7 +17,8 @@ use pyo3::{ use serde_json::Value; use crate::{ - DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger, + PublicCall, PythonLogger, after_deployment_failure, after_deployment_success, + before_deployment_call, deferred::{PendingLogging, PendingSuccess}, finalize, is_internal_call, prepare, python::Streaming, @@ -26,7 +27,7 @@ use crate::{ /// What the legacy contract needs to know about the route it is logging. #[derive(Clone, Copy, Debug)] -pub struct LegacySurface { +pub struct LegacyPythonSurface { pub call_type: &'static str, /// What `Logging.pre_call` is told the input was. pub input_description: &'static str, @@ -55,8 +56,8 @@ enum Pending { AsyncFailure, } -pub struct LegacyLogging { - surface: LegacySurface, +pub struct LegacyPythonLifecycle { + surface: LegacyPythonSurface, call: PublicCall, logger: Option, start: Py, @@ -76,14 +77,10 @@ fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult> { PyDateTime::from_timestamp(py, epoch_seconds, None).map(|value| value.into_any().unbind()) } -fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool { - !error.is_instance_of::(py) -} - -impl LegacyLogging { +impl LegacyPythonLifecycle { pub fn new( py: Python<'_>, - surface: LegacySurface, + surface: LegacyPythonSurface, call: PublicCall, asynchronous: bool, ) -> Self { @@ -118,6 +115,14 @@ impl LegacyLogging { } fn prepare(&mut self, py: Python<'_>) -> PyResult { + let model = self.call.lookup(py, "model")?; + let provider = self.call.lookup(py, "custom_llm_provider")?; + self.logger()?.initialize_failure_context( + py, + self.call.kwargs(), + model.and_then(|model| model.extract::().ok()), + provider.and_then(|provider| provider.extract::().ok()), + )?; let prepared = prepare(py, self.call.kwargs().bind(py), self.logger()?)?.unbind(); self.call.set_kwargs(prepared); Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py))) @@ -140,7 +145,7 @@ impl LegacyLogging { fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> { match self.try_dispatch_success(py) { - Err(error) if error.is_instance_of::(py) => { + Err(error) if !is_cancellation(py, &error) => { error.write_unraisable(py, self.logger.as_ref().map(|logger| logger.object(py))); Ok(()) } @@ -199,7 +204,7 @@ impl LegacyLogging { ), ); match billed { - Err(error) if error.is_instance_of::(py) => { + Err(error) if !is_cancellation(py, &error) => { error.write_unraisable(py, Some(logger.object(py))); Ok(()) } @@ -269,15 +274,15 @@ impl LegacyLogging { } } -impl PythonLifecycle for LegacyLogging { +impl PythonLifecycle for LegacyPythonLifecycle { fn begin( &mut self, py: Python<'_>, arguments: Py, - started_at: f64, + start_time: f64, ) -> PyResult { self.call.set_kwargs(arguments); - self.start = datetime(py, started_at)?; + self.start = datetime(py, start_time)?; self.internal = is_internal_call(py)?; let result = setup( py, @@ -291,7 +296,7 @@ impl PythonLifecycle for LegacyLogging { self.call.set_kwargs(result.kwargs()?); if self.runs_deployment_hooks() { self.pending = Some(Pending::DeploymentPreCall); - return Ok(LifecycleStep::Await(DeploymentHooks::before_call( + return Ok(LifecycleStep::Await(before_deployment_call( py, self.call.kwargs(), self.surface.call_type, @@ -354,7 +359,7 @@ impl PythonLifecycle for LegacyLogging { self.response = Some(response); if self.runs_deployment_hooks() { self.pending = Some(Pending::DeploymentPostCall); - return Ok(LifecycleStep::Await(DeploymentHooks::after_success( + return Ok(LifecycleStep::Await(after_deployment_success( py, self.call.kwargs(), &self.response, @@ -366,7 +371,6 @@ impl PythonLifecycle for LegacyLogging { fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { match event { - LifecycleEvent::Started { .. } => Ok(LifecycleStep::Done), LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => { let api_key = self .context @@ -407,7 +411,7 @@ impl PythonLifecycle for LegacyLogging { { let error = self.error.as_ref().ok_or_else(missing_state)?; self.pending = Some(Pending::DeploymentFailure); - return Ok(LifecycleStep::Await(DeploymentHooks::after_failure( + return Ok(LifecycleStep::Await(after_deployment_failure( py, self.call.kwargs(), error, @@ -464,7 +468,11 @@ impl PythonLifecycle for LegacyLogging { { error.write_unraisable(py, None); } + self.pending = None; + self.response = None; + self.error = None; self.body = None; + self.headers = None; self.context = None; self.stream = None; } @@ -482,7 +490,8 @@ impl PythonLifecycle for LegacyLogging { visit.call(&stream.chunks)?; visit.call(&stream.first_chunk)?; } - visit.call(&self.body) + visit.call(&self.body)?; + visit.call(&self.headers) } } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/call.rs b/litellm-rust/crates/callbacks-legacy-python/src/call.rs index b37790f60a8..022f1d5fcf2 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/call.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/call.rs @@ -3,16 +3,13 @@ //! lifetime. No other callback host has that obligation, which is why nothing outside //! this crate holds them. -use litellm_host::{machine::Machine, route::Route}; -use litellm_host_python::{RouteHost, lookup, run_call}; +use litellm_host_python::lookup; use pyo3::{ gc::{PyTraverseError, PyVisit}, prelude::*, types::{PyDict, PyTuple}, }; -use crate::{LegacyLogging, LegacySurface}; - pub struct PublicCall { args: Py, kwargs: Py, @@ -34,6 +31,11 @@ impl PublicCall { }) } + /// The keyword view the call starts from, which the driver hands to `begin`. + pub fn arguments(&self, py: Python<'_>) -> Py { + self.kwargs.clone_ref(py) + } + pub(crate) fn args(&self) -> &Py { &self.args } @@ -63,31 +65,6 @@ impl PublicCall { } } -/// Runs one native call under the legacy `Logging` contract: the route host projects from -/// the keyword view the contract prepares, and the contract observes the call. -pub fn run_legacy_call( - py: Python<'_>, - surface: LegacySurface, - call: PublicCall, - machine: M, - route: H, - asynchronous: bool, -) -> PyResult> -where - H: RouteHost + 'static, - M: Machine::Response> + 'static, -{ - let arguments = call.kwargs.clone_ref(py); - run_call( - py, - machine, - route, - Box::new(LegacyLogging::new(py, surface, call, asynchronous)), - arguments, - asynchronous, - ) -} - #[cfg(test)] mod tests { use super::*; diff --git a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs index 7caa787dd9d..cba711a6ab4 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs @@ -9,78 +9,10 @@ use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; use crate::logger::PythonLogger; use crate::python::{Logging, Wrapper}; -pub trait LegacyCallbacks { +impl PythonLogger { /// `Logging.update_from_kwargs`: what the logger is told about the request it is /// about to see, with consumed credentials redacted. - fn update_from_kwargs( - &self, - py: Python<'_>, - kwargs: &Py, - wire: &WireRequest, - context: &RequestContext, - ) -> PyResult<()>; - - /// `Logging.pre_call`. - fn pre_call( - &self, - py: Python<'_>, - input: &str, - api_key: Option<&str>, - body: &Bound<'_, PyDict>, - headers: &Bound<'_, PyDict>, - url: &str, - ) -> PyResult<()>; - - /// `Logging.post_call`. - fn post_call( - &self, - py: Python<'_>, - original_response: &str, - api_key: Option<&str>, - body: Option<&Py>, - headers: Option<&Py>, - ) -> PyResult<()>; - - fn defers_async_logging(&self, py: Python<'_>) -> bool; - - fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()>; - - fn sync_success_for_async_call( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()>; - - fn failure( - &self, - py: Python<'_>, - error: &Py, - start: &Py, - end: &Option>, - asynchronous: bool, - ) -> PyResult>>; - - fn submit_success( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()>; - - fn enqueue_success( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()>; -} - -impl LegacyCallbacks for PythonLogger { - fn update_from_kwargs( + pub(crate) fn update_from_kwargs( &self, py: Python<'_>, kwargs: &Py, @@ -128,7 +60,8 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn pre_call( + /// `Logging.pre_call`. + pub(crate) fn pre_call( &self, py: Python<'_>, input: &str, @@ -145,7 +78,8 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn post_call( + /// `Logging.post_call`. + pub(crate) fn post_call( &self, py: Python<'_>, original_response: &str, @@ -163,19 +97,19 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn defers_async_logging(&self, py: Python<'_>) -> bool { + pub(crate) fn defers_async_logging(&self, py: Python<'_>) -> bool { Logging::DefersAsync .call(py, (self.object(py),)) .and_then(|value| value.extract()) .unwrap_or(false) } - fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> { Logging::DeferSuccess.call(py, (self.object(py), pending))?; Ok(()) } - fn sync_success_for_async_call( + pub(crate) fn sync_success_for_async_call( &self, py: Python<'_>, response: &Option>, @@ -186,7 +120,7 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn failure( + pub(crate) fn failure( &self, py: Python<'_>, error: &Py, @@ -199,7 +133,7 @@ impl LegacyCallbacks for PythonLogger { Ok(asynchronous.then(|| value.unbind())) } - fn submit_success( + pub(crate) fn submit_success( &self, py: Python<'_>, response: &Option>, @@ -210,7 +144,7 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } - fn enqueue_success( + pub(crate) fn enqueue_success( &self, py: Python<'_>, response: &Option>, @@ -227,6 +161,55 @@ impl LegacyCallbacks for PythonLogger { } enqueue.map(|_| ()) } + + pub(crate) fn initialize_failure_context( + &self, + py: Python<'_>, + kwargs: &Py, + model: Option, + custom_llm_provider: Option, + ) -> PyResult<()> { + let Some(model) = model else { + return Ok(()); + }; + let (provider, model) = match custom_llm_provider.filter(|provider| !provider.is_empty()) { + Some(provider) => { + let model = model + .strip_prefix(provider.as_str()) + .and_then(|model| model.strip_prefix('/')) + .unwrap_or(&model) + .to_string(); + (provider, model) + } + None => match model.split_once('/') { + Some((provider, model)) if !provider.is_empty() && !model.is_empty() => { + (provider.to_string(), model.to_string()) + } + _ => return Ok(()), + }, + }; + let kwargs = kwargs.bind(py); + let metadata = PyDict::new(py); + for name in ["metadata", "litellm_metadata"] { + if let Some(value) = kwargs.get_item(name)? { + metadata.set_item(name, value)?; + } + } + let params = PyDict::new(py); + params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?; + Logging::Update.call( + py, + ( + self.object(py), + metadata, + model, + PyDict::new(py), + params, + provider, + ), + )?; + Ok(()) + } } fn custom_pricing_fields(py: Python<'_>) -> PyResult> { @@ -254,6 +237,6 @@ fn redact( } /// Proxy-internal calls skip the legacy success fan-out. -pub fn is_internal_call(py: Python<'_>) -> PyResult { +pub(crate) fn is_internal_call(py: Python<'_>) -> PyResult { Wrapper::IsInternalCall.call(py, ())?.extract() } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs b/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs index b18012f926e..6c94e565045 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs @@ -1,9 +1,10 @@ //! The proxy's deferred success release: the async success handler is queued only once //! the proxy accepts the response, and at most once. -use pyo3::{exceptions::PyException, prelude::*}; +use litellm_host_python::is_cancellation; +use pyo3::prelude::*; -use crate::{LegacyCallbacks, PythonLogger}; +use crate::PythonLogger; pub(crate) struct PendingSuccess { pub(crate) logger: PythonLogger, @@ -37,7 +38,7 @@ impl PendingLogging { && success { match pending.asynchronous(py) { - Err(error) if error.is_instance_of::(py) => { + Err(error) if !is_cancellation(py, &error) => { error.write_unraisable(py, Some(pending.logger.object(py))); } result => return result, diff --git a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs index 44393792d1f..5aaf7048b71 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs @@ -6,8 +6,8 @@ //! core never learn which Python object is on the other end. //! //! Legacy callbacks receive the caller's own objects and may mutate them. [`PublicCall`] -//! is where those objects live, and [`run_legacy_call`] is how a route hands them over -//! without keeping a copy. +//! is where those objects live; the bridge hands it to [`LegacyPythonLifecycle::new`] and +//! keeps no copy. mod adapter; mod call; @@ -20,9 +20,11 @@ mod python; #[path = "../tests/support.rs"] mod test_support; -pub(crate) use adapter::LegacyLogging; -pub use adapter::{LegacySurface, PassThroughStream}; -pub use call::{PublicCall, run_legacy_call}; -pub(crate) use callbacks::{LegacyCallbacks, is_internal_call}; -pub(crate) use logger::{DeploymentHooks, PythonLogger, finalize, setup}; +pub use adapter::{LegacyPythonLifecycle, LegacyPythonSurface, PassThroughStream}; +pub use call::PublicCall; +pub(crate) use callbacks::is_internal_call; +pub(crate) use logger::{ + PythonLogger, after_deployment_failure, after_deployment_success, before_deployment_call, + finalize, setup, +}; pub(crate) use preparation::prepare; diff --git a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs index 38f3bf29828..03c2313182a 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs @@ -1,3 +1,6 @@ +//! One call's `Logging` instance (the logger) and the `@client` wrapper functions that +//! create and finish it: `function_setup`, `finalize` and the awaited deployment hooks. + use pyo3::{ exceptions::PyBaseException, gc::{PyTraverseError, PyVisit}, @@ -5,10 +8,10 @@ use pyo3::{ types::{PyDict, PyTuple}, }; -use crate::python::{self, Wrapper}; +use crate::python::{DeploymentHooks, Wrapper}; /// The `Logging` instance one call fans out through. -pub struct PythonLogger { +pub(crate) struct PythonLogger { object: Py, } @@ -21,17 +24,17 @@ impl PythonLogger { self.object.bind(py) } - pub fn clone_ref(&self, py: Python<'_>) -> Self { + pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { Self { object: self.object.clone_ref(py), } } - pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.object) } - pub fn restore_context(&self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn restore_context(&self, py: Python<'_>) -> PyResult<()> { Wrapper::RestoreContext.call(py, (self.object(py),))?; Ok(()) } @@ -45,19 +48,19 @@ impl FromPyObject<'_, '_> for PythonLogger { } } -pub struct SetupResult<'py>(Bound<'py, PyAny>); +pub(crate) struct SetupResult<'py>(Bound<'py, PyAny>); impl SetupResult<'_> { - pub fn logger(&self) -> PyResult { + pub(crate) fn logger(&self) -> PyResult { Ok(PythonLogger::new(self.0.getattr("logger")?.unbind())) } - pub fn kwargs(&self) -> PyResult> { + pub(crate) fn kwargs(&self) -> PyResult> { Ok(self.0.getattr("kwargs")?.extract()?) } } -pub fn setup<'py>( +pub(crate) fn setup<'py>( py: Python<'py>, call_type: &str, args: &Py, @@ -70,7 +73,7 @@ pub fn setup<'py>( .map(SetupResult) } -pub fn finalize( +pub(crate) fn finalize( py: Python<'_>, response: &Option>, logger: &PythonLogger, @@ -82,40 +85,36 @@ pub fn finalize( Ok(()) } -pub struct DeploymentHooks; +pub(crate) fn before_deployment_call( + py: Python<'_>, + kwargs: &Py, + call_type: &str, +) -> PyResult> { + DeploymentHooks::BeforeDeploymentCall + .call(py, (kwargs, call_type)) + .map(Bound::unbind) +} -impl DeploymentHooks { - pub fn before_call( - py: Python<'_>, - kwargs: &Py, - call_type: &str, - ) -> PyResult> { - python::DeploymentHooks::BeforeDeploymentCall - .call(py, (kwargs, call_type)) - .map(Bound::unbind) - } +pub(crate) fn after_deployment_success( + py: Python<'_>, + kwargs: &Py, + response: &Option>, + call_type: &str, +) -> PyResult> { + DeploymentHooks::AfterDeploymentSuccess + .call(py, (kwargs, response, call_type)) + .map(Bound::unbind) +} - pub fn after_success( - py: Python<'_>, - kwargs: &Py, - response: &Option>, - call_type: &str, - ) -> PyResult> { - python::DeploymentHooks::AfterDeploymentSuccess - .call(py, (kwargs, response, call_type)) - .map(Bound::unbind) - } - - pub fn after_failure( - py: Python<'_>, - kwargs: &Py, - error: &Py, - call_type: &str, - ) -> PyResult> { - python::DeploymentHooks::AfterDeploymentFailure - .call(py, (kwargs, error, call_type)) - .map(Bound::unbind) - } +pub(crate) fn after_deployment_failure( + py: Python<'_>, + kwargs: &Py, + error: &Py, + call_type: &str, +) -> PyResult> { + DeploymentHooks::AfterDeploymentFailure + .call(py, (kwargs, error, call_type)) + .map(Bound::unbind) } #[cfg(test)] diff --git a/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs b/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs index aab654c9893..26f0039576a 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs @@ -1,3 +1,6 @@ +//! The kwargs rewrites the `@client` wrapper makes before the call: the logger is attached, +//! a named credential's values are inherited, and the budget and retry-count limits run. + use pyo3::{ prelude::*, types::{PyDict, PyList}, @@ -17,7 +20,7 @@ impl<'py> CredentialEntry<'py> { } } -pub fn prepare<'py>( +pub(crate) fn prepare<'py>( py: Python<'py>, kwargs: &Bound<'py, PyDict>, logger: &crate::PythonLogger, diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index cb609d52878..1b225e03bae 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -1,3 +1,6 @@ +//! The Python contract of this crate: every function of +//! `litellm/rust_bridge/callbacks_legacy_python.py` that Rust calls, by name. + use pyo3::prelude::*; use strum::{IntoStaticStr, VariantArray}; @@ -153,7 +156,7 @@ mod tests { use crate::test_support::PYTHON_CONTRACT; #[test] - fn every_borrowed_function_is_in_the_python_contract() { + fn every_function_is_in_the_python_contract() { let contract: serde_json::Map = serde_json::from_str(PYTHON_CONTRACT).unwrap(); let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect(); @@ -177,7 +180,7 @@ mod tests { ) .map(LegacyPython::name) .collect(); - assert_eq!(called.len(), declared.len(), "a function is borrowed twice"); + assert_eq!(called.len(), declared.len(), "a function is listed twice"); assert_eq!(called.into_iter().collect::>(), declared); } } diff --git a/litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs b/litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs index 52c5e47f83f..91bbc072b4f 100644 --- a/litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs +++ b/litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs @@ -7,7 +7,7 @@ use pyo3::prelude::*; use pyo3::types::PyDict; use rstest::rstest; -use super::LegacyLogging; +use super::LegacyPythonLifecycle; use crate::test_support::{legacy_call, local, namespace, run}; const CALL: &CStr = c" @@ -24,7 +24,7 @@ fn begin<'py>( py: Python<'py>, locals: &Bound<'py, PyDict>, asynchronous: bool, -) -> (LegacyLogging, LifecycleStep) { +) -> (LegacyPythonLifecycle, LifecycleStep) { let mut logging = legacy_call(py, locals, asynchronous); let kwargs = local(locals, "kwargs") .cast_into::() @@ -63,6 +63,59 @@ fn deployment_pre_call_hook_runs_only_for_asynchronous_calls(#[case] asynchronou }); } +#[test] +fn failure_context_is_initialized_before_argument_preparation() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +failure = RuntimeError('limit check failed') +metadata = {'request': 'ocr-1'} + +class FailingLogger(StubLogger): + def update_from_kwargs(self, **update): + self.record('update', update) + self.update = update + + def check_limits(self, arguments): + self.record('check_limits', arguments) + raise failure + +logger = FailingLogger() +kwargs = { + 'logger': logger, + 'model': 'provider/model', + 'metadata': metadata, + 'litellm_call_id': 'call-1', +} +", + ); + let mut logging = legacy_call(py, &locals, false); + let kwargs = local(&locals, "kwargs") + .cast_into::() + .unwrap() + .unbind(); + let result = logging.begin(py, kwargs, 0.0); + let error = match result { + Err(error) => error, + Ok(_) => panic!("argument preparation succeeded"), + }; + assert!(error.value(py).is(local(&locals, "failure"))); + run( + py, + &locals, + c" +assert logger.names() == ['update', 'check_limits'], logger.calls +assert logger.update['model'] == 'model', logger.update +assert logger.update['custom_llm_provider'] == 'provider', logger.update +assert logger.update['kwargs']['metadata'] is metadata, logger.update +assert logger.update['litellm_params']['litellm_call_id'] == 'call-1', logger.update +", + ); + }); +} + #[test] fn kwargs_returned_by_the_pre_call_hook_are_what_the_call_prepares() { Python::initialize(); diff --git a/litellm-rust/crates/callbacks-legacy-python/tests/payload.rs b/litellm-rust/crates/callbacks-legacy-python/tests/payload.rs index 5459b36af27..0b832579aa2 100644 --- a/litellm-rust/crates/callbacks-legacy-python/tests/payload.rs +++ b/litellm-rust/crates/callbacks-legacy-python/tests/payload.rs @@ -8,7 +8,7 @@ use pyo3::prelude::*; use rstest::rstest; use serde_json::{Map, Value, json}; -use super::LegacyLogging; +use super::LegacyPythonLifecycle; use crate::PythonLogger; use crate::test_support::{legacy_call, local, namespace, run}; @@ -77,7 +77,7 @@ fn before_send_bound( locals.set_item(name, to_py(py, value).unwrap()).unwrap(); } run(py, &locals, script); - let mut logging = LegacyLogging { + let mut logging = LegacyPythonLifecycle { logger: Some(PythonLogger::new(local(&locals, "logger").unbind())), ..legacy_call(py, &locals, false) }; diff --git a/litellm-rust/crates/callbacks-legacy-python/tests/support.rs b/litellm-rust/crates/callbacks-legacy-python/tests/support.rs index d0c02fa6da5..245cdfc89bf 100644 --- a/litellm-rust/crates/callbacks-legacy-python/tests/support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/tests/support.rs @@ -3,7 +3,7 @@ use std::ffi::CStr; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; -use crate::{LegacyLogging, LegacySurface, PublicCall}; +use crate::{LegacyPythonLifecycle, LegacyPythonSurface, PublicCall}; /// The parameters of every `callbacks_legacy_python` function, as the real module declares them. /// `tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python @@ -125,6 +125,9 @@ class StubLogger: self.hooks = {} self.on_enqueue = lambda coroutine: None + def update_from_kwargs(self, **update): + self.update = update + def record(self, name, value): self.calls.append((name, value)) @@ -181,7 +184,7 @@ pub(crate) fn legacy_call( py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool, -) -> LegacyLogging { +) -> LegacyPythonLifecycle { let request = locals .get_item("request") .unwrap() @@ -192,9 +195,9 @@ pub(crate) fn legacy_call( .map(|kwargs| kwargs.cast_into::().unwrap()) .unwrap_or_else(|| PyDict::new(py)); let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap(); - LegacyLogging::new( + LegacyPythonLifecycle::new( py, - LegacySurface { + LegacyPythonSurface { call_type: "test", input_description: "test input", stream: None, diff --git a/litellm-rust/crates/callbacks-legacy-python/tests/terminal.rs b/litellm-rust/crates/callbacks-legacy-python/tests/terminal.rs index f68209233f2..6d1dedba4e1 100644 --- a/litellm-rust/crates/callbacks-legacy-python/tests/terminal.rs +++ b/litellm-rust/crates/callbacks-legacy-python/tests/terminal.rs @@ -8,7 +8,7 @@ use pyo3::prelude::*; use pyo3::types::PyDict; use rstest::rstest; -use super::LegacyLogging; +use super::LegacyPythonLifecycle; use crate::PythonLogger; use crate::test_support::{legacy_call, local, namespace, run}; @@ -17,8 +17,8 @@ const TIMING: Timing = Timing { end_time: 1.0, }; -fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> LegacyLogging { - LegacyLogging { +fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> LegacyPythonLifecycle { + LegacyPythonLifecycle { logger: Some(PythonLogger::new(local(locals, "logger").unbind())), ..legacy_call(py, locals, asynchronous) } @@ -27,7 +27,7 @@ fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> Leg fn succeed( py: Python<'_>, locals: &Bound<'_, PyDict>, - logging: &mut LegacyLogging, + logging: &mut LegacyPythonLifecycle, ) -> LifecycleStep { let response = local(locals, "response").unbind(); logging @@ -41,7 +41,11 @@ fn succeed( .unwrap() } -fn fail(py: Python<'_>, locals: &Bound<'_, PyDict>, logging: &mut LegacyLogging) -> LifecycleStep { +fn fail( + py: Python<'_>, + locals: &Bound<'_, PyDict>, + logging: &mut LegacyPythonLifecycle, +) -> LifecycleStep { let failure = PyErr::from_value(local(locals, "failure")); logging .emit( @@ -105,7 +109,7 @@ fn internal_calls_skip_failure_callbacks_only_when_asynchronous( Python::initialize(); Python::attach(|py| { let locals = namespace(py, c"failure = ValueError('provider')"); - let mut logging = LegacyLogging { + let mut logging = LegacyPythonLifecycle { internal: true, ..logged(py, &locals, asynchronous) }; @@ -127,7 +131,7 @@ fn internal_async_calls_skip_the_async_success_fan_out() { Python::initialize(); Python::attach(|py| { let locals = namespace(py, c"response = object()"); - let mut logging = LegacyLogging { + let mut logging = LegacyPythonLifecycle { internal: true, ..logged(py, &locals, true) }; diff --git a/litellm-rust/crates/callbacks-v1-python/AGENTS.md b/litellm-rust/crates/callbacks-v1-python/AGENTS.md new file mode 100644 index 00000000000..c28c89134d2 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/AGENTS.md @@ -0,0 +1,27 @@ +- Target invariants, not completion claims +- Keep this crate the Python runtime adapter for `litellm-callbacks-v1` and nothing else: the subscriber snapshot and one `PythonLifecycle` that delivers the core contract + - Envelopes, sequencing, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1` + - `litellm/rust_bridge/callbacks_v1_python.py` is the only Python module behind it; `python_contract.json` pins its functions on both sides +- Terms (contract terms are in `callbacks-v1/AGENTS.md`, lifecycle terms in `host-python/AGENTS.md`) + - Registry: the process-wide subscriber list; it lives in Python (`register`/`unregister`), never in Rust + - Snapshot: that list read once per call, with one observer and one interceptor handler already selected per subscriber for the call's mode (`subscribers.rs`) + - Projection: the public response turned into JSON by `project_response`; a failed projection becomes `response_error`, never a failed call + - Report: logging an observer's swallowed error through `report` + - Continuation: what a step returns to the driver once every observer of its envelope has run + - "Callback" in the Python module (`Callback` protocol) is the user's object; once registered it is a subscriber +- No Python object of the call reaches a callback; a callback influences a call only through a patch Rust validates + - `begin` returns the dict it was given by identity; `after_success` and `delivered` stay the trait's pass-through defaults + - Every callback gets a freshly built envelope per event; never cache or share a converted one + - Contract values cross into Python only through a fresh conversion per callback +- Schema v1 is owned by `litellm-callbacks-v1`; this crate only marshals it + - Everything rejectable is rejected at `register`, never during a call; the snapshot only parses and the lifecycle never re-validates a subscriber +- Observers cannot affect a call: ordinary exceptions are reported and swallowed at every event, awaited or not; an interceptor failure fails the call + - A cancellation is returned at once, with no further dispatch and no report + - Exactly one terminal envelope, last; none after a cancellation +- In a sync call no step suspends: handler selection happens once, at snapshot, from what registration recorded + - Every suspension is awaited inline in the caller's task; never `into_future`, never spawn +- The subscriber snapshot is taken once per call before the driver starts; later registrations do not reach that call + - `V1PythonLifecycle::subscribed` is the only constructor callers get; it returns `None` for an empty snapshot, so an unobserved call builds no envelope and projects no response + - This lifecycle may run after another one in a `LifecycleChain`; it observes the arguments and the wire as that one left them and never assumes it is alone +- `close` is idempotent; `traverse` visits every retained Python edge and calls no Python +- This crate has none of `callbacks-legacy-python`'s obligations and must not grow them; shared behaviour belongs to `PythonLifecycle`, not to either adapter diff --git a/litellm-rust/crates/callbacks-v1-python/Cargo.toml b/litellm-rust/crates/callbacks-v1-python/Cargo.toml new file mode 100644 index 00000000000..8fb27024a98 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "litellm-callbacks-v1-python" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +autotests = false + +[dependencies] +litellm-host.workspace = true +litellm-host-python.workspace = true +litellm-callbacks-v1.workspace = true +pyo3.workspace = true +serde_json.workspace = true +strum.workspace = true + +[dev-dependencies] +litellm-auth.workspace = true diff --git a/litellm-rust/crates/callbacks-v1-python/python_contract.json b/litellm-rust/crates/callbacks-v1-python/python_contract.json new file mode 100644 index 00000000000..d8326d13b60 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/python_contract.json @@ -0,0 +1,6 @@ +{ + "snapshot": [], + "project_response": ["response"], + "new_call_id": [], + "report": ["name", "event", "error"] +} diff --git a/litellm-rust/crates/callbacks-v1-python/src/adapter.rs b/litellm-rust/crates/callbacks-v1-python/src/adapter.rs new file mode 100644 index 00000000000..d834d2c69b9 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/src/adapter.rs @@ -0,0 +1,359 @@ +//! The v1 lifecycle: each step builds one envelope, hands every subscribed observer a +//! fresh conversion of it, and, before the request is sent, folds every interceptor's +//! patch into the wire request. + +use litellm_callbacks_v1::{ + CallFacts, Envelope, ErrorFacts, Event, RequestFacts, Sequencer, WirePatch, apply, +}; +use litellm_host::event::{MachineEvent, RequestContext, WireRequest}; +use litellm_host_python::{ + LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py, +}; +use pyo3::exceptions::PyValueError; +use pyo3::gc::{PyTraverseError, PyVisit}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyString}; +use serde_json::{Map, Value}; + +use crate::call::V1PythonSurface; +use crate::python::V1Python; +use crate::subscribers::Subscriber; + +/// What a step hands back to the driver once every observer of its envelope has run. +enum Continuation { + Done, + Arguments(Py), + Wire(Box), +} + +/// The suspended step; `index` is the subscriber whose awaitable the driver is awaiting. +enum Pending { + Observe { + envelope: Envelope, + index: usize, + then: Continuation, + }, + Intercept { + wire: Box, + context: RequestContext, + index: usize, + }, +} + +pub struct V1PythonLifecycle { + surface: V1PythonSurface, + subscribers: Vec, + asynchronous: bool, + sequencer: Option, + streamed: bool, + pending: Option, + closed: bool, +} + +impl Continuation { + fn into_step(self) -> LifecycleStep { + match self { + Self::Done => LifecycleStep::Done, + Self::Arguments(arguments) => LifecycleStep::Arguments(arguments), + Self::Wire(wire) => LifecycleStep::Wire(wire), + } + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + match self { + Self::Arguments(arguments) => visit.call(arguments), + Self::Done | Self::Wire(_) => Ok(()), + } + } +} + +impl V1PythonLifecycle { + pub(crate) fn new( + surface: V1PythonSurface, + subscribers: Vec, + asynchronous: bool, + ) -> Self { + Self { + surface, + subscribers, + asynchronous, + sequencer: None, + streamed: false, + pending: None, + closed: false, + } + } + + fn envelope(&mut self, event: Event) -> PyResult { + self.sequencer + .as_mut() + .map(|sequencer| sequencer.envelope(event)) + .ok_or_else(missing_state) + } + + fn report( + &self, + py: Python<'_>, + index: usize, + envelope: &Envelope, + error: PyErr, + ) -> PyResult<()> { + let name = &self.subscribers[index].name; + let event: &'static str = envelope.event.kind().into(); + match V1Python::Report.call(py, (name, event, error.value(py))) { + Err(report_error) if is_cancellation(py, &report_error) => Err(report_error), + _ => Ok(()), + } + } + + fn observe( + &mut self, + py: Python<'_>, + envelope: Envelope, + from: usize, + then: Continuation, + ) -> PyResult { + for index in from..self.subscribers.len() { + let Some(handler) = self.subscribers[index] + .observer(envelope.event.kind()) + .map(|handler| handler.clone_ref(py)) + else { + continue; + }; + let outcome = to_py(py, &envelope).and_then(|argument| handler.call(py, argument)); + match outcome { + Ok(awaitable) if handler.is_async() => { + self.pending = Some(Pending::Observe { + envelope, + index, + then, + }); + return Ok(LifecycleStep::Await(awaitable.unbind())); + } + Ok(_) => {} + Err(error) if is_cancellation(py, &error) => return Err(error), + Err(error) => self.report(py, index, &envelope, error)?, + } + } + Ok(then.into_step()) + } + + fn apply_patch( + &self, + py: Python<'_>, + index: usize, + wire: WireRequest, + result: Py, + ) -> PyResult { + let patch = if result.bind(py).is_none() { + WirePatch::default() + } else { + from_py::(result.bind(py))? + }; + let name = &self.subscribers[index].name; + apply(wire, patch) + .map_err(|error| PyValueError::new_err(format!("callback {name}: {error}"))) + } + + fn intercept( + &mut self, + py: Python<'_>, + wire: Box, + context: RequestContext, + from: usize, + ) -> PyResult { + let mut current = wire; + for index in from..self.subscribers.len() { + let Some(handler) = self.subscribers[index] + .intercept + .as_ref() + .map(|handler| handler.clone_ref(py)) + else { + continue; + }; + let request = to_py(py, &RequestFacts::new(¤t, &context))?; + let result = handler.call(py, request)?; + if handler.is_async() { + self.pending = Some(Pending::Intercept { + wire: current, + context, + index, + }); + return Ok(LifecycleStep::Await(result.unbind())); + } + current = Box::new(self.apply_patch(py, index, *current, result.unbind())?); + } + let envelope = + self.envelope(Event::RequestSending(RequestFacts::new(¤t, &context)))?; + self.observe(py, envelope, 0, Continuation::Wire(current)) + } + + fn metadata(arguments: &Bound<'_, PyDict>) -> PyResult<(Map, Vec)> { + let Some(value) = arguments.get_item("metadata")? else { + return Ok((Map::new(), Vec::new())); + }; + let Ok(metadata) = value.cast::() else { + return Ok((Map::new(), vec!["metadata".to_string()])); + }; + let mut included = Map::new(); + let mut dropped = Vec::new(); + for (key, value) in metadata.iter() { + match key.extract::() { + Ok(name) => match from_py::(&value) { + Ok(value) => { + included.insert(name, value); + } + Err(_) => dropped.push(name), + }, + Err(_) => dropped.push(key.str()?.to_string()), + } + } + Ok((included, dropped)) + } + + fn call_id(py: Python<'_>, arguments: &Bound<'_, PyDict>) -> PyResult { + if let Some(value) = arguments.get_item("litellm_call_id")? + && value.is_instance_of::() + { + return value.extract(); + } + V1Python::NewCallId.call(py, ())?.extract() + } + + fn error_facts(py: Python<'_>, error: &PyErr) -> PyResult { + let class = error.get_type(py); + let module = class.getattr("__module__")?.extract::()?; + let qualified = class.getattr("__qualname__")?.extract::()?; + let status_code = error + .value(py) + .getattr("status_code") + .ok() + .and_then(|value| value.extract::().ok()); + Ok(ErrorFacts { + class: format!("{module}.{qualified}"), + message: error.value(py).str()?.to_string(), + status_code, + }) + } +} + +impl PythonLifecycle for V1PythonLifecycle { + fn begin( + &mut self, + py: Python<'_>, + arguments: Py, + start_time: f64, + ) -> PyResult { + let call_id = Self::call_id(py, arguments.bind(py))?; + let (metadata, metadata_dropped) = Self::metadata(arguments.bind(py))?; + self.sequencer = Some(Sequencer::new(call_id, self.surface.call_type)); + let envelope = self.envelope(Event::CallStarted(CallFacts { + start_time, + asynchronous: self.asynchronous, + metadata, + metadata_dropped, + }))?; + self.observe(py, envelope, 0, Continuation::Arguments(arguments)) + } + + fn before_send( + &mut self, + py: Python<'_>, + wire: Box, + context: &RequestContext, + ) -> PyResult { + self.intercept(py, wire, context.clone(), 0) + } + + fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { + match event { + LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => { + let envelope = self.envelope(Event::ResponseReceived { + body: raw.body.clone(), + })?; + self.observe(py, envelope, 0, Continuation::Done) + } + LifecycleEvent::Succeeded { timing, response } => { + let projected = V1Python::ProjectResponse + .call(py, (response,)) + .and_then(|value| from_py::(&value)); + let (response, response_error) = match projected { + Ok(response) => (response, None), + Err(error) if is_cancellation(py, &error) => return Err(error), + Err(error) => (Value::Null, Some(error.to_string())), + }; + let envelope = self.envelope(Event::CallSucceeded { + timing: timing.into(), + streamed: self.streamed, + response, + response_error, + })?; + self.observe(py, envelope, 0, Continuation::Done) + } + LifecycleEvent::Failed { + timing, + origin, + error, + } => { + let facts = Self::error_facts(py, error)?; + let envelope = self.envelope(Event::CallFailed { + timing: timing.into(), + streamed: self.streamed, + origin: origin.into(), + error: facts, + })?; + self.observe(py, envelope, 0, Continuation::Done) + } + } + } + + fn opened(&mut self, _py: Python<'_>) -> PyResult<()> { + self.streamed = true; + Ok(()) + } + + fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult { + match self.pending.take().ok_or_else(missing_state)? { + Pending::Observe { + envelope, + index, + then, + } => { + if let Err(error) = result { + if is_cancellation(py, &error) { + return Err(error); + } + self.report(py, index, &envelope, error)?; + } + self.observe(py, envelope, index + 1, then) + } + Pending::Intercept { + wire, + context, + index, + } => { + let patched = self.apply_patch(py, index, *wire, result?)?; + self.intercept(py, Box::new(patched), context, index + 1) + } + } + } + + fn close(&mut self, _py: Python<'_>) { + if self.closed { + return; + } + self.closed = true; + self.pending = None; + self.subscribers.clear(); + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + for subscriber in &self.subscribers { + subscriber.traverse(visit)?; + } + match &self.pending { + Some(Pending::Observe { then, .. }) => then.traverse(visit), + Some(Pending::Intercept { .. }) | None => Ok(()), + } + } +} diff --git a/litellm-rust/crates/callbacks-v1-python/src/call.rs b/litellm-rust/crates/callbacks-v1-python/src/call.rs new file mode 100644 index 00000000000..5826b993446 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/src/call.rs @@ -0,0 +1,25 @@ +//! What a route tells the v1 contract about itself, and the one constructor callers get. + +use pyo3::prelude::*; + +use crate::adapter::V1PythonLifecycle; +use crate::subscribers::snapshot; + +#[derive(Clone, Copy, Debug)] +pub struct V1PythonSurface { + pub call_type: &'static str, +} + +impl V1PythonLifecycle { + /// The lifecycle for one call, over the subscribers registered right now; later + /// registrations do not reach that call. `None` when nothing is subscribed, so an + /// unobserved call builds no envelope at all. + pub fn subscribed( + py: Python<'_>, + surface: V1PythonSurface, + asynchronous: bool, + ) -> PyResult> { + let subscribers = snapshot(py, asynchronous)?; + Ok((!subscribers.is_empty()).then(|| Self::new(surface, subscribers, asynchronous))) + } +} diff --git a/litellm-rust/crates/callbacks-v1-python/src/lib.rs b/litellm-rust/crates/callbacks-v1-python/src/lib.rs new file mode 100644 index 00000000000..665034a07a5 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/src/lib.rs @@ -0,0 +1,20 @@ +//! The Python adapter of the v1 callback contract: the per-call subscriber snapshot and +//! one [`PythonLifecycle`](litellm_host_python::PythonLifecycle) that marshals +//! `litellm-callbacks-v1` envelopes and patches to Python handlers. The contract itself +//! lives in `litellm-callbacks-v1`. + +mod adapter; +mod call; +mod python; +mod subscribers; + +pub use adapter::V1PythonLifecycle; +pub use call::V1PythonSurface; + +#[cfg(test)] +#[path = "../tests/lifecycle.rs"] +mod lifecycle_tests; + +#[cfg(test)] +#[path = "../tests/support.rs"] +mod test_support; diff --git a/litellm-rust/crates/callbacks-v1-python/src/python.rs b/litellm-rust/crates/callbacks-v1-python/src/python.rs new file mode 100644 index 00000000000..0fd75cab0d2 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/src/python.rs @@ -0,0 +1,56 @@ +//! The Python contract of this crate: every function of +//! `litellm/rust_bridge/callbacks_v1_python.py` that Rust calls, by name. + +use pyo3::prelude::*; +use strum::{IntoStaticStr, VariantArray}; + +const MODULE: &str = "litellm.rust_bridge.callbacks_v1_python"; + +#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)] +pub(crate) enum V1Python { + #[strum(serialize = "snapshot")] + Snapshot, + #[strum(serialize = "project_response")] + ProjectResponse, + #[strum(serialize = "new_call_id")] + NewCallId, + #[strum(serialize = "report")] + Report, +} + +impl V1Python { + fn name(self) -> &'static str { + self.into() + } + + pub(crate) fn call<'py, A>(self, py: Python<'py>, args: A) -> PyResult> + where + A: pyo3::call::PyCallArgs<'py>, + { + py.import(MODULE)?.getattr(self.name())?.call1(args) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use strum::VariantArray; + + use super::V1Python; + use crate::test_support::PYTHON_CONTRACT; + + #[test] + fn every_function_is_in_the_python_contract() { + let contract: serde_json::Map = + serde_json::from_str(PYTHON_CONTRACT).unwrap(); + let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect(); + let called: Vec<&str> = V1Python::VARIANTS + .iter() + .copied() + .map(V1Python::name) + .collect(); + assert_eq!(called.len(), declared.len(), "a function is listed twice"); + assert_eq!(called.into_iter().collect::>(), declared); + } +} diff --git a/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs b/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs new file mode 100644 index 00000000000..a63e3aa842f --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/src/subscribers.rs @@ -0,0 +1,113 @@ +//! The per-call subscriber snapshot: what Python's `register` recorded, read once, with +//! one handler chosen per role for the call's execution mode. `register` has already +//! rejected everything rejectable, so this only parses. + +use std::collections::BTreeSet; + +use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, select_handler}; +use pyo3::exceptions::PyValueError; +use pyo3::gc::{PyTraverseError, PyVisit}; +use pyo3::prelude::*; + +use crate::python::V1Python; + +/// One callable of a subscriber, tagged with whether calling it returns an awaitable. +pub enum Handler { + Sync(Py), + Async(Py), +} + +pub struct Subscriber { + pub name: String, + pub events: BTreeSet, + /// `on_event` or `async_on_event`: cannot affect the call. + pub observe: Option, + /// `before_send` or `async_before_send`: returns a patch; its failure fails the call. + pub intercept: Option, +} + +fn selected_handler( + subscriber: &Bound<'_, PyAny>, + mode: ExecutionMode, + sync_name: &str, + async_name: &str, +) -> PyResult> { + let sync = subscriber.getattr(sync_name)?; + let asynchronous = subscriber.getattr(async_name)?; + Ok( + match select_handler(mode, !sync.is_none(), !asynchronous.is_none()) { + HandlerSelection::Sync => Some(Handler::Sync(sync.unbind())), + HandlerSelection::Async => Some(Handler::Async(asynchronous.unbind())), + HandlerSelection::Skip => None, + }, + ) +} + +pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult> { + let mode = if asynchronous { + ExecutionMode::Async + } else { + ExecutionMode::Sync + }; + V1Python::Snapshot + .call(py, ())? + .try_iter()? + .map(|item| { + let subscriber = item?; + let events = subscriber + .getattr("events")? + .extract::>()? + .into_iter() + .map(|name| { + name.parse::().map_err(|_| { + PyValueError::new_err(format!("unknown callback event: {name}")) + }) + }) + .collect::>>()?; + Ok(Subscriber { + name: subscriber.getattr("name")?.extract()?, + events, + observe: selected_handler(&subscriber, mode, "on_event", "async_on_event")?, + intercept: selected_handler(&subscriber, mode, "before_send", "async_before_send")?, + }) + }) + .collect() +} + +impl Subscriber { + pub fn observer(&self, kind: EventKind) -> Option<&Handler> { + self.observe + .as_ref() + .filter(|_| self.events.contains(&kind)) + } + + pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + for handler in self.observe.iter().chain(&self.intercept) { + visit.call(handler.object())?; + } + Ok(()) + } +} + +impl Handler { + fn object(&self) -> &Py { + match self { + Self::Sync(value) | Self::Async(value) => value, + } + } + + pub fn is_async(&self) -> bool { + matches!(self, Self::Async(_)) + } + + pub fn clone_ref(&self, py: Python<'_>) -> Self { + match self { + Self::Sync(value) => Self::Sync(value.clone_ref(py)), + Self::Async(value) => Self::Async(value.clone_ref(py)), + } + } + + pub fn call<'py>(&self, py: Python<'py>, argument: Py) -> PyResult> { + self.object().bind(py).call1((argument,)) + } +} diff --git a/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs b/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs new file mode 100644 index 00000000000..6d961e4ca9c --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs @@ -0,0 +1,497 @@ +use std::collections::BTreeSet; + +use litellm_auth::SecretValue; +use litellm_host::event::{MachineEvent, RawResponse, RequestContext, Timing, WireRequest}; +use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle, from_py}; +use pyo3::exceptions::asyncio::CancelledError; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use serde_json::{Value, json}; + +use crate::adapter::V1PythonLifecycle; +use crate::call::V1PythonSurface; +use crate::subscribers::{Handler, Subscriber}; +use crate::test_support::{local, namespace}; +use litellm_callbacks_v1::EventKind; + +const TIMING: Timing = Timing { + start_time: 10.0, + end_time: 12.5, +}; + +fn subscriber( + locals: &Bound<'_, PyDict>, + name: &str, + events: &[EventKind], + observe: Option<(&str, bool)>, + intercept: Option<(&str, bool)>, +) -> Subscriber { + let handler = |spec: Option<(&str, bool)>| { + spec.map(|(local_name, asynchronous)| { + let callable = local(locals, local_name).unbind(); + if asynchronous { + Handler::Async(callable) + } else { + Handler::Sync(callable) + } + }) + }; + Subscriber { + name: name.to_string(), + events: events.iter().copied().collect::>(), + observe: handler(observe), + intercept: handler(intercept), + } +} + +fn lifecycle(subscribers: Vec, asynchronous: bool) -> V1PythonLifecycle { + V1PythonLifecycle::new( + V1PythonSurface { call_type: "ocr" }, + subscribers, + asynchronous, + ) +} + +fn context(secret: &str) -> RequestContext { + RequestContext { + model: "model".to_string(), + custom_llm_provider: "provider".to_string(), + optional_params: json!({"api_key": secret, "temperature": 0.2}), + secret_fields: vec!["api_key".to_string()], + api_key: Some(SecretValue::new(secret.to_string())), + } +} + +fn wire(secret: &str) -> WireRequest { + WireRequest { + url: "https://provider.example/v1".to_string(), + headers: vec![ + ("Authorization".to_string(), secret.to_string()), + ("X-Trace".to_string(), "one".to_string()), + ], + body: json!({"input": "hello"}), + } +} + +#[test] +fn full_sync_success_preserves_identity_sequence_and_secrets() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +import copy +seen = [] +def observe(event): + seen.append(copy.deepcopy(event)) +response = {'id': 'response'} +", + ); + let events = [ + EventKind::CallStarted, + EventKind::RequestSending, + EventKind::ResponseReceived, + EventKind::CallSucceeded, + ]; + let observer = subscriber( + &locals, + "sync-success", + &events, + Some(("observe", false)), + None, + ); + let mut adapter = lifecycle(vec![observer], false); + let arguments = PyDict::new(py); + arguments.set_item("litellm_call_id", "call-123").unwrap(); + let LifecycleStep::Arguments(returned) = + adapter.begin(py, arguments.clone().unbind(), 10.0).unwrap() + else { + panic!("begin did not return arguments"); + }; + assert!(returned.bind(py).is(&arguments)); + + let secret = "sentinel-secret-78e4"; + let LifecycleStep::Wire(sent) = adapter + .before_send(py, Box::new(wire(secret)), &context(secret)) + .unwrap() + else { + panic!("before_send did not return wire"); + }; + assert_eq!(sent.headers[0].1, secret); + let raw = MachineEvent::ResponseReceived { + raw: RawResponse { + body: "raw response".to_string(), + }, + }; + assert!(matches!( + adapter.emit(py, LifecycleEvent::Machine(&raw)).unwrap(), + LifecycleStep::Done + )); + let response = local(&locals, "response").unbind(); + let LifecycleStep::Response(returned) = adapter + .after_success(py, response.clone_ref(py), TIMING) + .unwrap() + else { + panic!("after_success did not return response"); + }; + assert!(returned.bind(py).is(response.bind(py))); + assert!(matches!( + adapter + .emit( + py, + LifecycleEvent::Succeeded { + timing: TIMING, + response: &response, + }, + ) + .unwrap(), + LifecycleStep::Done + )); + + let seen = from_py::>(&local(&locals, "seen")).unwrap(); + assert_eq!( + seen.iter() + .map(|envelope| envelope["event"]["type"].as_str().unwrap()) + .collect::>(), + [ + "call.started", + "request.sending", + "response.received", + "call.succeeded", + ] + ); + assert_eq!( + seen.iter() + .map(|envelope| envelope["seq"].as_u64().unwrap()) + .collect::>(), + [0, 1, 2, 3] + ); + assert!(!serde_json::to_string(&seen).unwrap().contains(secret)); + }); +} + +#[test] +fn every_observer_gets_a_fresh_envelope() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +import copy +seen = [] +def mutate(event): + event['event']['body']['input'] = 'changed' +def record(event): + seen.append(copy.deepcopy(event)) +", + ); + let event = [EventKind::RequestSending]; + let subscribers = vec![ + subscriber(&locals, "mutator", &event, Some(("mutate", false)), None), + subscriber(&locals, "recorder", &event, Some(("record", false)), None), + ]; + let mut adapter = lifecycle(subscribers, false); + adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(); + let LifecycleStep::Wire(sent) = adapter + .before_send(py, Box::new(wire("secret")), &context("secret")) + .unwrap() + else { + panic!("before_send did not return wire"); + }; + assert_eq!(sent.body, json!({"input": "hello"})); + let seen = from_py::>(&local(&locals, "seen")).unwrap(); + assert_eq!(seen[0]["event"]["body"], json!({"input": "hello"})); + }); +} + +#[test] +fn ordinary_observer_errors_are_reported_and_swallowed() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +seen = [] +def fail(event): + raise ValueError('observer failed') +def record(event): + seen.append(event['event']['type']) +", + ); + let event = [EventKind::CallStarted]; + let subscribers = vec![ + subscriber( + &locals, + "observer-error-401", + &event, + Some(("fail", false)), + None, + ), + subscriber( + &locals, + "after-error", + &event, + Some(("record", false)), + None, + ), + ]; + let mut adapter = lifecycle(subscribers, false); + assert!(matches!( + adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(), + LifecycleStep::Arguments(_) + )); + assert_eq!( + local(&locals, "seen").extract::>().unwrap(), + ["call.started"] + ); + let reports = py + .import("litellm.rust_bridge.callbacks_v1_python") + .unwrap() + .getattr("reports") + .unwrap(); + let names = reports + .try_iter() + .unwrap() + .filter_map(Result::ok) + .filter_map(|report| report.get_item(0).ok()?.extract::().ok()) + .collect::>(); + assert!(names.contains(&"observer-error-401".to_string())); + }); +} + +#[test] +fn cancellation_stops_observer_fanout() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +import asyncio +seen = [] +def cancel(event): + raise asyncio.CancelledError() +def record(event): + seen.append(event) +", + ); + let event = [EventKind::CallStarted]; + let subscribers = vec![ + subscriber(&locals, "cancel", &event, Some(("cancel", false)), None), + subscriber( + &locals, + "after-cancel", + &event, + Some(("record", false)), + None, + ), + ]; + let mut adapter = lifecycle(subscribers, false); + let error = match adapter.begin(py, PyDict::new(py).unbind(), 10.0) { + Err(error) => error, + Ok(_) => panic!("cancellation was swallowed"), + }; + assert!(error.is_instance_of::(py)); + assert_eq!(local(&locals, "seen").len().unwrap(), 0); + }); +} + +#[test] +fn async_observers_resume_in_order_and_return_arguments() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +calls = [] +def first(event): + calls.append('first') + return object() +def second(event): + calls.append('second') + return object() +", + ); + let event = [EventKind::CallStarted]; + let subscribers = vec![ + subscriber(&locals, "async-first", &event, Some(("first", true)), None), + subscriber( + &locals, + "async-second", + &event, + Some(("second", true)), + None, + ), + ]; + let mut adapter = lifecycle(subscribers, true); + let arguments = PyDict::new(py).unbind(); + assert!(matches!( + adapter.begin(py, arguments.clone_ref(py), 10.0).unwrap(), + LifecycleStep::Await(_) + )); + assert!(matches!( + adapter.resume(py, Ok(py.None())).unwrap(), + LifecycleStep::Await(_) + )); + let LifecycleStep::Arguments(returned) = adapter.resume(py, Ok(py.None())).unwrap() else { + panic!("resume did not return arguments"); + }; + assert!(returned.bind(py).is(arguments.bind(py))); + assert_eq!( + local(&locals, "calls").extract::>().unwrap(), + ["first", "second"] + ); + }); +} + +#[test] +fn interceptors_chain_and_observers_see_the_final_wire() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +seen = [] +def first(request): + return {'headers': {'set': [['X-Added', 'yes']]}, 'body': {'step': 1}} +def second(request): + seen.append(request) + return None +def observe(event): + seen.append(event) +", + ); + let subscribers = vec![ + subscriber(&locals, "first", &[], None, Some(("first", false))), + subscriber(&locals, "second", &[], None, Some(("second", false))), + subscriber( + &locals, + "observer", + &[EventKind::RequestSending], + Some(("observe", false)), + None, + ), + ]; + let mut adapter = lifecycle(subscribers, false); + adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(); + let LifecycleStep::Wire(sent) = adapter + .before_send(py, Box::new(wire("secret")), &context("secret")) + .unwrap() + else { + panic!("before_send did not return wire"); + }; + assert_eq!(sent.body, json!({"step": 1})); + assert_eq!( + sent.headers.last().unwrap(), + &("X-Added".to_string(), "yes".to_string()) + ); + let seen = from_py::>(&local(&locals, "seen")).unwrap(); + assert_eq!(seen[0]["body"], json!({"step": 1})); + assert_eq!(seen[1]["event"]["body"], json!({"step": 1})); + }); +} + +#[test] +fn invalid_interceptor_patch_stops_later_handlers() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +seen = [] +def invalid(request): + return {'headers': {'set': [['Authorization', 'replacement']]}} +def later(request): + seen.append('later') +def observe(event): + seen.append('observe') +", + ); + let subscribers = vec![ + subscriber( + &locals, + "invalid-patch", + &[], + None, + Some(("invalid", false)), + ), + subscriber(&locals, "later", &[], None, Some(("later", false))), + subscriber( + &locals, + "observer", + &[EventKind::RequestSending], + Some(("observe", false)), + None, + ), + ]; + let mut adapter = lifecycle(subscribers, false); + adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(); + let error = match adapter.before_send(py, Box::new(wire("secret")), &context("secret")) { + Err(error) => error, + Ok(_) => panic!("invalid patch was accepted"), + }; + assert!(error.to_string().contains("invalid-patch")); + assert_eq!(local(&locals, "seen").len().unwrap(), 0); + }); +} + +#[test] +fn projection_failure_is_observed_without_failing_terminal_dispatch() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +seen = [] +def observe(event): + seen.append(event) +class Broken: + project_error = True +response = Broken() +", + ); + let observer = subscriber( + &locals, + "terminal", + &[EventKind::CallSucceeded], + Some(("observe", false)), + None, + ); + let mut adapter = lifecycle(vec![observer], false); + adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(); + adapter.opened(py).unwrap(); + let response = local(&locals, "response").unbind(); + assert!(matches!( + adapter + .emit( + py, + LifecycleEvent::Succeeded { + timing: TIMING, + response: &response, + }, + ) + .unwrap(), + LifecycleStep::Done + )); + let seen = from_py::>(&local(&locals, "seen")).unwrap(); + assert_eq!(seen[0]["event"]["streamed"], true); + assert!(seen[0]["event"]["response"].is_null()); + assert!( + seen[0]["event"]["response_error"] + .as_str() + .unwrap() + .contains("projection failed") + ); + }); +} + +#[test] +fn resume_without_pending_fails_and_close_is_idempotent() { + Python::initialize(); + Python::attach(|py| { + let _locals = namespace(py, c""); + let mut adapter = lifecycle(Vec::new(), false); + assert!(adapter.resume(py, Ok(py.None())).is_err()); + adapter.close(py); + adapter.close(py); + }); +} diff --git a/litellm-rust/crates/callbacks-v1-python/tests/support.rs b/litellm-rust/crates/callbacks-v1-python/tests/support.rs new file mode 100644 index 00000000000..62898c57e04 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1-python/tests/support.rs @@ -0,0 +1,58 @@ +use std::ffi::CStr; + +use pyo3::prelude::*; +use pyo3::types::PyDict; + +pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json"); + +const STUBS: &CStr = c" +import inspect +import json +import sys +import types + +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1_python'): + sys.modules.setdefault(name, types.ModuleType(name)) + +module = sys.modules['litellm.rust_bridge.callbacks_v1_python'] +contract = json.loads(python_contract) + +def contracted(name, fake): + signature = inspect.Signature([ + inspect.Parameter(parameter, inspect.Parameter.POSITIONAL_OR_KEYWORD) + for parameter in contract[name] + ]) + def checked(*args, **kwargs): + signature.bind(*args, **kwargs) + return fake(*args, **kwargs) + return checked + +if not hasattr(module, 'reports'): + module.reports = [] + +def project_response(response): + if getattr(response, 'project_error', False): + raise ValueError('projection failed') + return response + +functions = { + 'snapshot': lambda: (), + 'project_response': project_response, + 'new_call_id': lambda: 'generated-call-id', + 'report': lambda name, event, error: module.reports.append((name, event, error)), +} +for name, function in functions.items(): + setattr(module, name, contracted(name, function)) +"; + +pub(crate) fn namespace<'py>(py: Python<'py>, script: &CStr) -> Bound<'py, PyDict> { + let locals = PyDict::new(py); + locals.set_item("python_contract", PYTHON_CONTRACT).unwrap(); + py.run(STUBS, Some(&locals), Some(&locals)).unwrap(); + py.run(script, Some(&locals), Some(&locals)).unwrap(); + locals +} + +pub(crate) fn local<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py, PyAny> { + locals.get_item(name).unwrap().unwrap() +} diff --git a/litellm-rust/crates/callbacks-v1/AGENTS.md b/litellm-rust/crates/callbacks-v1/AGENTS.md new file mode 100644 index 00000000000..2b989ffa1e6 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/AGENTS.md @@ -0,0 +1,17 @@ +- Target invariants, not completion claims +- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction, handler selection and validated wire patches + - This crate may depend on `litellm-host`, but never on `litellm-host-python`, PyO3 or a runtime registry + - `golden/v1` pins every serialized envelope shape; every `EventKind` has a golden file by construction, and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are tested against those files +- Terms (shared ones are in `host-python/AGENTS.md`) + - Subscriber: one registered consumer of the contract: a name, a schema, the event kinds it wants and its handlers + - Handler: one callable of a subscriber; an observer receives envelopes and cannot affect the call; an interceptor receives `RequestFacts` and returns a patch + - Handler selection (`select_handler`): an async call prefers the async handler and falls back to the sync one; a sync call never selects an async handler + - Event: one thing that happened in a call; `EventKind` is its dotted name (`call.started`, `request.sending`, `response.received`, `call.succeeded`, `call.failed`), the serde tag and the subscription name at once + - Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; `Sequencer` numbers a call's envelopes from zero + - Facts (`CallFacts`, `RequestFacts`, `TimingFacts`, `ErrorFacts`): a redacted, serializable projection of runtime state; never a live object + - Patch (`WirePatch`): an interceptor's edit of the wire: header `remove`, then header `set`, optional `body` replacement; `apply` validates it + - Redaction: credential headers by name (`CREDENTIAL_HEADERS`) and optional params by the route's secret fields become `[REDACTED]`; shape never changes +- No runtime object of a call reaches a callback contract value +- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction; its fields are private so no other crate can build one +- A patch cannot change the URL or set or remove a credential header +- Schema v1 is additive-only; a rename, removal or retype requires a new contract version diff --git a/litellm-rust/crates/callbacks-v1/Cargo.toml b/litellm-rust/crates/callbacks-v1/Cargo.toml new file mode 100644 index 00000000000..196c2f58ebb --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "litellm-callbacks-v1" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +autotests = false + +[dependencies] +litellm-host.workspace = true +serde.workspace = true +serde_json.workspace = true +strum.workspace = true +thiserror.workspace = true + +[dev-dependencies] +litellm-auth.workspace = true +proptest.workspace = true +rstest.workspace = true diff --git a/litellm-rust/crates/callbacks-v1/golden/v1/call.failed.json b/litellm-rust/crates/callbacks-v1/golden/v1/call.failed.json new file mode 100644 index 00000000000..a55bf17a0bd --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/golden/v1/call.failed.json @@ -0,0 +1,13 @@ +{ + "schema": 1, + "call_id": "call-123", + "call_type": "ocr", + "seq": 4, + "event": { + "type": "call.failed", + "timing": {"start_time": 10.0, "end_time": 11.0, "duration": 1.0}, + "streamed": true, + "origin": "call", + "error": {"class": "builtins.ValueError", "message": "bad request", "status_code": 400} + } +} diff --git a/litellm-rust/crates/callbacks-v1/golden/v1/call.started.json b/litellm-rust/crates/callbacks-v1/golden/v1/call.started.json new file mode 100644 index 00000000000..a5e66498fe1 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/golden/v1/call.started.json @@ -0,0 +1,13 @@ +{ + "schema": 1, + "call_id": "call-123", + "call_type": "ocr", + "seq": 0, + "event": { + "type": "call.started", + "start_time": 10.0, + "asynchronous": true, + "metadata": {"team": "core"}, + "metadata_dropped": ["broken"] + } +} diff --git a/litellm-rust/crates/callbacks-v1/golden/v1/call.succeeded.json b/litellm-rust/crates/callbacks-v1/golden/v1/call.succeeded.json new file mode 100644 index 00000000000..b97ba1be14f --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/golden/v1/call.succeeded.json @@ -0,0 +1,13 @@ +{ + "schema": 1, + "call_id": "call-123", + "call_type": "ocr", + "seq": 3, + "event": { + "type": "call.succeeded", + "timing": {"start_time": 10.0, "end_time": 12.5, "duration": 2.5}, + "streamed": false, + "response": {"id": "response"}, + "response_error": null + } +} diff --git a/litellm-rust/crates/callbacks-v1/golden/v1/request.sending.json b/litellm-rust/crates/callbacks-v1/golden/v1/request.sending.json new file mode 100644 index 00000000000..98531a19d68 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/golden/v1/request.sending.json @@ -0,0 +1,15 @@ +{ + "schema": 1, + "call_id": "call-123", + "call_type": "ocr", + "seq": 1, + "event": { + "type": "request.sending", + "model": "model", + "custom_llm_provider": "provider", + "optional_params": {"temperature": 0.2}, + "url": "https://provider.example/v1", + "headers": [["Authorization", "[REDACTED]"]], + "body": {"input": "hello"} + } +} diff --git a/litellm-rust/crates/callbacks-v1/golden/v1/response.received.json b/litellm-rust/crates/callbacks-v1/golden/v1/response.received.json new file mode 100644 index 00000000000..0e43de8c8f9 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/golden/v1/response.received.json @@ -0,0 +1,10 @@ +{ + "schema": 1, + "call_id": "call-123", + "call_type": "ocr", + "seq": 2, + "event": { + "type": "response.received", + "body": "raw response" + } +} diff --git a/litellm-rust/crates/callbacks-v1/src/envelope.rs b/litellm-rust/crates/callbacks-v1/src/envelope.rs new file mode 100644 index 00000000000..7f78b999a32 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/src/envelope.rs @@ -0,0 +1,315 @@ +//! The v1 envelope: what a subscriber receives. An [`Envelope`] is the schema version, the +//! call's identity, a sequence number and one [`Event`]; every `*Facts` type is a +//! redacted, serializable projection that never holds a runtime object of the call. + +use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest}; +use serde::Serialize; +use serde_json::{Map, Value}; +use strum::{EnumString, IntoStaticStr, VariantArray}; + +use crate::redact; + +pub const SCHEMA_V1: u32 = 1; + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Envelope { + pub schema: u32, + pub call_id: String, + pub call_type: &'static str, + pub seq: u32, + pub event: Event, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "type")] +pub enum Event { + #[serde(rename = "call.started")] + CallStarted(CallFacts), + #[serde(rename = "request.sending")] + RequestSending(RequestFacts), + #[serde(rename = "response.received")] + ResponseReceived { body: String }, + #[serde(rename = "call.succeeded")] + CallSucceeded { + timing: TimingFacts, + streamed: bool, + response: Value, + response_error: Option, + }, + #[serde(rename = "call.failed")] + CallFailed { + timing: TimingFacts, + streamed: bool, + origin: Origin, + error: ErrorFacts, + }, +} + +#[derive( + EnumString, + IntoStaticStr, + VariantArray, + Clone, + Copy, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Debug, +)] +pub enum EventKind { + #[strum(serialize = "call.started")] + CallStarted, + #[strum(serialize = "request.sending")] + RequestSending, + #[strum(serialize = "response.received")] + ResponseReceived, + #[strum(serialize = "call.succeeded")] + CallSucceeded, + #[strum(serialize = "call.failed")] + CallFailed, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct CallFacts { + pub start_time: f64, + pub asynchronous: bool, + pub metadata: Map, + pub metadata_dropped: Vec, +} + +/// The request as a subscriber may see it. The fields are private so that +/// [`RequestFacts::new`], which redacts, is the only way to build one. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct RequestFacts { + model: String, + custom_llm_provider: String, + optional_params: Value, + url: String, + headers: Vec<(String, String)>, + body: Value, +} + +impl RequestFacts { + pub fn new(wire: &WireRequest, context: &RequestContext) -> Self { + Self { + model: context.model.clone(), + custom_llm_provider: context.custom_llm_provider.clone(), + optional_params: redact::params(&context.optional_params, &context.secret_fields), + url: wire.url.clone(), + headers: redact::headers(&wire.headers), + body: wire.body.clone(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Serialize)] +pub struct TimingFacts { + pub start_time: f64, + pub end_time: f64, + pub duration: f64, +} + +impl From for TimingFacts { + fn from(timing: Timing) -> Self { + Self { + start_time: timing.start_time, + end_time: timing.end_time, + duration: timing.end_time - timing.start_time, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Origin { + Call, + Host, +} + +impl From for Origin { + fn from(origin: FailureOrigin) -> Self { + match origin { + FailureOrigin::Call => Self::Call, + FailureOrigin::Host => Self::Host, + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ErrorFacts { + pub class: String, + pub message: String, + pub status_code: Option, +} + +impl Event { + pub fn kind(&self) -> EventKind { + match self { + Self::CallStarted(_) => EventKind::CallStarted, + Self::RequestSending(_) => EventKind::RequestSending, + Self::ResponseReceived { .. } => EventKind::ResponseReceived, + Self::CallSucceeded { .. } => EventKind::CallSucceeded, + Self::CallFailed { .. } => EventKind::CallFailed, + } + } +} + +pub struct Sequencer { + call_id: String, + call_type: &'static str, + next: u32, +} + +impl Sequencer { + pub fn new(call_id: String, call_type: &'static str) -> Self { + Self { + call_id, + call_type, + next: 0, + } + } + + pub fn envelope(&mut self, event: Event) -> Envelope { + let envelope = Envelope { + schema: SCHEMA_V1, + call_id: self.call_id.clone(), + call_type: self.call_type, + seq: self.next, + event, + }; + self.next += 1; + envelope + } +} + +#[cfg(test)] +mod tests { + use litellm_auth::SecretValue; + use proptest::prelude::*; + use serde_json::json; + use strum::VariantArray; + + use super::*; + + fn event(kind: EventKind) -> Event { + match kind { + EventKind::CallStarted => Event::CallStarted(CallFacts { + start_time: 10.0, + asynchronous: true, + metadata: Map::from_iter([("team".to_string(), json!("core"))]), + metadata_dropped: vec!["broken".to_string()], + }), + EventKind::RequestSending => Event::RequestSending(RequestFacts { + model: "model".to_string(), + custom_llm_provider: "provider".to_string(), + optional_params: json!({"temperature": 0.2}), + url: "https://provider.example/v1".to_string(), + headers: vec![("Authorization".to_string(), redact::REDACTED.to_string())], + body: json!({"input": "hello"}), + }), + EventKind::ResponseReceived => Event::ResponseReceived { + body: "raw response".to_string(), + }, + EventKind::CallSucceeded => Event::CallSucceeded { + timing: TimingFacts { + start_time: 10.0, + end_time: 12.5, + duration: 2.5, + }, + streamed: false, + response: json!({"id": "response"}), + response_error: None, + }, + EventKind::CallFailed => Event::CallFailed { + timing: TimingFacts { + start_time: 10.0, + end_time: 11.0, + duration: 1.0, + }, + streamed: true, + origin: Origin::Call, + error: ErrorFacts { + class: "builtins.ValueError".to_string(), + message: "bad request".to_string(), + status_code: Some(400), + }, + }, + } + } + + fn golden(kind: EventKind, seq: u32) -> Envelope { + Envelope { + schema: SCHEMA_V1, + call_id: "call-123".to_string(), + call_type: "ocr", + seq, + event: event(kind), + } + } + + /// Exhaustive on purpose: a new event kind does not compile without a golden file, + /// and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are pinned to the golden files. + fn golden_json(kind: EventKind) -> &'static str { + match kind { + EventKind::CallStarted => include_str!("../golden/v1/call.started.json"), + EventKind::RequestSending => include_str!("../golden/v1/request.sending.json"), + EventKind::ResponseReceived => include_str!("../golden/v1/response.received.json"), + EventKind::CallSucceeded => include_str!("../golden/v1/call.succeeded.json"), + EventKind::CallFailed => include_str!("../golden/v1/call.failed.json"), + } + } + + #[test] + fn serialized_v1_envelopes_match_golden_contract() { + for (seq, &kind) in EventKind::VARIANTS.iter().enumerate() { + assert_eq!( + serde_json::to_value(golden(kind, seq as u32)).unwrap(), + serde_json::from_str::(golden_json(kind)).unwrap() + ); + } + } + + #[test] + fn event_tags_equal_subscription_names() { + for &kind in EventKind::VARIANTS { + let value = serde_json::to_value(event(kind)).unwrap(); + let expected: &'static str = kind.into(); + assert_eq!(value["type"], expected); + } + } + + #[test] + fn request_facts_never_serialize_credentials() { + let sentinel = "sentinel-secret-78e4"; + let wire = WireRequest { + url: "https://provider.example".to_string(), + headers: vec![("Authorization".to_string(), sentinel.to_string())], + body: json!({"input": "safe"}), + }; + let context = RequestContext { + model: "model".to_string(), + custom_llm_provider: "provider".to_string(), + optional_params: json!({"api_key": sentinel}), + secret_fields: vec!["api_key".to_string()], + api_key: Some(SecretValue::new(sentinel.to_string())), + }; + assert!( + !serde_json::to_string(&RequestFacts::new(&wire, &context)) + .unwrap() + .contains(sentinel) + ); + } + + proptest! { + #[test] + fn sequence_numbers_increase_from_zero(length in 0usize..100) { + let mut sequencer = Sequencer::new("call".to_string(), "ocr"); + let sequence: Vec = (0..length) + .map(|_| sequencer.envelope(event(EventKind::ResponseReceived)).seq) + .collect(); + prop_assert_eq!(sequence, (0..length as u32).collect::>()); + } + } +} diff --git a/litellm-rust/crates/callbacks-v1/src/lib.rs b/litellm-rust/crates/callbacks-v1/src/lib.rs new file mode 100644 index 00000000000..2d5b27df591 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/src/lib.rs @@ -0,0 +1,15 @@ +//! The runtime-neutral v1 callback contract: envelopes and their sequencing, credential +//! redaction, handler selection and validated wire patches. No PyO3 and no registry; a +//! language host marshals these values to its own subscribers. + +mod envelope; +mod patch; +mod policy; +mod redact; + +pub use envelope::{ + CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, Sequencer, + TimingFacts, +}; +pub use patch::{HeaderPatch, PatchError, WirePatch, apply}; +pub use policy::{ExecutionMode, HandlerSelection, select_handler}; diff --git a/litellm-rust/crates/callbacks-v1/src/patch.rs b/litellm-rust/crates/callbacks-v1/src/patch.rs new file mode 100644 index 00000000000..50505536885 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/src/patch.rs @@ -0,0 +1,209 @@ +//! The v1 patch: the only way an interceptor changes a request. A [`WirePatch`] removes +//! and sets headers and may replace the body; it cannot name the URL, and [`apply`] +//! rejects any credential header. + +use litellm_host::event::WireRequest; +use serde::Deserialize; +use serde_json::Value; +use thiserror::Error; + +use crate::redact::is_credential_header; + +#[derive(Debug, Default, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WirePatch { + #[serde(default)] + pub headers: HeaderPatch, + #[serde(default)] + pub body: Option, +} + +#[derive(Debug, Default, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HeaderPatch { + #[serde(default)] + pub remove: Vec, + #[serde(default)] + pub set: Vec<(String, String)>, +} + +#[derive(Debug, PartialEq, Error)] +pub enum PatchError { + #[error("protected credential header: {0}")] + ProtectedHeader(String), + #[error("invalid header name: {0}")] + InvalidHeaderName(String), + #[error("invalid header value for: {0}")] + InvalidHeaderValue(String), +} + +fn valid_header_name(name: &str) -> bool { + !name.is_empty() + && name.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' + ) + }) +} + +fn validate_name(name: &str) -> Result<(), PatchError> { + if is_credential_header(name) { + return Err(PatchError::ProtectedHeader(name.to_string())); + } + if !valid_header_name(name) { + return Err(PatchError::InvalidHeaderName(name.to_string())); + } + Ok(()) +} + +pub fn apply(wire: WireRequest, patch: WirePatch) -> Result { + for name in &patch.headers.remove { + validate_name(name)?; + } + for (name, value) in &patch.headers.set { + validate_name(name)?; + if value + .bytes() + .any(|byte| matches!(byte, b'\r' | b'\n' | b'\0')) + { + return Err(PatchError::InvalidHeaderValue(name.clone())); + } + } + + let HeaderPatch { remove, set } = patch.headers; + let retained: Vec<(String, String)> = wire + .headers + .into_iter() + .filter(|(name, _)| { + !remove + .iter() + .chain(set.iter().map(|(name, _)| name)) + .any(|changed| name.eq_ignore_ascii_case(changed)) + }) + .collect(); + let headers = retained.into_iter().chain(set).collect(); + Ok(WireRequest { + url: wire.url, + headers, + body: patch.body.unwrap_or(wire.body), + }) +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn wire() -> WireRequest { + WireRequest { + url: "https://provider.example/v1".to_string(), + headers: vec![ + ("Authorization".to_string(), "secret".to_string()), + ("X-Trace".to_string(), "one".to_string()), + ("x-trace".to_string(), "two".to_string()), + ("Keep".to_string(), "yes".to_string()), + ], + body: json!({"old": true}), + } + } + + #[test] + fn default_patch_is_identity() { + let original = wire(); + assert_eq!(apply(original.clone(), WirePatch::default()), Ok(original)); + } + + #[test] + fn remove_precedes_set_and_names_match_case_insensitively() { + let patch = WirePatch { + headers: HeaderPatch { + remove: vec!["keep".to_string()], + set: vec![("X-TRACE".to_string(), "three".to_string())], + }, + body: Some(json!({"new": true})), + }; + let result = apply(wire(), patch).unwrap(); + assert_eq!( + result.headers, + vec![ + ("Authorization".to_string(), "secret".to_string()), + ("X-TRACE".to_string(), "three".to_string()), + ] + ); + assert_eq!(result.body, json!({"new": true})); + assert_eq!(result.url, "https://provider.example/v1"); + } + + #[rstest] + #[case(json!({"url": "https://attacker.example"}))] + #[case(json!({"headers": {"append": []}}))] + fn unknown_patch_fields_are_rejected(#[case] value: Value) { + assert!(serde_json::from_value::(value).is_err()); + } + + #[rstest] + #[case("Authorization", "value", PatchError::ProtectedHeader("Authorization".to_string()))] + #[case("bad name", "value", PatchError::InvalidHeaderName("bad name".to_string()))] + #[case("X-Test", "bad\r\nvalue", PatchError::InvalidHeaderValue("X-Test".to_string()))] + fn invalid_set_is_rejected( + #[case] name: &str, + #[case] value: &str, + #[case] expected: PatchError, + ) { + let patch = WirePatch { + headers: HeaderPatch { + remove: vec![], + set: vec![(name.to_string(), value.to_string())], + }, + body: None, + }; + assert_eq!(apply(wire(), patch), Err(expected)); + } + + #[test] + fn credential_headers_cannot_be_removed() { + let patch = WirePatch { + headers: HeaderPatch { + remove: vec!["AUTHORIZATION".to_string()], + set: vec![], + }, + body: None, + }; + assert_eq!( + apply(wire(), patch), + Err(PatchError::ProtectedHeader("AUTHORIZATION".to_string())) + ); + } + + proptest! { + #[test] + fn successful_patches_preserve_url(value in "[A-Za-z0-9]{0,20}") { + let original = wire(); + let patch = WirePatch { + headers: HeaderPatch { remove: vec![], set: vec![("X-Safe".to_string(), value)] }, + body: None, + }; + let result = apply(original.clone(), patch).unwrap(); + prop_assert_eq!(result.url, original.url); + prop_assert_eq!(result.headers[0].clone(), original.headers[0].clone()); + } + } +} diff --git a/litellm-rust/crates/callbacks-v1/src/policy.rs b/litellm-rust/crates/callbacks-v1/src/policy.rs new file mode 100644 index 00000000000..07ff392af05 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/src/policy.rs @@ -0,0 +1,47 @@ +//! Which of a subscriber's handlers a call uses: an async call prefers the async handler +//! and falls back to the sync one; a sync call never selects an async handler. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExecutionMode { + Sync, + Async, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HandlerSelection { + Sync, + Async, + Skip, +} + +pub fn select_handler(mode: ExecutionMode, has_sync: bool, has_async: bool) -> HandlerSelection { + match (mode, has_sync, has_async) { + (ExecutionMode::Async, _, true) => HandlerSelection::Async, + (_, true, _) => HandlerSelection::Sync, + _ => HandlerSelection::Skip, + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case(ExecutionMode::Async, true, true, HandlerSelection::Async)] + #[case(ExecutionMode::Async, false, true, HandlerSelection::Async)] + #[case(ExecutionMode::Async, true, false, HandlerSelection::Sync)] + #[case(ExecutionMode::Sync, true, true, HandlerSelection::Sync)] + #[case(ExecutionMode::Sync, true, false, HandlerSelection::Sync)] + #[case(ExecutionMode::Sync, false, true, HandlerSelection::Skip)] + #[case(ExecutionMode::Sync, false, false, HandlerSelection::Skip)] + fn handler_selection_never_selects_async_for_sync_calls( + #[case] mode: ExecutionMode, + #[case] has_sync: bool, + #[case] has_async: bool, + #[case] expected: HandlerSelection, + ) { + assert_eq!(select_handler(mode, has_sync, has_async), expected); + } +} diff --git a/litellm-rust/crates/callbacks-v1/src/redact.rs b/litellm-rust/crates/callbacks-v1/src/redact.rs new file mode 100644 index 00000000000..dacfc304086 --- /dev/null +++ b/litellm-rust/crates/callbacks-v1/src/redact.rs @@ -0,0 +1,104 @@ +//! The contract's credential redaction: credential headers by name, optional params by +//! the route's secret fields. Redaction replaces values and never changes shape. + +use serde_json::{Map, Value}; + +pub const REDACTED: &str = "[REDACTED]"; + +pub const CREDENTIAL_HEADERS: &[&str] = &[ + "authorization", + "proxy-authorization", + "x-api-key", + "api-key", + "x-goog-api-key", + "x-amz-security-token", + "cookie", +]; + +pub fn is_credential_header(name: &str) -> bool { + CREDENTIAL_HEADERS + .iter() + .any(|credential| name.eq_ignore_ascii_case(credential)) +} + +pub fn headers(values: &[(String, String)]) -> Vec<(String, String)> { + values + .iter() + .map(|(name, value)| { + let redacted = if is_credential_header(name) { + REDACTED.to_string() + } else { + value.clone() + }; + (name.clone(), redacted) + }) + .collect() +} + +pub fn params(value: &Value, secret_fields: &[String]) -> Value { + let Value::Object(object) = value else { + return value.clone(); + }; + Value::Object( + object + .iter() + .map(|(name, value)| { + let redacted = if secret_fields.iter().any(|secret| secret == name) { + Value::String(REDACTED.to_string()) + } else { + value.clone() + }; + (name.clone(), redacted) + }) + .collect::>(), + ) +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + use serde_json::json; + + use super::*; + + #[test] + fn credential_header_matching_is_ascii_case_insensitive() { + assert!(is_credential_header("AuThOrIzAtIoN")); + assert!(!is_credential_header("x-request-id")); + } + + #[test] + fn redaction_replaces_values_without_changing_shape() { + let input = vec![ + ("X-Request-ID".to_string(), "abc".to_string()), + ("Authorization".to_string(), "secret".to_string()), + ("authorization".to_string(), "other".to_string()), + ]; + assert_eq!( + headers(&input), + vec![ + ("X-Request-ID".to_string(), "abc".to_string()), + ("Authorization".to_string(), REDACTED.to_string()), + ("authorization".to_string(), REDACTED.to_string()), + ] + ); + assert_eq!( + params( + &json!({"api_key": "secret", "model": "m"}), + &["api_key".to_string()] + ), + json!({"api_key": REDACTED, "model": "m"}) + ); + } + + proptest! { + #[test] + fn redaction_is_idempotent( + name in "[A-Za-z0-9-]{1,20}", + value in "[A-Za-z0-9]{0,20}" + ) { + let input = vec![(name, value)]; + prop_assert_eq!(headers(&headers(&input)), headers(&input)); + } + } +} diff --git a/litellm-rust/crates/core-utils/Cargo.toml b/litellm-rust/crates/core-utils/Cargo.toml index eb353bc060c..0ee47b36c1c 100644 --- a/litellm-rust/crates/core-utils/Cargo.toml +++ b/litellm-rust/crates/core-utils/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [dependencies] fancy-regex.workspace = true litellm-types.workspace = true +percent-encoding.workspace = true serde.workspace = true serde_json.workspace = true serde_path_to_error = "0.1" diff --git a/litellm-rust/crates/core-utils/src/url_utils.rs b/litellm-rust/crates/core-utils/src/url_utils.rs index 1f690752c7a..77d959b1185 100644 --- a/litellm-rust/crates/core-utils/src/url_utils.rs +++ b/litellm-rust/crates/core-utils/src/url_utils.rs @@ -1,5 +1,6 @@ use std::marker::PhantomData; +use percent_encoding::{AsciiSet, utf8_percent_encode}; use url::Url; #[derive(Debug, thiserror::Error)] @@ -64,6 +65,35 @@ impl ApiUrl { self } + /// Like [`Self::append_query_pairs`], but percent-encodes keys and values with `escape` + /// instead of form encoding, so callers choose which reserved characters stay literal. + /// An existing query is kept; an empty result leaves no dangling `?`. + pub fn append_query_pairs_escaped<'a>( + mut self, + pairs: impl IntoIterator, + escape: &'static AsciiSet, + ) -> Self { + let appended = pairs.into_iter().map(|(key, value)| { + format!( + "{}={}", + utf8_percent_encode(key, escape), + utf8_percent_encode(value, escape) + ) + }); + let query = self + .url + .query() + .filter(|existing| !existing.is_empty()) + .map(str::to_owned) + .into_iter() + .chain(appended) + .collect::>() + .join("&"); + self.url + .set_query(Some(query.as_str()).filter(|query| !query.is_empty())); + self + } + pub fn into_string(self) -> String { self.url.into() } @@ -111,4 +141,41 @@ mod tests { "https://example.test/analyze?model=name+with+spaces" ); } + + const KEEP_COMMAS: &AsciiSet = &percent_encoding::NON_ALPHANUMERIC.remove(b','); + + #[rstest::rstest] + #[case::fresh_query("https://example.test", &[("pages", "1,2")], "?pages=1,2")] + #[case::existing_query_kept( + "https://example.test?tenant=a", + &[("pages", "1,2")], + "?tenant=a&pages=1,2" + )] + #[case::bare_question_mark("https://example.test?", &[("pages", "1")], "?pages=1")] + #[case::no_pairs_no_query("https://example.test", &[], "")] + #[case::no_pairs_keeps_query("https://example.test?tenant=a", &[], "?tenant=a")] + #[case::outside_set_escaped( + "https://example.test", + &[("q", "a b&c=d+e%2C#")], + "?q=a%20b%26c%3Dd%2Be%252C%23" + )] + #[case::keys_escaped_too("https://example.test", &[("a&b", "1")], "?a%26b=1")] + #[case::non_ascii_as_utf8("https://example.test", &[("q", "é")], "?q=%C3%A9")] + fn escaped_query_pairs( + #[case] base: &str, + #[case] pairs: &[(&str, &str)], + #[case] expected_query: &str, + ) { + let actual = ApiUrl::parse(base) + .and_then(|url| url.complete_path(&["analyze"])) + .map(|url| { + url.append_query_pairs_escaped(pairs.iter().copied(), KEEP_COMMAS) + .into_string() + }) + .expect("url builds"); + assert_eq!( + actual, + format!("https://example.test/analyze{expected_query}") + ); + } } diff --git a/litellm-rust/crates/core/src/ocr/provider_config.rs b/litellm-rust/crates/core/src/ocr/provider_config.rs index d38d87b92cc..ab3a59669b6 100644 --- a/litellm-rust/crates/core/src/ocr/provider_config.rs +++ b/litellm-rust/crates/core/src/ocr/provider_config.rs @@ -1,4 +1,4 @@ -use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; +use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; use litellm_llms::{ aws_textract::ocr::{ analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation, @@ -168,11 +168,8 @@ pub(crate) fn resolve_provider_config( model: &str, custom_llm_provider: Option<&str>, ) -> Result<(String, OcrConfigKind), Error> { - let provider = - get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider { - model, - custom_llm_provider: OcrProvider::Mistral.into(), - }); + let provider = get_custom_llm_provider(model, custom_llm_provider) + .ok_or_else(|| Error::ProviderNotProvided(model.to_string()))?; let ocr_provider = provider .custom_llm_provider .parse::() @@ -482,6 +479,15 @@ mod tests { ); } + #[rstest] + #[case::bare("mistral-ocr-latest")] + #[case::other_route_model("gpt-4o")] + fn ocr_contract_model_without_a_provider_is_rejected_like_python(#[case] model: &str) { + let error = resolve_provider_config(model, None).unwrap_err(); + assert!(matches!(&error, Error::ProviderNotProvided(passed) if passed == model)); + assert_eq!(error.http_status_code(), Some(400)); + } + #[rstest] #[case::prefix("not_a_provider/model", None)] #[case::explicit("model", Some("not_a_provider"))] diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs index b9c60f57e3c..99210e210fb 100644 --- a/litellm-rust/crates/core/src/ocr/wire.rs +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -96,7 +96,17 @@ pub fn decode_request_input>( } pub fn decode_document(value: Value) -> Result { - let kind = value.get("type").and_then(Value::as_str); + let Some(fields) = value.as_object() else { + return Err(Error::DocumentNotObject); + }; + let kind = match fields.get("type") { + Some(Value::String(kind)) if matches!(kind.as_str(), "document_url" | "image_url") => { + Some(kind.as_str()) + } + Some(Value::String(kind)) => return Err(Error::InvalidDocumentType(kind.clone())), + Some(other) => return Err(Error::InvalidDocumentType(other.to_string())), + None => return Err(Error::InvalidDocumentType("None".into())), + }; if matches!(kind, Some("document_url")) && value.get("document_url").is_none() || matches!(kind, Some("image_url")) && value.get("image_url").is_none() { @@ -122,9 +132,12 @@ mod tests { } #[rstest] - #[case::non_object(json!([]), "document")] - #[case::missing_type(json!({"document_url":"https://example.com/a.pdf"}), "document")] - #[case::unsupported_type(json!({"type":"text"}), "type")] + #[case::non_object(json!([]), "document must be a dict")] + #[case::missing_type( + json!({"document_url":"https://example.com/a.pdf"}), + "Invalid document type: None" + )] + #[case::unsupported_type(json!({"type":"text"}), "Invalid document type: text")] #[case::missing_document_url(json!({"type":"document_url"}), "Document URL")] #[case::missing_image_url(json!({"type":"image_url"}), "Document URL")] fn ocr_contract_malformed_document_is_bad_request( @@ -134,7 +147,7 @@ mod tests { let error = decode_document(document).unwrap_err(); assert!(matches!( error, - Error::RequestField { .. } | Error::MissingDocumentUrl + Error::DocumentNotObject | Error::InvalidDocumentType(_) | Error::MissingDocumentUrl )); assert_eq!(error.http_status_code(), Some(400)); assert!(error.to_string().contains(field), "{error}"); diff --git a/litellm-rust/crates/host-python/AGENTS.md b/litellm-rust/crates/host-python/AGENTS.md index 5aca13eeb18..ba2142b7412 100644 --- a/litellm-rust/crates/host-python/AGENTS.md +++ b/litellm-rust/crates/host-python/AGENTS.md @@ -5,6 +5,26 @@ - `RouteHost::invoke` receives the keyword view the adapter's `begin` returned, not the caller's dict; a route host that projects from it inherits that adapter's rewrites (for the legacy adapter: setup, deployment hooks, credential inheritance) - A native failure, including one a host op returns as `InvokeError::Native`, is classified exactly once through the route's `classify`; a Python exception raised inside the call, and a failure in `begin` or `after_success`, is raised as is - A failing `classify` is raised with the native error's text as its `__context__`, never swallowed +- Terms; the callback crates build on these and never redefine them + - Machine: the native call; it never calls out, it yields a `HostOp` (`Route`, `BeforeSend`, `Emit`, `Open`, `Deliver`) and waits for the `HostResult` + - Host: whatever answers those ops; the Python host is two capabilities the driver composes: `RouteHost` (route ops, argument and response projection) and `PythonLifecycle` (callbacks) + - Host language = callback language: a lifecycle speaks `Py<...>` and `PyErr` directly; another host language gets its own crate of this shape and reuses only `litellm-host` and `litellm-callbacks-v1` + - Driver: runs one call; owns the machine, one route host and exactly one lifecycle + - Lifecycle: one implementation of `PythonLifecycle`, one callback contract's view of a call; chain: several lifecycles behind one, in a fixed order + - Step: one driver-to-lifecycle method call, answered with a `LifecycleStep`; a threading step (`begin`, `before_send`, `after_success`) returns the value the call continues with and defaults to returning what it was given; an observing step (`emit`, `opened`, `delivered`) returns nothing to the call + - Suspension: a step answered with `Await`; the driver awaits it in the caller's task and continues the same step through `resume` + - Terminal event: `Succeeded` or `Failed`, exactly once, last + - Cancellation: a raised `BaseException` that is not an `Exception`; `is_cancellation` is the only definition; never reported, never swallowed, never followed by dispatch + - Keyword view: the kwargs dict as rewritten so far; `begin` threads it and the route projects from the result + - Wire: the `WireRequest` (URL, headers, body) about to be sent; `before_send` threads it + - Surface: the static facts a route declares about itself to a contract (`call_type`, ...) + - Python contract: a crate's `python.rs` enum, its `python_contract.json` and the one `litellm/rust_bridge/*.py` module behind them, tested from both sides +- `PythonLifecycle` is the whole callback boundary: the driver holds exactly one and never learns how many contracts are behind it + - `LifecycleChain` composes several in a fixed order: `begin`, `before_send` and `after_success` thread their value forward, events reach each in turn, any error ends the step with no further dispatch + - `begin` carries the call's start time and is the first thing a lifecycle hears of a call; there is no separate started event + - A lifecycle whose `begin` was never reached receives nothing of that call; the terminal event goes to the ones `begin` reached + - A suspension resumes the lifecycle that suspended, then continues the same step with the ones after it; a retained failure is the raised exception object itself + - Ordering policy (which contract runs first) belongs to whoever builds the chain, never to this crate - Use standard PyO3 ownership and conversion APIs - Prefer `Bound<'py, T>` for attached operations/results, `Py` for retention; binding/unbinding does not copy payloads - Use `pythonize` for selected Serde data, never a JSON-text round trip; share conversion with `Pythonized` diff --git a/litellm-rust/crates/host-python/src/adapter.rs b/litellm-rust/crates/host-python/src/adapter.rs index 3a4cb49be4d..7f61e759536 100644 --- a/litellm-rust/crates/host-python/src/adapter.rs +++ b/litellm-rust/crates/host-python/src/adapter.rs @@ -1,15 +1,22 @@ use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest}; use litellm_host::route::Route; -use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::{PyException, PyRuntimeError}; use pyo3::gc::{PyTraverseError, PyVisit}; use pyo3::prelude::*; use pyo3::types::PyDict; +/// Whether an error ends the call outright: anything that is not a `PyException`, such +/// as `CancelledError` or `KeyboardInterrupt`. A cancellation is never reported, never +/// swallowed and never followed by further dispatch. +pub fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool { + !error.is_instance_of::(py) +} + pub fn missing_state() -> PyErr { PyRuntimeError::new_err("missing native call state") } -/// What an adapter step produced: either the value the driver asked for, or a Python +/// What a lifecycle step produced: either the value the driver asked for, or a Python /// awaitable the driver hands back to the caller's task before asking again. pub enum LifecycleStep { Await(Py), @@ -19,12 +26,10 @@ pub enum LifecycleStep { Done, } -/// What a lifecycle observes: the driver's start, the machine's own events, and one -/// terminal event carrying the public value the caller receives. +/// What a lifecycle observes: the machine's own events, and one terminal event +/// carrying the public value the caller receives. +#[derive(Clone, Copy)] pub enum LifecycleEvent<'a> { - Started { - start_time: f64, - }, Machine(&'a MachineEvent), Succeeded { timing: Timing, @@ -37,46 +42,73 @@ pub enum LifecycleEvent<'a> { }, } -/// One consumer of a call's lifecycle on the Python side. The driver calls the steps in -/// order: `begin` before the machine starts, `before_send` and `emit` while it runs, -/// `after_success` and one terminal `emit` after it completes. Whenever a step returns -/// [`LifecycleStep::Await`], the driver awaits it in the caller's task and continues the -/// same step through `resume`. +/// One consumer of a call's lifecycle on the Python side: the callback capability of the +/// Python host, next to [`RouteHost`], the route capability. +/// +/// The steps come in two kinds. A threading step receives a value and returns the value +/// the call continues with, so a lifecycle may rewrite it; each defaults to returning +/// what it was given. An observing step receives an event or a notice and returns +/// nothing to the call. Two steps answer an operation the machine raised +/// ([`HostOp`](litellm_host::host::HostOp)); the rest the driver originates around it: +/// +/// | step | kind | raised by | +/// |---|---|---| +/// | `begin` | threads the keyword view | driver, before the machine starts | +/// | `before_send` | threads the wire request | machine, `HostOp::BeforeSend` | +/// | `emit(Machine)` | observes | machine, `HostOp::Emit` | +/// | `opened`, `delivered` | observe | driver, answering `HostOp::Open` and `Deliver` | +/// | `after_success` | threads the public response | driver, after the machine completes | +/// | `emit(Succeeded \| Failed)` | observes | driver, exactly once, last | +/// +/// Whenever a step returns [`LifecycleStep::Await`], the driver awaits it in the caller's +/// task and continues the same step through `resume`. /// /// A step that fails with an ordinary exception fails the call with that exception, -/// except on a terminal event, where the adapter is expected to report and swallow its -/// own errors. An exception that is not a `PyException`, such as a cancellation, ends -/// the call without further dispatch. +/// except on a terminal event, where the lifecycle is expected to report and swallow its +/// own errors. A cancellation ([`is_cancellation`]) ends the call without further +/// dispatch. pub trait PythonLifecycle: Send + Sync { + /// `start_time` is the call's start in epoch seconds, the same value the terminal + /// event's `Timing` carries. fn begin( &mut self, - py: Python<'_>, + _py: Python<'_>, arguments: Py, - started_at: f64, - ) -> PyResult; + _start_time: f64, + ) -> PyResult { + Ok(LifecycleStep::Arguments(arguments)) + } fn before_send( &mut self, - py: Python<'_>, + _py: Python<'_>, wire: Box, - context: &RequestContext, - ) -> PyResult; + _context: &RequestContext, + ) -> PyResult { + Ok(LifecycleStep::Wire(wire)) + } fn after_success( &mut self, - py: Python<'_>, + _py: Python<'_>, response: Py, - timing: Timing, - ) -> PyResult; + _timing: Timing, + ) -> PyResult { + Ok(LifecycleStep::Response(response)) + } fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult; /// The call streams and its stream was handed to the caller. The caller is not /// inside an await here, so this step and `delivered` cannot suspend. - fn opened(&mut self, py: Python<'_>) -> PyResult<()>; + fn opened(&mut self, _py: Python<'_>) -> PyResult<()> { + Ok(()) + } /// One chunk of an open stream is about to reach the caller. - fn delivered(&mut self, py: Python<'_>, chunk: &Py) -> PyResult<()>; + fn delivered(&mut self, _py: Python<'_>, _chunk: &Py) -> PyResult<()> { + Ok(()) + } fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult; diff --git a/litellm-rust/crates/host-python/src/chain.rs b/litellm-rust/crates/host-python/src/chain.rs new file mode 100644 index 00000000000..cc817ea11eb --- /dev/null +++ b/litellm-rust/crates/host-python/src/chain.rs @@ -0,0 +1,560 @@ +//! Several lifecycles consuming one call as one [`PythonLifecycle`]. The driver keeps a +//! single consumer; which contracts a call runs under is decided where the chain is +//! built, never by the driver and never by a route. + +use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest}; +use pyo3::exceptions::PyBaseException; +use pyo3::gc::{PyTraverseError, PyVisit}; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +use crate::adapter::{LifecycleEvent, LifecycleStep, PythonLifecycle, missing_state}; + +/// An event kept across a suspension, so the lifecycles after the suspended one still +/// receive it. A failure keeps the raised exception object itself. +enum RetainedEvent { + Machine(MachineEvent), + Succeeded { + timing: Timing, + response: Py, + }, + Failed { + timing: Timing, + origin: FailureOrigin, + error: Py, + }, +} + +/// The step a suspended lifecycle was answering, with what the later lifecycles need to +/// answer it too. The value being threaded stays with the suspended lifecycle. +enum Step { + Begin { start_time: f64 }, + BeforeSend { context: RequestContext }, + AfterSuccess { timing: Timing }, + Emit(RetainedEvent), +} + +struct Suspended { + index: usize, + step: Step, +} + +/// Runs every step through its lifecycles in the order they were given. +/// +/// `begin`, `before_send` and `after_success` thread their value: each lifecycle +/// receives what the one before it returned, so a later lifecycle sees an earlier one's +/// rewrites and never the reverse. Events reach each lifecycle in turn. An error from +/// any lifecycle ends the step at once, with no dispatch to the ones after it. +/// +/// A lifecycle observes a call only once its `begin` was reached: when an earlier +/// lifecycle fails the call in `begin`, the later ones never learn of the call, and the +/// terminal event goes to those that did. +pub struct LifecycleChain { + lifecycles: Vec>, + /// How many lifecycles `begin` has reached. + entered: usize, + pending: Option, +} + +impl RetainedEvent { + fn retain(py: Python<'_>, event: LifecycleEvent<'_>) -> Self { + match event { + LifecycleEvent::Machine(event) => Self::Machine(event.clone()), + LifecycleEvent::Succeeded { timing, response } => Self::Succeeded { + timing, + response: response.clone_ref(py), + }, + LifecycleEvent::Failed { + timing, + origin, + error, + } => Self::Failed { + timing, + origin, + error: error.clone_ref(py).into_value(py), + }, + } + } + + fn emit(&self, py: Python<'_>, lifecycle: &mut dyn PythonLifecycle) -> PyResult { + match self { + Self::Machine(event) => lifecycle.emit(py, LifecycleEvent::Machine(event)), + Self::Succeeded { timing, response } => lifecycle.emit( + py, + LifecycleEvent::Succeeded { + timing: *timing, + response, + }, + ), + Self::Failed { + timing, + origin, + error, + } => { + let error = PyErr::from_value(error.bind(py).clone().into_any()); + lifecycle.emit( + py, + LifecycleEvent::Failed { + timing: *timing, + origin: *origin, + error: &error, + }, + ) + } + } + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + match self { + Self::Succeeded { response, .. } => visit.call(response), + Self::Failed { error, .. } => visit.call(error), + Self::Machine(_) => Ok(()), + } + } +} + +impl LifecycleChain { + pub fn new(lifecycles: Vec>) -> Self { + Self { + lifecycles, + entered: 0, + pending: None, + } + } + + /// How many lifecycles a step reaches: `begin` tries all of them, every later step + /// reaches the ones `begin` reached. + fn audience(&self, step: &Step) -> usize { + match step { + Step::Begin { .. } => self.lifecycles.len(), + Step::BeforeSend { .. } | Step::AfterSuccess { .. } | Step::Emit(_) => self.entered, + } + } + + fn call( + &mut self, + py: Python<'_>, + index: usize, + step: &Step, + carried: LifecycleStep, + ) -> PyResult { + let lifecycle = self.lifecycles[index].as_mut(); + match (step, carried) { + (Step::Begin { start_time }, LifecycleStep::Arguments(arguments)) => { + self.entered = index + 1; + lifecycle.begin(py, arguments, *start_time) + } + (Step::BeforeSend { context }, LifecycleStep::Wire(wire)) => { + lifecycle.before_send(py, wire, context) + } + (Step::AfterSuccess { timing }, LifecycleStep::Response(response)) => { + lifecycle.after_success(py, response, *timing) + } + (Step::Emit(event), LifecycleStep::Done) => event.emit(py, lifecycle), + _ => Err(missing_state()), + } + } + + /// Continues `step` from the lifecycle at `next`, handing each one what the previous + /// produced, until one suspends or the last has answered. + fn proceed( + &mut self, + py: Python<'_>, + mut next: usize, + mut carried: LifecycleStep, + step: Step, + ) -> PyResult { + while next < self.audience(&step) { + carried = match self.call(py, next, &step, carried)? { + LifecycleStep::Await(awaitable) => { + self.pending = Some(Suspended { index: next, step }); + return Ok(LifecycleStep::Await(awaitable)); + } + produced => produced, + }; + next += 1; + } + Ok(carried) + } +} + +impl PythonLifecycle for LifecycleChain { + fn begin( + &mut self, + py: Python<'_>, + arguments: Py, + start_time: f64, + ) -> PyResult { + self.proceed( + py, + 0, + LifecycleStep::Arguments(arguments), + Step::Begin { start_time }, + ) + } + + fn before_send( + &mut self, + py: Python<'_>, + wire: Box, + context: &RequestContext, + ) -> PyResult { + self.proceed( + py, + 0, + LifecycleStep::Wire(wire), + Step::BeforeSend { + context: context.clone(), + }, + ) + } + + fn after_success( + &mut self, + py: Python<'_>, + response: Py, + timing: Timing, + ) -> PyResult { + self.proceed( + py, + 0, + LifecycleStep::Response(response), + Step::AfterSuccess { timing }, + ) + } + + fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { + for index in 0..self.entered { + match self.lifecycles[index].emit(py, event)? { + LifecycleStep::Await(awaitable) => { + self.pending = Some(Suspended { + index, + step: Step::Emit(RetainedEvent::retain(py, event)), + }); + return Ok(LifecycleStep::Await(awaitable)); + } + LifecycleStep::Done => {} + _ => return Err(missing_state()), + } + } + Ok(LifecycleStep::Done) + } + + fn opened(&mut self, py: Python<'_>) -> PyResult<()> { + self.lifecycles[..self.entered] + .iter_mut() + .try_for_each(|lifecycle| lifecycle.opened(py)) + } + + fn delivered(&mut self, py: Python<'_>, chunk: &Py) -> PyResult<()> { + self.lifecycles[..self.entered] + .iter_mut() + .try_for_each(|lifecycle| lifecycle.delivered(py, chunk)) + } + + fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult { + let Suspended { index, step } = self.pending.take().ok_or_else(missing_state)?; + match self.lifecycles[index].resume(py, result)? { + LifecycleStep::Await(awaitable) => { + self.pending = Some(Suspended { index, step }); + Ok(LifecycleStep::Await(awaitable)) + } + produced => self.proceed(py, index + 1, produced, step), + } + } + + fn close(&mut self, py: Python<'_>) { + self.pending = None; + for lifecycle in &mut self.lifecycles { + lifecycle.close(py); + } + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + for lifecycle in &self.lifecycles { + lifecycle.traverse(visit)?; + } + match &self.pending { + Some(Suspended { + step: Step::Emit(event), + .. + }) => event.traverse(visit), + _ => Ok(()), + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use pyo3::exceptions::PyValueError; + + use super::*; + + type Log = Arc>>; + + #[derive(Clone, Copy, PartialEq)] + enum Script { + Plain, + FailBegin, + SuspendBegin, + SuspendTerminal, + } + + struct Recorder { + name: &'static str, + script: Script, + log: Log, + suspended: Option, + } + + impl Recorder { + fn record(&self, entry: &str) { + self.log + .lock() + .unwrap() + .push(format!("{}.{entry}", self.name)); + } + + fn suspend(&mut self, py: Python<'_>, then: LifecycleStep) -> PyResult { + self.suspended = Some(then); + Ok(LifecycleStep::Await(py.None())) + } + } + + impl PythonLifecycle for Recorder { + fn begin( + &mut self, + py: Python<'_>, + arguments: Py, + _: f64, + ) -> PyResult { + let seen: Vec = arguments + .bind(py) + .keys() + .iter() + .map(|key| key.extract().unwrap()) + .collect(); + self.record(&format!("begin{seen:?}")); + arguments.bind(py).set_item(self.name, true)?; + match self.script { + Script::FailBegin => Err(PyValueError::new_err("begin failed")), + Script::SuspendBegin => self.suspend(py, LifecycleStep::Arguments(arguments)), + _ => Ok(LifecycleStep::Arguments(arguments)), + } + } + + fn before_send( + &mut self, + _: Python<'_>, + mut wire: Box, + _: &RequestContext, + ) -> PyResult { + self.record(&format!("before_send:{}", wire.url)); + wire.url.push_str(self.name); + Ok(LifecycleStep::Wire(wire)) + } + + fn after_success( + &mut self, + _: Python<'_>, + response: Py, + _: Timing, + ) -> PyResult { + self.record("after_success"); + Ok(LifecycleStep::Response(response)) + } + + fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { + let terminal = match event { + LifecycleEvent::Machine(_) => { + self.record("machine"); + false + } + LifecycleEvent::Succeeded { response, .. } => { + self.record(&format!("succeeded:{}", response.bind(py))); + true + } + LifecycleEvent::Failed { error, .. } => { + self.record(&format!("failed:{}", error.value(py))); + true + } + }; + if terminal && self.script == Script::SuspendTerminal { + return self.suspend(py, LifecycleStep::Done); + } + Ok(LifecycleStep::Done) + } + + fn opened(&mut self, _: Python<'_>) -> PyResult<()> { + self.record("opened"); + Ok(()) + } + + fn delivered(&mut self, _: Python<'_>, _: &Py) -> PyResult<()> { + self.record("delivered"); + Ok(()) + } + + fn resume(&mut self, _: Python<'_>, _: PyResult>) -> PyResult { + self.record("resume"); + self.suspended.take().ok_or_else(missing_state) + } + + fn close(&mut self, _: Python<'_>) { + self.record("close"); + } + + fn traverse(&self, _: &PyVisit<'_>) -> Result<(), PyTraverseError> { + Ok(()) + } + } + + fn chain(scripts: [(&'static str, Script); 2]) -> (LifecycleChain, Log) { + let log = Log::default(); + let lifecycles = scripts + .into_iter() + .map(|(name, script)| { + Box::new(Recorder { + name, + script, + log: Arc::clone(&log), + suspended: None, + }) as Box + }) + .collect(); + (LifecycleChain::new(lifecycles), log) + } + + fn entries(log: &Log) -> Vec { + std::mem::take(&mut *log.lock().unwrap()) + } + + fn timing() -> Timing { + Timing { + start_time: 1.0, + end_time: 2.0, + } + } + + fn context() -> RequestContext { + RequestContext { + model: "model".to_string(), + custom_llm_provider: "provider".to_string(), + optional_params: serde_json::Value::Null, + secret_fields: Vec::new(), + api_key: None, + } + } + + #[test] + fn a_later_lifecycle_receives_what_an_earlier_one_returned() { + crate::initialize_python(); + Python::attach(|py| { + let (mut chain, log) = chain([("first", Script::Plain), ("second", Script::Plain)]); + let arguments = PyDict::new(py).unbind(); + let step = chain.begin(py, arguments.clone_ref(py), 1.0).unwrap(); + assert!(matches!(step, LifecycleStep::Arguments(returned) if returned.is(&arguments))); + let wire = Box::new(WireRequest { + url: "url:".to_string(), + headers: Vec::new(), + body: serde_json::Value::Null, + }); + let step = chain.before_send(py, wire, &context()).unwrap(); + assert!(matches!(step, LifecycleStep::Wire(wire) if wire.url == "url:firstsecond")); + assert_eq!( + entries(&log), + [ + "first.begin[]", + "second.begin[\"first\"]", + "first.before_send:url:", + "second.before_send:url:first", + ] + ); + }); + } + + #[test] + fn a_lifecycle_begin_never_reached_observes_nothing_of_the_call() { + crate::initialize_python(); + Python::attach(|py| { + let (mut chain, log) = chain([("first", Script::FailBegin), ("second", Script::Plain)]); + let error = chain + .begin(py, PyDict::new(py).unbind(), 1.0) + .err() + .expect("begin fails"); + let failed = LifecycleEvent::Failed { + timing: timing(), + origin: FailureOrigin::Host, + error: &error, + }; + assert!(matches!( + chain.emit(py, failed).unwrap(), + LifecycleStep::Done + )); + chain.close(py); + assert_eq!( + entries(&log), + [ + "first.begin[]", + "first.failed:begin failed", + "first.close", + "second.close", + ] + ); + }); + } + + #[test] + fn a_suspended_step_resumes_its_lifecycle_and_then_reaches_the_later_ones() { + crate::initialize_python(); + Python::attach(|py| { + let (mut chain, log) = chain([ + ("first", Script::SuspendBegin), + ("second", Script::SuspendTerminal), + ]); + let step = chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap(); + assert!(matches!(step, LifecycleStep::Await(_))); + assert_eq!(entries(&log), ["first.begin[]"]); + let step = chain.resume(py, Ok(py.None())).unwrap(); + assert!(matches!(step, LifecycleStep::Arguments(_))); + assert_eq!(entries(&log), ["first.resume", "second.begin[\"first\"]"]); + }); + } + + #[test] + fn an_event_kept_across_a_suspension_reaches_the_later_lifecycles_unchanged() { + crate::initialize_python(); + Python::attach(|py| { + let (mut chain, log) = chain([ + ("first", Script::SuspendTerminal), + ("second", Script::Plain), + ]); + chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap(); + entries(&log); + let raised = PyValueError::new_err("provider failed"); + let failed = LifecycleEvent::Failed { + timing: timing(), + origin: FailureOrigin::Call, + error: &raised, + }; + assert!(matches!( + chain.emit(py, failed).unwrap(), + LifecycleStep::Await(_) + )); + assert!(matches!( + chain.resume(py, Ok(py.None())).unwrap(), + LifecycleStep::Done + )); + assert_eq!( + entries(&log), + [ + "first.failed:provider failed", + "first.resume", + "second.failed:provider failed", + ] + ); + }); + } +} diff --git a/litellm-rust/crates/host-python/src/driver.rs b/litellm-rust/crates/host-python/src/driver.rs index 77a294d274b..624e05377d9 100644 --- a/litellm-rust/crates/host-python/src/driver.rs +++ b/litellm-rust/crates/host-python/src/driver.rs @@ -46,7 +46,6 @@ enum Stage { #[derive(Clone, Copy)] enum Expect { - Started, Arguments, Wire, Emitted, @@ -159,13 +158,7 @@ where match (self.pending.take(), result) { (None, None) => { self.started_at = epoch_seconds(); - let started = LifecycleEvent::Started { - start_time: self.started_at, - }; - match self.adapter.emit(py, started) { - Ok(step) => self.on_adapter(py, step, Expect::Started), - Err(error) => self.adapter_failed(py, error), - } + self.begin(py) } (Some(Pending::Native), Some(Ok(_))) => { let result = self.take_native_result()?; @@ -201,7 +194,6 @@ where self.pending = Some(Pending::Adapter(expect)); Ok(ExecutionStep::Await(awaitable)) } - (Expect::Started, LifecycleStep::Done) => self.begin(py), (Expect::Arguments, LifecycleStep::Arguments(arguments)) => { self.arguments = Some(arguments); self.stage = Stage::Call; @@ -789,7 +781,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { self.log.push(match event { - LifecycleEvent::Started { .. } => "started".into(), LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => { format!("response:{}", raw.body) } @@ -921,7 +912,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "before_send", @@ -967,7 +957,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "classify:provider exploded", @@ -1001,7 +990,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "classify:op rejected", @@ -1033,7 +1021,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "failed:Call:op failed", @@ -1071,7 +1058,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "route:project", "classify:provider exploded", @@ -1102,7 +1088,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri assert_eq!( log, [ - "started", "begin", "failed:Host:begin failed", "adapter.close", @@ -1225,10 +1210,7 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri ) .unwrap_err(); assert!(!error.is_instance_of::(py)); - assert_eq!( - log.entries(), - ["started", "begin", "route", "adapter.close"] - ); + assert_eq!(log.entries(), ["begin", "route", "adapter.close"]); }); } diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 7d164ab7535..7fdc242f292 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -6,6 +6,7 @@ mod adapter; mod argument; mod callable; +mod chain; mod driver; mod execution; mod fork_gate; @@ -14,10 +15,12 @@ mod handle; mod marshal; pub use adapter::{ - InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, missing_state, + InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, is_cancellation, + missing_state, }; pub use argument::lookup; pub use callable::wrap_failure; +pub use chain::LifecycleChain; pub use driver::run_call; pub use execution::{ ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value, diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 0cc7af1836f..efece8114af 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -22,6 +22,7 @@ base64.workspace = true bytes.workspace = true data-url = "0.3.2" futures-util.workspace = true +percent-encoding.workspace = true reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index 9b27fdbb568..e6a3288fe44 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -8,6 +8,7 @@ use litellm_core_utils::{ serde_compat::{FiniteF64, LaxI64}, url_utils::ApiUrl, }; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC}; use reqwest::Url; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::{Map, Value}; @@ -549,6 +550,17 @@ async fn poll_operation( } } +/// Python parity: the Python provider builds this query with +/// `urllib.parse.quote(value, safe=",")`, which escapes everything except RFC 3986 unreserved +/// characters and `,`. The literal commas are also what the service documents for the +/// `pages` and `features` lists; form encoding would send `%2C`. +const PYTHON_QUOTE_KEEP_COMMAS: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~') + .remove(b','); + impl AzureDocumentIntelligenceOcrConfig { fn build_ocr_url( &self, @@ -561,16 +573,15 @@ impl AzureDocumentIntelligenceOcrConfig { ApiUrl::parse(endpoint) .and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model])) .map(|url| { - url.append_query_pairs( - [("api-version", api_version)] - .into_iter() - .chain(params.pages.iter().map(|pages| ("pages", pages.as_str()))) - .chain( - params - .features - .iter() - .map(|features| ("features", features.as_str())), - ), + url.append_query_pairs_escaped( + [ + ("api-version", Some(api_version)), + ("pages", params.pages.as_deref()), + ("features", params.features.as_deref()), + ] + .into_iter() + .filter_map(|(key, value)| Some((key, value?))), + PYTHON_QUOTE_KEEP_COMMAS, ) .into_string() }) @@ -649,6 +660,54 @@ mod tests { use super::*; + #[test] + fn query_escape_set_matches_python_quote() { + // Expected output from Python: + // quote("".join(map(chr, range(0x20, 0x7f))) + "é", safe=",") + let input = (0x20u8..0x7f) + .map(char::from) + .chain(['é']) + .collect::(); + assert_eq!( + percent_encoding::utf8_percent_encode(&input, PYTHON_QUOTE_KEEP_COMMAS).to_string(), + "%20%21%22%23%24%25%26%27%28%29%2A%2B,-.%2F0123456789%3A%3B%3C%3D%3E%3F%40\ + ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%C3%A9" + ); + } + + #[rstest] + #[case::no_lists(None, None, "")] + #[case::pages_only(Some("1,3-4"), None, "&pages=1,3-4")] + #[case::features_only( + None, + Some("keyValuePairs,languages"), + "&features=keyValuePairs,languages" + )] + #[case::both_lists( + Some("1,3-4"), + Some("keyValuePairs,languages"), + "&pages=1,3-4&features=keyValuePairs,languages" + )] + fn ocr_url_query_matches_python( + #[case] pages: Option<&str>, + #[case] features: Option<&str>, + #[case] expected_lists: &str, + ) { + let params = DocumentIntelligenceParams { + pages: pages.map(Into::into), + features: features.map(Into::into), + }; + assert_eq!( + AzureDocumentIntelligenceOcrConfig + .build_ocr_url("https://di.test", "prebuilt-layout", ¶ms, "2024-11-30") + .unwrap(), + format!( + "https://di.test/documentintelligence/documentModels/prebuilt-layout:analyze\ + ?api-version=2024-11-30{expected_lists}" + ) + ); + } + fn map(value: Value) -> Result { let arguments = serde_json::from_value(value).unwrap(); AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model") diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/document.rs b/litellm-rust/crates/llms/src/base_llm/ocr/document.rs index 724625b8208..b950033512b 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/document.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/document.rs @@ -198,7 +198,7 @@ mod tests { client.document_fetcher(), OcrDocument::ImageUrl { image_url: format!("http://{address}/image"), - extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]), + extra_fields: Map::from_iter([("detail".into(), "high".into())]), }, &OcrConnection::default(), ) @@ -210,7 +210,7 @@ mod tests { converted, OcrDocument::ImageUrl { image_url: "data:image/png;base64,YWJj".into(), - extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]), + extra_fields: Map::from_iter([("detail".into(), "high".into())]), } ); assert!(!request.to_ascii_lowercase().contains("authorization")); diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs index e09842e2856..67345aecb19 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs @@ -30,6 +30,10 @@ pub enum Error { MissingField(&'static str), #[error("Document URL is required")] MissingDocumentUrl, + #[error("document must be a dict with 'type' and URL/file field")] + DocumentNotObject, + #[error("Invalid document type: {0}. Must be 'document_url', 'image_url', or 'file'")] + InvalidDocumentType(String), #[error("invalid OCR document data URI")] InvalidDataUri, #[error( @@ -76,6 +80,10 @@ pub enum Error { Unsupported(&'static str), #[error("invalid provider: {0}")] InvalidProvider(String), + #[error( + "LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={0}" + )] + ProviderNotProvided(String), #[error("invalid model: {provider} has no model {model:?} - use one of: {}", supported.join(", "))] InvalidModel { provider: &'static str, @@ -149,6 +157,8 @@ impl Error { | Self::RequestField { .. } | Self::MissingField(_) | Self::MissingDocumentUrl + | Self::DocumentNotObject + | Self::InvalidDocumentType(_) | Self::InvalidDataUri | Self::ReductoSource | Self::InlineDocumentTooLarge @@ -161,6 +171,7 @@ impl Error { | Self::DotModel | Self::InvalidRequest(_) | Self::InvalidProvider(_) + | Self::ProviderNotProvided(_) | Self::InvalidModel { .. } | Self::Params(_) | Self::Headers(_) diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index e02a4b7f266..9c075b20c23 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -32,16 +32,31 @@ pub enum OcrDocument { DocumentUrl { document_url: String, #[serde(flatten)] - extra_fields: BTreeMap>, + extra_fields: BTreeMap, }, #[serde(rename = "image_url")] ImageUrl { + #[serde(deserialize_with = "image_url_source")] image_url: String, #[serde(flatten)] - extra_fields: BTreeMap>, + extra_fields: BTreeMap, }, } +/// Mistral accepts `image_url` as a string or as `{"url": ...}`; providers only consume the +/// source, so the object form is read down to its URL. +fn image_url_source<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum ImageUrl { + Source(String), + Object { url: String }, + } + Ok(match ImageUrl::deserialize(deserializer)? { + ImageUrl::Source(url) | ImageUrl::Object { url } => url, + }) +} + impl OcrDocument { pub fn source(&self) -> &str { match self { @@ -346,8 +361,13 @@ pub struct DecodedOcrResponse { pub fn decode_request_value(value: Value, prefix: &str) -> Result { serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { + let path = error.path().to_string(); Error::RequestField { - path: format!("{prefix}.{}", error.path()), + path: if path == "." { + prefix.to_string() + } else { + format!("{prefix}.{path}") + }, } }) } @@ -639,10 +659,8 @@ mod tests { assert!(serde_json::from_value::(Value::Object(payload)).is_err()); } assert!( - serde_json::from_value::(json!({ - "type":"image_url", "image_url":"https://example.com/image", "detail":42 - })) - .is_err() + serde_json::from_value::(json!({"type":"image_url", "image_url":42})) + .is_err() ); } @@ -694,6 +712,20 @@ mod tests { ); } + #[test] + fn documents_accept_what_python_forwards_to_the_provider() { + let document: OcrDocument = serde_json::from_value(json!({ + "type":"image_url", "image_url":{"url":"https://example.com/image", "detail":"high"} + })) + .unwrap(); + assert_eq!(document.source(), "https://example.com/image"); + let document = json!({ + "type":"document_url", "document_url":"https://example.com/a.pdf", "n":3 + }); + let decoded: OcrDocument = serde_json::from_value(document.clone()).unwrap(); + assert_eq!(serde_json::to_value(decoded).unwrap(), document); + } + #[test] fn response_serialization_flattens_extra_fields_and_omits_absent_native_response() { let response = LiteLLMOcrResponse { diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index 79e78d150e0..d299aca1f4a 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -1,7 +1,11 @@ - Target invariants, not completion claims; these supersede the crate guidance below where they conflict - Keep this crate the product-specific PyO3 consumer of `litellm-host-python` - Own registration, input projection, the route host and the caller callables it answers operations with (file readers, token providers), public response/error construction and the per-call composition of machine, route host and callback contract - - Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `run_legacy_call`; the bridge hands the public call over and keeps no copy + - Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `LegacyPythonLifecycle`; the bridge hands the public call over and keeps no copy + - Routes never select a callback contract: `callbacks::run_python_call` is the only way a route starts a call, and `src/callbacks.rs` is the only place that names a contract crate + - Every call runs the legacy `Logging` lifecycle, then the v1 lifecycle when anything is subscribed; with no subscriber the call is exactly the legacy-only call + - A route describes itself once as a `CallSurface`; fields only the legacy contract reads are marked so and go when it goes + - Retiring legacy for a route, or for all of them, is a change to that one composition, never to a route, the driver or an adapter - Value-oriented execution, sync waiting, nested-runtime checks, signal polling and panic containment live in `litellm-host-python`; native async work uses `pyo3-async-runtimes`, Serde output uses `Pythonized` - Core owns typed native state, the route machine, provider preparation/I/O and normalization; the host driver owns terminal events; the legacy adapter in `litellm-callbacks-legacy-python` owns `Logging` dispatch policy - Python, Rust SDK and gateway use one lifecycle-bearing core route entrypoint; provider helpers stay private, never bridge-accessible transport drivers diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 3a4a579efa3..4d0a3f6d349 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -19,10 +19,12 @@ panic-test = [] bytes.workspace = true litellm-auth.workspace = true litellm-callbacks-legacy-python.workspace = true +litellm-callbacks-v1-python.workspace = true litellm-core.workspace = true litellm-core-utils.workspace = true litellm-auth-gcp.workspace = true litellm-http.workspace = true +litellm-host.workspace = true litellm-llms.workspace = true litellm-types.workspace = true litellm-host-python.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/callbacks.rs b/litellm-rust/crates/python-bridge/src/callbacks.rs new file mode 100644 index 00000000000..ef5f9fa898d --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/callbacks.rs @@ -0,0 +1,79 @@ +//! The callback contracts a Python call runs under, composed here once so that no route +//! selects one. Every call runs the legacy `Logging` contract, for as long as Python's +//! `Logging` exists, and then the v1 contract whenever anything is subscribed to it. +//! Both are a [`PythonLifecycle`]; that trait is the whole boundary the driver knows. + +use litellm_callbacks_legacy_python::{LegacyPythonLifecycle, LegacyPythonSurface}; +pub(crate) use litellm_callbacks_legacy_python::{PassThroughStream, PublicCall}; +use litellm_callbacks_v1_python::{V1PythonLifecycle, V1PythonSurface}; +use litellm_host::{machine::Machine, route::Route}; +use litellm_host_python::{LifecycleChain, PythonLifecycle, RouteHost, run_call}; +use pyo3::prelude::*; + +/// What a route tells the callback contracts about itself. +#[derive(Clone, Copy, Debug)] +pub(crate) struct CallSurface { + pub call_type: &'static str, + /// Legacy only: what `Logging.pre_call` is told the input was. + pub input_description: &'static str, + /// Legacy only: how a streamed response is billed; `None` for a route that never + /// streams. + pub stream: Option, +} + +impl CallSurface { + fn legacy(self) -> LegacyPythonSurface { + LegacyPythonSurface { + call_type: self.call_type, + input_description: self.input_description, + stream: self.stream, + } + } + + fn v1(self) -> V1PythonSurface { + V1PythonSurface { + call_type: self.call_type, + } + } +} + +/// Legacy first: v1 subscribers observe the keyword view and the wire request as the +/// legacy contract left them, and share the `litellm_call_id` its setup assigned. +fn lifecycle( + py: Python<'_>, + surface: CallSurface, + call: PublicCall, + asynchronous: bool, +) -> PyResult> { + let legacy = Box::new(LegacyPythonLifecycle::new( + py, + surface.legacy(), + call, + asynchronous, + )); + Ok( + match V1PythonLifecycle::subscribed(py, surface.v1(), asynchronous)? { + Some(v1) => Box::new(LifecycleChain::new(vec![legacy, Box::new(v1)])), + None => legacy, + }, + ) +} + +/// Runs one native call for a Python caller. The route host projects from the keyword +/// view the lifecycle's `begin` prepares. +pub(crate) fn run_python_call( + py: Python<'_>, + surface: CallSurface, + call: PublicCall, + machine: M, + route: H, + asynchronous: bool, +) -> PyResult> +where + H: RouteHost + 'static, + M: Machine::Response> + 'static, +{ + let arguments = call.arguments(py); + let lifecycle = lifecycle(py, surface, call, asynchronous)?; + run_call(py, machine, route, lifecycle, arguments, asynchronous) +} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 46f98736aa1..2a19a169335 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,3 +1,4 @@ +mod callbacks; mod credentials; mod diagnostics; mod errors; diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index b606293f79f..7f6c1981467 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -1,18 +1,18 @@ mod host; use host::MessagesRouteHost; -use litellm_callbacks_legacy_python::{ - LegacySurface, PassThroughStream, PublicCall, run_legacy_call, -}; use litellm_core::messages::route::{messages_machine, supports}; use pyo3::{ prelude::*, types::{PyDict, PyTuple}, }; -use crate::errors::RustBridgeDeclined; +use crate::{ + callbacks::{CallSurface, PassThroughStream, PublicCall, run_python_call}, + errors::RustBridgeDeclined, +}; -const SURFACE: LegacySurface = LegacySurface { +const SURFACE: CallSurface = CallSurface { call_type: "anthropic_messages", input_description: "Messages", stream: Some(PassThroughStream { @@ -39,7 +39,7 @@ fn run_messages( "the Rust Messages route does not serve this provider", )); } - run_legacy_call( + run_python_call( py, SURFACE, PublicCall::capture(&request, &args, &kwargs)?, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs index b0a6acdebfd..6111506f4e5 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs @@ -26,6 +26,14 @@ pub(super) fn to_pyerr(error: Error) -> PyErr { .ok(); error } + Error::InvalidProvider(provider) => { + let error = core_error_to_pyerr(Error::InvalidProvider(provider.clone()).into()); + error + .value(py) + .setattr("ocr_invalid_provider", provider) + .ok(); + error + } Error::FileRead { path, source } if source.kind() == std::io::ErrorKind::NotFound => { PyFileNotFoundError::new_err(format!("File not found: {}", path.display())) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index e518f972bac..ca49dc27784 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -7,7 +7,6 @@ use std::sync::{Arc, LazyLock}; use host::OcrRouteHost; use litellm_auth_gcp::VertexAuth; -use litellm_callbacks_legacy_python::{LegacySurface, PublicCall, run_legacy_call}; use litellm_core::ocr::route::ocr_machine; use litellm_core_utils::settings::ProcessEnvironment; use litellm_llms::base_llm::ocr::{ @@ -19,15 +18,20 @@ use pyo3::{ types::{PyDict, PyTuple}, }; -use crate::{errors::RustBridgeDeclined, http, python_settings::PythonSettings}; +use crate::{ + callbacks::{CallSurface, PublicCall, run_python_call}, + errors::RustBridgeDeclined, + http, + python_settings::PythonSettings, +}; -const SURFACE: LegacySurface = LegacySurface { +const SURFACE: CallSurface = CallSurface { call_type: "ocr", input_description: "OCR document processing", stream: None, }; -const ASYNC_SURFACE: LegacySurface = LegacySurface { +const ASYNC_SURFACE: CallSurface = CallSurface { call_type: "aocr", ..SURFACE }; @@ -52,7 +56,7 @@ fn run_ocr( secrets, ) .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?; - run_legacy_call( + run_python_call( py, if asynchronous { ASYNC_SURFACE } else { SURFACE }, PublicCall::capture(&request, &args, &kwargs)?, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 697b935a1d4..65012622781 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -83,21 +83,27 @@ enum ProjectedDocument { impl ProjectedDocument { fn project(document: &Bound<'_, PyAny>) -> PyResult { - let kind: String = document - .get_item("type") - .and_then(|value| value.extract()) - .map_err(|error| { - let py = document.py(); - if error.is_instance_of::(py) - || error.is_instance_of::(py) - { - ocr_error_to_pyerr(Error::RequestField { - path: "document.type".into(), - }) - } else { - error - } - })?; + let py = document.py(); + if !document.is_instance_of::() && !document.hasattr("__getitem__")? { + return Err(ocr_error_to_pyerr(Error::DocumentNotObject)); + } + let kind = match document.get_item("type") { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + return Err(ocr_error_to_pyerr(Error::InvalidDocumentType( + "None".into(), + ))); + } + Err(error) if error.is_instance_of::(py) => { + return Err(ocr_error_to_pyerr(Error::DocumentNotObject)); + } + Err(error) => return Err(error), + }; + let Ok(kind) = kind.extract::() else { + return Err(ocr_error_to_pyerr(Error::InvalidDocumentType( + kind.str()?.to_string(), + ))); + }; if kind != "file" { return Ok(Self::Other(from_py(document)?)); } @@ -471,21 +477,25 @@ document = Document() } #[rstest::rstest] - #[case::missing(c"{}")] - #[case::non_string(c"{'type': 1}")] - #[case::list(c"[]")] - fn malformed_document_discriminators_are_bad_requests_naming_the_field( + #[case::missing( + c"{}", + "Invalid document type: None. Must be 'document_url', 'image_url', or 'file'" + )] + #[case::non_string( + c"{'type': 1}", + "Invalid document type: 1. Must be 'document_url', 'image_url', or 'file'" + )] + #[case::list(c"[]", "document must be a dict with 'type' and URL/file field")] + fn malformed_document_discriminators_are_bad_requests_with_the_python_message( #[case] document: &std::ffi::CStr, + #[case] expected: &str, ) { Python::initialize(); Python::attach(|py| { let error = project_document(&py.eval(document, None, None).unwrap()).unwrap_err(); let value = error.value(py); assert!(error.is_instance_of::(py)); - assert_eq!( - value.to_string(), - "invalid OCR request field: document.type" - ); + assert_eq!(value.to_string(), expected); assert_eq!( value .getattr("status_code") diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 06830ed4b53..12ff897f4bb 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -73,7 +73,7 @@ def _prepare_ocr_request( raise litellm.BadRequestError( message="document must be a dict with 'type' and URL/file field", model=model, - llm_provider=_error_provider(model, custom_llm_provider) or "", + llm_provider=resolve_error_provider(model, custom_llm_provider) or "", ) normalized_document: Final = ( @@ -85,13 +85,13 @@ def _prepare_ocr_request( raise litellm.BadRequestError( message=f"Invalid document type: {doc_type}. Must be 'document_url', 'image_url', or 'file'", model=model, - llm_provider=_error_provider(model, custom_llm_provider) or "", + llm_provider=resolve_error_provider(model, custom_llm_provider) or "", ) if not normalized_document.get(doc_type): raise litellm.BadRequestError( message="Document URL is required", model=model, - llm_provider=_error_provider(model, custom_llm_provider) or "", + llm_provider=resolve_error_provider(model, custom_llm_provider) or "", ) ( @@ -182,7 +182,7 @@ def _prepare_ocr_request( ) -def _error_provider(model: str, custom_llm_provider: str | None) -> str | None: +def resolve_error_provider(model: str, custom_llm_provider: str | None) -> str | None: if custom_llm_provider is not None: return custom_llm_provider prefix: Final = model.partition("/")[0] @@ -252,7 +252,7 @@ async def aocr( return response except Exception as e: - error_provider: Final = _error_provider(model, custom_llm_provider) + error_provider: Final = resolve_error_provider(model, custom_llm_provider) error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model raise litellm.exception_type( model=error_model, @@ -412,7 +412,7 @@ def ocr( return response except Exception as e: - error_provider: Final = _error_provider(model, custom_llm_provider) + error_provider: Final = resolve_error_provider(model, custom_llm_provider) error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model raise litellm.exception_type( model=error_model, diff --git a/litellm/rust_bridge/callbacks_v1_python.py b/litellm/rust_bridge/callbacks_v1_python.py new file mode 100644 index 00000000000..2de77e9034e --- /dev/null +++ b/litellm/rust_bridge/callbacks_v1_python.py @@ -0,0 +1,272 @@ +import inspect +import json +import threading +import uuid +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Final, Literal, Protocol, TypeAlias, TypeGuard # noqa: TID251 # recursive JSON validation + +from typing_extensions import ReadOnly, TypeAliasType, TypedDict + +JSONScalar: TypeAlias = None | bool | int | float | str +JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation + "JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"] +) + +SUPPORTED_SCHEMAS: Final = frozenset({1}) +EVENTS: Final = frozenset( + { + "call.started", + "request.sending", + "response.received", + "call.succeeded", + "call.failed", + } +) + + +class TimingFactsV1(TypedDict): + start_time: ReadOnly[float] + end_time: ReadOnly[float] + duration: ReadOnly[float] + + +class CallFactsV1(TypedDict): + start_time: ReadOnly[float] + asynchronous: ReadOnly[bool] + metadata: ReadOnly[Mapping[str, JSONValue]] + metadata_dropped: ReadOnly[Sequence[str]] + + +class RequestFactsV1(TypedDict): + model: ReadOnly[str] + custom_llm_provider: ReadOnly[str] + optional_params: ReadOnly[JSONValue] + url: ReadOnly[str] + headers: ReadOnly[Sequence[tuple[str, str]]] + body: ReadOnly[JSONValue] + + +ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field + "ErrorFactsV1", + { # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field + "class": ReadOnly[str], + "message": ReadOnly[str], + "status_code": ReadOnly[int | None], + }, +) + + +class CallStartedV1(CallFactsV1): + type: ReadOnly[Literal["call.started"]] + + +class RequestSendingV1(RequestFactsV1): + type: ReadOnly[Literal["request.sending"]] + + +class ResponseReceivedV1(TypedDict): + type: ReadOnly[Literal["response.received"]] + body: ReadOnly[str] + + +class CallSucceededV1(TypedDict): + type: ReadOnly[Literal["call.succeeded"]] + timing: ReadOnly[TimingFactsV1] + streamed: ReadOnly[bool] + response: ReadOnly[JSONValue] + response_error: ReadOnly[str | None] + + +class CallFailedV1(TypedDict): + type: ReadOnly[Literal["call.failed"]] + timing: ReadOnly[TimingFactsV1] + streamed: ReadOnly[bool] + origin: ReadOnly[Literal["call", "host"]] + error: ReadOnly[ErrorFactsV1] + + +EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1 + + +class EnvelopeV1(TypedDict): + schema: ReadOnly[Literal[1]] + call_id: ReadOnly[str] + call_type: ReadOnly[str] + seq: ReadOnly[int] + event: ReadOnly[EventV1] + + +class HeaderPatchV1(TypedDict, total=False): + remove: ReadOnly[Sequence[str]] + set: ReadOnly[Sequence[tuple[str, str]]] + + +class WirePatchV1(TypedDict, total=False): + headers: ReadOnly[HeaderPatchV1] + body: ReadOnly[JSONValue] + + +class Callback(Protocol): + name: str + schema: int + events: frozenset[str] + + def on_event(self, event: EnvelopeV1) -> None: ... + + async def async_on_event(self, event: EnvelopeV1) -> None: ... + + def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ... + + async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ... + + +class RegistrationError(ValueError): + pass + + +class UnsupportedSchema(RegistrationError): + pass + + +class UnknownEvent(RegistrationError): + pass + + +class MissingHandler(RegistrationError): + pass + + +class EmptyObserverEvents(RegistrationError): + pass + + +class HandlerKindMismatch(RegistrationError): + pass + + +class DuplicateName(RegistrationError): + pass + + +@dataclass(frozen=True, slots=True) +class Subscriber: + name: str + schema: int + events: frozenset[str] + on_event: object | None + async_on_event: object | None + before_send: object | None + async_before_send: object | None + + +class _Registry: + def __init__(self) -> None: + self.subscribers: tuple[Subscriber, ...] = () + self.lock: Final = threading.Lock() + + +_REGISTRY: Final = _Registry() + + +def _handler(callback: Callback, name: str) -> object | None: + return getattr(callback, name, None) + + +def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None: + if handler is None: + return + if not callable(handler): + raise HandlerKindMismatch(f"{name} must be callable") + is_async: Final = inspect.iscoroutinefunction(handler) + if is_async != asynchronous: + expected: Final = "async" if asynchronous else "sync" + raise HandlerKindMismatch(f"{name} must be {expected}") + + +def register(callback: Callback) -> None: + schema: Final = callback.schema + if schema not in SUPPORTED_SCHEMAS: + raise UnsupportedSchema(f"unsupported callback schema: {schema}") + events: Final = frozenset(callback.events) + unknown: Final = events - EVENTS + if unknown: + raise UnknownEvent(f"unknown callback events: {sorted(unknown)}") + + on_event: Final = _handler(callback, "on_event") + async_on_event: Final = _handler(callback, "async_on_event") + before_send: Final = _handler(callback, "before_send") + async_before_send: Final = _handler(callback, "async_before_send") + handlers: Final = (on_event, async_on_event, before_send, async_before_send) + if all(handler is None for handler in handlers): + raise MissingHandler("callback must define at least one handler") + if (on_event is not None or async_on_event is not None) and not events: + raise EmptyObserverEvents("observer callback events cannot be empty") + + _validate_kind("on_event", on_event, False) + _validate_kind("async_on_event", async_on_event, True) + _validate_kind("before_send", before_send, False) + _validate_kind("async_before_send", async_before_send, True) + + subscriber: Final = Subscriber( + name=callback.name, + schema=schema, + events=events, + on_event=on_event, + async_on_event=async_on_event, + before_send=before_send, + async_before_send=async_before_send, + ) + with _REGISTRY.lock: + if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers): + raise DuplicateName(f"duplicate callback name: {subscriber.name}") + _REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber) + + +def unregister(callback: Callback) -> None: + with _REGISTRY.lock: + _REGISTRY.subscribers = tuple( + subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != callback.name + ) + + +def snapshot() -> tuple[Subscriber, ...]: + return _REGISTRY.subscribers + + +def _is_json_value( + value: object, +) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value + if value is None or isinstance(value, (bool, int, float, str)): + return True + if isinstance(value, list): + return all( + _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively + for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively + ) + if isinstance(value, dict): + return all( + isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively + for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively + ) + return False + + +def project_response(response: object) -> JSONValue: + model_dump: Final = getattr(response, "model_dump", None) + projected: Final = model_dump(mode="json") if callable(model_dump) else response + json.dumps(projected, allow_nan=False) + if _is_json_value(projected): + return projected + raise TypeError(f"unsupported response type: {type(response).__qualname__}") + + +def new_call_id() -> str: + return str(uuid.uuid4()) + + +def report(name: str, event: str, error: BaseException) -> None: + from litellm._logging import verbose_logger + + exception: Final = (type(error), error, error.__traceback__) + verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception) diff --git a/litellm/rust_bridge/failures.py b/litellm/rust_bridge/failures.py index 80805b7ff69..83e33f7ce30 100644 --- a/litellm/rust_bridge/failures.py +++ b/litellm/rust_bridge/failures.py @@ -17,8 +17,8 @@ _UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]]) class UpstreamFailure(Exception): def __init__(self, response: httpx.Response, cause: Exception) -> None: - super().__init__(str(cause)) - self.message: Final = str(cause) + super().__init__(response.text) + self.message: Final = response.text self.response: Final = response self.status_code: Final = response.status_code self.__cause__ = cause @@ -49,13 +49,13 @@ class ExceptionMapper(Protocol): ) -> Exception: ... -def map_failure(error: Exception, model: str, request_provider: str, kwargs: Mapping[str, object]) -> Exception: +def map_failure(error: Exception, model: str, request_provider: str | None, kwargs: Mapping[str, object]) -> Exception: mapper: Final = cast( # cast-ok: bounded adapter for the legacy public exception mapper ExceptionMapper, litellm.exception_type ) try: return mapper( - model=model.removeprefix(f"{request_provider}/"), + model=model.removeprefix(f"{request_provider}/") if request_provider else model, custom_llm_provider=request_provider, original_exception=error, completion_kwargs=dict(kwargs), # mutable-ok: exception mapper requires owned kwargs @@ -67,7 +67,11 @@ def map_failure(error: Exception, model: str, request_provider: str, kwargs: Map def map_native_failure( - error: Exception, model: str, request_provider: str, kwargs: Mapping[str, object], api_base: str | None = None + error: Exception, + model: str, + request_provider: str | None, + kwargs: Mapping[str, object], + api_base: str | None = None, ) -> Exception: """`map_failure`, reading a native `(status, body)` provider failure as the HTTP response it was.""" original: Final = _upstream_failure(error, api_base) diff --git a/litellm/rust_bridge/ocr/route_host.py b/litellm/rust_bridge/ocr/route_host.py index bfbd5c11d4e..26b9b5e8fc6 100644 --- a/litellm/rust_bridge/ocr/route_host.py +++ b/litellm/rust_bridge/ocr/route_host.py @@ -8,6 +8,7 @@ from pydantic import TypeAdapter import litellm from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse +from litellm.ocr.main import resolve_error_provider from litellm.rust_bridge import failures from litellm.rust_bridge.failures import UpstreamFailure from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest @@ -31,11 +32,31 @@ def arguments(request: LiteLLMOcrRequest) -> Mapping[str, object]: return request.kwargs -def map_failure(error: Exception, request: LiteLLMOcrRequest, request_provider: str) -> Exception: +def map_failure(error: Exception, request: LiteLLMOcrRequest, projected_provider: str) -> Exception: + request_provider: Final = projected_provider or resolve_error_provider(request.model, request.custom_llm_provider) if getattr(error, "ocr_request_format_error", False): return litellm.UnsupportedParamsError( - message=f"Invalid `req_format`: {request.kwargs.get('req_format')!r}. Expected 'native' or 'litellm'.", - model=request.model.removeprefix(f"{request_provider}/"), - llm_provider=request_provider, + message=f"Invalid `req_format`: {request.kwargs.get('req_format')!r}. Expected one of litellm, native.", + model=request.model.removeprefix(f"{request_provider}/") if request_provider else request.model, + llm_provider=request_provider or "", ) + invalid_provider: Final = getattr(error, "ocr_invalid_provider", None) + if isinstance(invalid_provider, str): + return _invalid_provider_failure(invalid_provider, request) return failures.map_native_failure(error, request.model, request_provider, arguments(request), request.api_base) + + +def _invalid_provider_failure(provider: str, request: LiteLLMOcrRequest) -> Exception: + """Python rejects a LiteLLM provider without OCR support differently from an unknown provider prefix.""" + if provider in litellm.provider_list: + return failures.map_failure( + ValueError(f"OCR is not supported for provider: {provider}"), request.model, None, arguments(request) + ) + return litellm.BadRequestError( + message=( + "LLM Provider NOT provided. Pass in the LLM provider you are trying to call. " + f"You passed model={request.model}" + ), + model=request.model, + llm_provider="", + ) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 071191183df..9b88d6d0209 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -68,6 +68,7 @@ IGNORE_FUNCTIONS = [ "_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side. "_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible). "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. + "_is_json_value", # walks a tree that already passed json.dumps(allow_nan=False): finite, acyclic, and bounded by JSON nesting depth. ] diff --git a/tests/rust-python-harness/shared/parity/runner.py b/tests/rust-python-harness/shared/parity/runner.py index feae0201aee..453063db9da 100644 --- a/tests/rust-python-harness/shared/parity/runner.py +++ b/tests/rust-python-harness/shared/parity/runner.py @@ -52,13 +52,21 @@ class ExecutionVariant: class SubprocessWorker: - def __init__(self, runner: SubprocessRunner, provider: ReplayServer, variant: ExecutionVariant) -> None: + def __init__( + self, + runner: SubprocessRunner, + provider: ReplayServer, + variant: ExecutionVariant, + *, + baseline: bool, + ) -> None: project_root: Final = str(PROJECT_ROOT) existing_pythonpath: Final = os.environ.get("PYTHONPATH") env: Final = { **os.environ, **dict(variant.environment), - "LITELLM_USER_AGENT": runner.baseline_user_agent, + # Both implementations honor LITELLM_USER_AGENT, so only the baseline carries the sentinel. + **({"LITELLM_USER_AGENT": runner.baseline_user_agent} if baseline else {}), "PYTHONPATH": os.pathsep.join(path for path in (project_root, existing_pythonpath) if path), } self.mode: Final = variant.name @@ -159,9 +167,11 @@ class SubprocessWorker: def execution_worker( runner: SubprocessRunner, variant: ExecutionVariant, + *, + baseline: bool, ) -> Generator[SubprocessWorker]: with replay_server() as provider: - worker: Final = SubprocessWorker(runner, provider, variant) + worker: Final = SubprocessWorker(runner, provider, variant, baseline=baseline) try: yield worker finally: @@ -174,8 +184,8 @@ def execution_worker_pair( baseline: ExecutionVariant, candidate: ExecutionVariant, ) -> Generator[tuple[SubprocessWorker, SubprocessWorker]]: - with execution_worker(runner, baseline) as baseline_worker: - with execution_worker(runner, candidate) as candidate_worker: + with execution_worker(runner, baseline, baseline=True) as baseline_worker: + with execution_worker(runner, candidate, baseline=False) as candidate_worker: yield baseline_worker, candidate_worker diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py index f14296fbf06..0092f2daae6 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py @@ -234,14 +234,6 @@ INVALID_OCR_CASES: Final = ( expected_status_code=500, expected_message="OCR is not supported for provider: openai", ), - InvalidOcrCase( - name="unsupported_reducto_model", - model="reducto/parse-v4", - document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, - expected_message="OCR is not supported for provider: reducto", - ), InvalidOcrCase( name="unknown_provider_prefix", model="not_a_provider/model", @@ -254,40 +246,40 @@ INVALID_OCR_CASES: Final = ( name="non_object_document", model="mistral/mistral-ocr-latest", document=[], - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="document must be a dict", ), InvalidOcrCase( name="missing_document_type", model="mistral/mistral-ocr-latest", document={}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Invalid document type: None", ), InvalidOcrCase( name="unsupported_document_type", model="mistral/mistral-ocr-latest", document={"type": "text", "text": "not a document"}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Invalid document type: text", ), InvalidOcrCase( name="missing_document_url", model="azure_ai/doc-intelligence/prebuilt-read", document={"type": "document_url"}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Document URL is required", ), InvalidOcrCase( name="missing_image_url", model="azure_ai/doc-intelligence/prebuilt-read", document={"type": "image_url"}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Document URL is required", ), InvalidOcrCase( @@ -303,8 +295,8 @@ INVALID_OCR_CASES: Final = ( name="invalid_document_intelligence_pages", model="azure_ai/doc-intelligence/prebuilt-read", document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="`pages` integers must be >= 0", extra_kwargs=(("pages", [-1]),), ), @@ -312,8 +304,8 @@ INVALID_OCR_CASES: Final = ( name="invalid_document_intelligence_features", model="azure_ai/doc-intelligence/prebuilt-read", document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Invalid `features` for Azure Document Intelligence", extra_kwargs=(("features", [1]),), ), diff --git a/tests/test_litellm/rust_bridge/ocr/test_route_host.py b/tests/test_litellm/rust_bridge/ocr/test_route_host.py index 699492e4424..744fe437ba1 100644 --- a/tests/test_litellm/rust_bridge/ocr/test_route_host.py +++ b/tests/test_litellm/rust_bridge/ocr/test_route_host.py @@ -83,3 +83,33 @@ def test_map_failure_leaves_non_upstream_errors_unwrapped() -> None: def test_map_failure_reports_invalid_request_format_as_unsupported_params() -> None: with pytest.raises(litellm.UnsupportedParamsError, match="Invalid `req_format`: 'markdown'"): raise map_failure(RustFormatError(), REQUEST, "mistral") + + +def test_failure_before_projection_reports_the_provider_python_reports() -> None: + error: Final = ValueError("Document URL is required") + error.status_code = 400 # type: ignore[attr-defined] + + mapped: Final = map_failure(error, REQUEST, "") + + assert isinstance(mapped, litellm.BadRequestError) + assert mapped.llm_provider == "mistral" + assert "BadRequestError: - " not in str(mapped) + + +@pytest.mark.parametrize( + ("provider", "expected_type", "expected_message"), + [ + ("openai", litellm.APIConnectionError, "OCR is not supported for provider: openai"), + ("not_a_provider", litellm.BadRequestError, "LLM Provider NOT provided"), + ], +) +def test_invalid_provider_failures_follow_the_python_route( + provider: str, expected_type: type[Exception], expected_message: str +) -> None: + error: Final = ValueError(f"invalid provider: {provider}") + error.ocr_invalid_provider = provider # type: ignore[attr-defined] + + mapped: Final = map_failure(error, REQUEST, "") + + assert isinstance(mapped, expected_type) + assert expected_message in str(mapped) diff --git a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py b/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py index 1f6a214398a..cb3c6358fd0 100644 --- a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py @@ -10,7 +10,7 @@ from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.rust_bridge import callbacks_legacy_python as legacy +from litellm.rust_bridge import callbacks_legacy_python as callbacks from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup _OCR_KWARGS: Final = MappingProxyType( @@ -81,12 +81,10 @@ def test_setup_builds_a_logger_when_none_is_supplied(call_type: str, kwargs: Map assert result.logger.litellm_call_id == result.kwargs["litellm_call_id"] -CONTRACT_PATH: Final = ( - Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" -) +CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" def test_the_rust_contract_matches_the_shim_signatures() -> None: contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text()) - assert contract == {name: list(inspect.signature(getattr(legacy, name)).parameters) for name in contract} + assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract} diff --git a/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py b/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py new file mode 100644 index 00000000000..f854ea4b121 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_callbacks_v1_python.py @@ -0,0 +1,178 @@ +import inspect +import warnings +from collections.abc import Generator +from pathlib import Path +from typing import Final, cast + +import pytest +from pydantic import BaseModel, TypeAdapter + +from litellm.rust_bridge import callbacks_v1_python as callbacks + +ROOT: Final = Path(__file__).parents[3] +CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json" +GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1" + + +def _clear_registry() -> None: + for subscriber in callbacks.snapshot(): + callbacks.unregister(cast(callbacks.Callback, subscriber)) + + +@pytest.fixture(autouse=True) +def clear_registry() -> Generator[None, None, None]: + _clear_registry() + yield + _clear_registry() + + +def test_the_rust_contract_matches_the_shim_signatures() -> None: + contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text()) + assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract} + + +def test_v1_golden_envelopes_match_the_typed_contract() -> None: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning) + adapter: Final = TypeAdapter(callbacks.EnvelopeV1) + envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")] + assert {envelope["event"]["type"] for envelope in envelopes} == callbacks.EVENTS + assert {envelope["schema"] for envelope in envelopes} == callbacks.SUPPORTED_SCHEMAS + + +class ValidCallback: + name: str = "valid" + schema: int = 1 + events: frozenset[str] = frozenset({"call.started"}) + + def on_event(self, event: callbacks.EnvelopeV1) -> None: + del event + + +@pytest.mark.parametrize( + ("callback", "error"), + [ + ( + type( + "Unsupported", + (), + {"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None}, + )(), + callbacks.UnsupportedSchema, + ), + ( + type( + "Unknown", + (), + {"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None}, + )(), + callbacks.UnknownEvent, + ), + ( + type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(), + callbacks.MissingHandler, + ), + ( + type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(), + callbacks.EmptyObserverEvents, + ), + ( + type( + "AsyncFunction", + (), + { + "name": "async-function", + "schema": 1, + "events": frozenset({"call.started"}), + "async_on_event": lambda *_: None, + }, + )(), + callbacks.HandlerKindMismatch, + ), + ], + ids=["schema", "event", "missing", "empty-events", "async-kind"], +) +def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None: + with pytest.raises(error): + callbacks.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test + + +class WrongSyncKind: + name: str = "wrong-sync-kind" + schema: int = 1 + events: frozenset[str] = frozenset({"call.started"}) + + async def on_event(self, event: callbacks.EnvelopeV1) -> None: + del event + + +def test_register_rejects_coroutine_in_sync_slot() -> None: + with pytest.raises(callbacks.HandlerKindMismatch): + callbacks.register(cast(callbacks.Callback, WrongSyncKind())) + + +class AsyncObserver: + name: str = "async-observer" + schema: int = 1 + events: frozenset[str] = frozenset({"call.started"}) + + async def async_on_event(self, event: callbacks.EnvelopeV1) -> None: + del event + + +class SyncInterceptor: + name: str = "sync-interceptor" + schema: int = 1 + events: frozenset[str] = frozenset() + + def before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None: + del request + return None + + +class AsyncInterceptor: + name: str = "async-interceptor" + schema: int = 1 + events: frozenset[str] = frozenset() + + async def async_before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None: + del request + return None + + +@pytest.mark.parametrize("callback", [ValidCallback(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()]) +def test_register_accepts_each_handler_family(callback: callbacks.Callback) -> None: + callbacks.register(callback) + assert len(callbacks.snapshot()) == 1 + + +def test_register_rejects_duplicate_names() -> None: + callbacks.register(cast(callbacks.Callback, ValidCallback())) + with pytest.raises(callbacks.DuplicateName): + callbacks.register(cast(callbacks.Callback, ValidCallback())) + + +def test_snapshot_is_stable_across_later_registry_writes() -> None: + first: Final = ValidCallback() + second: Final = AsyncObserver() + callbacks.register(cast(callbacks.Callback, first)) + captured: Final = callbacks.snapshot() + callbacks.register(cast(callbacks.Callback, second)) + callbacks.unregister(cast(callbacks.Callback, first)) + assert callbacks.snapshot() is not captured + assert tuple(subscriber.name for subscriber in captured) == ("valid",) + + +class ResponseModel(BaseModel): + value: int + + +def test_project_response_supports_pydantic_and_json_values() -> None: + assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3} + original: Final = {"nested": [1, True, None]} + assert callbacks.project_response(original) is original + + +def test_project_response_rejects_unsupported_objects() -> None: + with pytest.raises(TypeError): + callbacks.project_response(object()) diff --git a/tests/test_litellm/rust_bridge/test_failures.py b/tests/test_litellm/rust_bridge/test_failures.py index 80057b816d3..5bfe98e699c 100644 --- a/tests/test_litellm/rust_bridge/test_failures.py +++ b/tests/test_litellm/rust_bridge/test_failures.py @@ -52,3 +52,18 @@ def test_kwargs_are_handed_to_the_mapper_as_owned_copies(monkeypatch: pytest.Mon assert seen[0]["completion_kwargs"] is not request_kwargs assert seen[0]["model"] == "gpt-4o" assert seen[0]["custom_llm_provider"] == "openai" + + +def test_native_provider_failure_message_is_the_provider_body_not_the_args_tuple() -> None: + class NativeUpstream(Exception): + headers = [("retry-after", "7")] + + native_error: Final = NativeUpstream(429, '{"message": "slow down"}') + + mapped: Final = failures.map_native_failure( + native_error, "mistral/mistral-ocr-latest", "mistral", MappingProxyType({}), "https://api.mistral.ai/v1" + ) + + assert isinstance(mapped, litellm.RateLimitError) + assert '{"message": "slow down"}' in str(mapped) + assert "(429," not in str(mapped) diff --git a/tests/test_litellm_rust/ocr/test_callbacks_v1.py b/tests/test_litellm_rust/ocr/test_callbacks_v1.py new file mode 100644 index 00000000000..3da44a6c394 --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_callbacks_v1.py @@ -0,0 +1,141 @@ +from collections.abc import Generator +from typing import Final + +import pytest + +from litellm.integrations.custom_logger import CustomLogger +from litellm.rust_bridge import callbacks_v1_python +from litellm.rust_bridge.callbacks_v1_python import EVENTS, EnvelopeV1, RequestFactsV1, WirePatchV1 +from tests.test_litellm_rust.support.callback_recorder import drain_logging +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import ( + OCR_RESPONSE, + call_native_aocr, + call_native_ocr, + request_body, +) + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +class Observer: + name: Final = "observer" + schema: Final = 1 + events: Final = EVENTS + + def __init__(self) -> None: + self.envelopes: Final[list[EnvelopeV1]] = [] + + def on_event(self, event: EnvelopeV1) -> None: + self.envelopes.append(event) + + +class Interceptor: + name: Final = "interceptor" + schema: Final = 1 + events: Final[frozenset[str]] = frozenset() + + def __init__(self) -> None: + self.bodies: Final[list[object]] = [] + + def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: + self.bodies.append(request["body"]) + return {"headers": {"set": [("x-v1-interceptor", "seen")]}} + + +@pytest.fixture +def observer() -> Generator[Observer]: + subscriber: Final = Observer() + callbacks_v1_python.register(subscriber) + try: + yield subscriber + finally: + callbacks_v1_python.unregister(subscriber) + + +@pytest.fixture +def interceptor() -> Generator[Interceptor]: + subscriber: Final = Interceptor() + callbacks_v1_python.register(subscriber) + try: + yield subscriber + finally: + callbacks_v1_python.unregister(subscriber) + + +class LegacyCallId(CustomLogger): + def __init__(self) -> None: + self.call_ids: Final[list[str]] = [] + + def log_pre_api_call(self, model, messages, kwargs): + self.call_ids.append(kwargs["litellm_call_id"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_ocr_runs_legacy_and_v1_callbacks_on_the_same_call( + ocr_server: RecordingServer, observer: Observer, asynchronous: bool +) -> None: + legacy: Final = LegacyCallId() + + if asynchronous: + await call_native_aocr(ocr_server, callbacks=[legacy]) + else: + call_native_ocr(ocr_server, callbacks=[legacy]) + await drain_logging() + + assert [envelope["event"]["type"] for envelope in observer.envelopes] == [ + "call.started", + "request.sending", + "response.received", + "call.succeeded", + ] + assert [envelope["seq"] for envelope in observer.envelopes] == sorted( + envelope["seq"] for envelope in observer.envelopes + ) + assert len(legacy.call_ids) == 1 + assert {envelope["call_id"] for envelope in observer.envelopes} == set(legacy.call_ids) + + +def test_native_ocr_v1_interceptor_sees_the_wire_as_the_legacy_callback_left_it( + ocr_server: RecordingServer, interceptor: Interceptor +) -> None: + class Edit(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + request_body(kwargs)["include_image_base64"] = True + + call_native_ocr(ocr_server, callbacks=[Edit()]) + + assert len(interceptor.bodies) == 1 + body: Final = interceptor.bodies[0] + assert isinstance(body, dict) + assert body["include_image_base64"] is True + request: Final = ocr_server.requests[-1] + assert request.headers["x-v1-interceptor"] == "seen" + + +def test_native_ocr_failure_reaches_both_contracts_once( + recording_server: RecordingServer, observer: Observer +) -> None: + recording_server.default_response = ResponseSpec(status=400, body={"message": "bad document"}) + failures: Final[list[object]] = [] + + class Failure(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + failures.append(kwargs["exception"]) + + with pytest.raises(Exception) as raised: # noqa: PT011 # the mapped provider error class is covered elsewhere + call_native_ocr(recording_server, callbacks=[Failure()]) + + assert failures == [raised.value] + terminal: Final = [ + envelope for envelope in observer.envelopes if envelope["event"]["type"] in {"call.succeeded", "call.failed"} + ] + assert [envelope["event"]["type"] for envelope in terminal] == ["call.failed"] + assert observer.envelopes[-1] is terminal[0]