This commit is contained in:
devin-ai-integration[bot] 2026-09-20 12:24:55 +00:00 • committed by GitHub
commit a8bef91c62
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
72 changed files with 3969 additions and 364 deletions

View file

@ -2043,6 +2043,33 @@ dependencies = [
"strum",
]
[[package]]
name = "litellm-callbacks-v1"
version = "0.1.0"
dependencies = [
"litellm-auth",
"litellm-host",
"proptest",
"rstest",
"serde",
"serde_json",
"strum",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-callbacks-v1-python"
version = "0.1.0"
dependencies = [
"litellm-auth",
"litellm-callbacks-v1",
"litellm-host",
"litellm-host-python",
"pyo3",
"serde_json",
"strum",
]
[[package]]
name = "litellm-core"
version = "0.1.0"
@ -2085,6 +2112,7 @@ version = "0.1.0"
dependencies = [
"fancy-regex",
"litellm-types",
"percent-encoding",
"rstest",
"serde",
"serde_json",
@ -2170,6 +2198,7 @@ dependencies = [
"litellm-host",
"litellm-http",
"litellm-types",
"percent-encoding",
"reqwest 0.12.28",
"rstest",
"serde",
@ -2193,8 +2222,10 @@ dependencies = [
"litellm-auth",
"litellm-auth-gcp",
"litellm-callbacks-legacy-python",
"litellm-callbacks-v1-python",
"litellm-core",
"litellm-core-utils",
"litellm-host",
"litellm-host-python",
"litellm-http",
"litellm-llms",

View file

@ -12,6 +12,8 @@ repository = "https://github.com/BerriAI/litellm"
litellm-core = { path = "crates/core" }
litellm-host = { path = "crates/host" }
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
litellm-callbacks-v1 = { path = "crates/callbacks-v1" }
litellm-callbacks-v1-python = { path = "crates/callbacks-v1-python" }
litellm-framing = { path = "crates/framer" }
litellm-auth = { path = "crates/auth" }
litellm-auth-aws = { path = "crates/auth-aws" }
@ -52,6 +54,7 @@ base64 = "0.22"
moka = { version = "0.12.16", features = ["future"] }
strum = { version = "0.28.0", features = ["derive"] }
url = "2.5.8"
percent-encoding = "2.3.2"
webpki-roots = "1"
time = { version = "0.3.53", features = ["parsing"] }
criterion = "0.8.2"

View file

@ -1,10 +1,20 @@
- Target invariants, not completion claims
- Keep this crate the legacy `@client` wrapper as the native call sees it, and nothing else: the `Logging` contract (`function_setup`, the deployment hooks, `pre_call`/`post_call`, the sync and async success and failure fan-out, the deferred proxy release, the argument sharing those callbacks rely on) plus the kwargs rewrites the wrapper makes on the way in (credential-name inheritance, the budget and retry-count limits)
- The driver in `litellm-host-python`, the routes and core see one `PythonLifecycle`; they never learn which Python objects consume a call
- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`)
- Terms (lifecycle terms are in `host-python/AGENTS.md`)
- `Logging`: litellm's Python class; logger: the one `Logging` instance of this call (`PythonLogger`), reused from `litellm_logging_obj` or built by `function_setup`
- Borrowed function: a litellm Python internal Rust still calls, one variant of `LegacyPython`; a user's own callback is never one
- Fan-out: `Logging` dispatching to every registered callback; family: the sync or the async set of success and failure handlers
- Deployment hooks: the awaited pre-call, post-call and failure hooks on each callback; async calls only, as in `@client`
- Re-aliasing: replacing body values with the caller's equal objects before `pre_call`, so a callback mutates what the caller holds
- Deferred release: the proxy holds back async success logging until it accepts the response (`PendingLogging.release`)
- Internal call: a proxy-internal call; when it is async, the async success dispatch and both failure families are skipped, and the sync success handlers still run
- Pass-through billing (`PassThroughStream`): end-of-stream cost logging from the delivered chunks; a failure after the stream opened bills them as partial usage, and a sync call falls back to the plain failure handler
- Redaction here is `****`, matching what the Python path logs; `[REDACTED]` belongs to the v1 contract
- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`, `Streaming`)
- The enum only shrinks: when Rust owns a subsystem, delete its group rather than adding a Rust path beside it
- Calling a user's own callback directly is permanent Python surface and gets its own type outside `LegacyPython`
- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes hand it over through `run_legacy_call` and keep no copy
- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes capture it and the bridge's `run_python_call` hands it to `LegacyPythonLifecycle::new`, keeping no copy
- `setup` reuses a `Logging` the caller passed as `litellm_logging_obj` (the proxy and Router are the live cases) and otherwise builds one through `function_setup`, as `@client` does
- Either way every phase calls the same `Logging` method the Python path calls; which callbacks run is `Logging`'s decision, never this crate's
- Callbacks receive the caller's own objects and may mutate them; this crate alone carries that obligation

View file

@ -9,13 +9,11 @@ autotests = false
[dependencies]
litellm-host.workspace = true
litellm-host-python.workspace = true
pyo3.workspace = true
strum.workspace = true
serde_json.workspace = true
strum.workspace = true
[dev-dependencies]
litellm-auth.workspace = true
proptest.workspace = true
rstest.workspace = true
serde_json.workspace = true

View file

@ -6,10 +6,10 @@ use litellm_host::event::{
FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest, epoch_seconds,
};
use litellm_host_python::{
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, missing_state, to_py,
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py,
};
use pyo3::{
exceptions::{PyBaseException, PyException},
exceptions::PyBaseException,
gc::{PyTraverseError, PyVisit},
prelude::*,
types::{PyDateTime, PyDict, PyList},
@ -17,7 +17,8 @@ use pyo3::{
use serde_json::Value;
use crate::{
DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger,
PublicCall, PythonLogger, after_deployment_failure, after_deployment_success,
before_deployment_call,
deferred::{PendingLogging, PendingSuccess},
finalize, is_internal_call, prepare,
python::Streaming,
@ -26,7 +27,7 @@ use crate::{
/// What the legacy contract needs to know about the route it is logging.
#[derive(Clone, Copy, Debug)]
pub struct LegacySurface {
pub struct LegacyPythonSurface {
pub call_type: &'static str,
/// What `Logging.pre_call` is told the input was.
pub input_description: &'static str,
@ -55,8 +56,8 @@ enum Pending {
AsyncFailure,
}
pub struct LegacyLogging {
surface: LegacySurface,
pub struct LegacyPythonLifecycle {
surface: LegacyPythonSurface,
call: PublicCall,
logger: Option<PythonLogger>,
start: Py<PyAny>,
@ -76,14 +77,10 @@ fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult<Py<PyAny>> {
PyDateTime::from_timestamp(py, epoch_seconds, None).map(|value| value.into_any().unbind())
}
fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool {
!error.is_instance_of::<PyException>(py)
}
impl LegacyLogging {
impl LegacyPythonLifecycle {
pub fn new(
py: Python<'_>,
surface: LegacySurface,
surface: LegacyPythonSurface,
call: PublicCall,
asynchronous: bool,
) -> Self {
@ -118,6 +115,14 @@ impl LegacyLogging {
}
fn prepare(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
let model = self.call.lookup(py, "model")?;
let provider = self.call.lookup(py, "custom_llm_provider")?;
self.logger()?.initialize_failure_context(
py,
self.call.kwargs(),
model.and_then(|model| model.extract::<String>().ok()),
provider.and_then(|provider| provider.extract::<String>().ok()),
)?;
let prepared = prepare(py, self.call.kwargs().bind(py), self.logger()?)?.unbind();
self.call.set_kwargs(prepared);
Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py)))
@ -140,7 +145,7 @@ impl LegacyLogging {
fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
match self.try_dispatch_success(py) {
Err(error) if error.is_instance_of::<PyException>(py) => {
Err(error) if !is_cancellation(py, &error) => {
error.write_unraisable(py, self.logger.as_ref().map(|logger| logger.object(py)));
Ok(())
}
@ -199,7 +204,7 @@ impl LegacyLogging {
),
);
match billed {
Err(error) if error.is_instance_of::<PyException>(py) => {
Err(error) if !is_cancellation(py, &error) => {
error.write_unraisable(py, Some(logger.object(py)));
Ok(())
}
@ -269,15 +274,15 @@ impl LegacyLogging {
}
}
impl PythonLifecycle for LegacyLogging {
impl PythonLifecycle for LegacyPythonLifecycle {
fn begin(
&mut self,
py: Python<'_>,
arguments: Py<PyDict>,
started_at: f64,
start_time: f64,
) -> PyResult<LifecycleStep> {
self.call.set_kwargs(arguments);
self.start = datetime(py, started_at)?;
self.start = datetime(py, start_time)?;
self.internal = is_internal_call(py)?;
let result = setup(
py,
@ -291,7 +296,7 @@ impl PythonLifecycle for LegacyLogging {
self.call.set_kwargs(result.kwargs()?);
if self.runs_deployment_hooks() {
self.pending = Some(Pending::DeploymentPreCall);
return Ok(LifecycleStep::Await(DeploymentHooks::before_call(
return Ok(LifecycleStep::Await(before_deployment_call(
py,
self.call.kwargs(),
self.surface.call_type,
@ -354,7 +359,7 @@ impl PythonLifecycle for LegacyLogging {
self.response = Some(response);
if self.runs_deployment_hooks() {
self.pending = Some(Pending::DeploymentPostCall);
return Ok(LifecycleStep::Await(DeploymentHooks::after_success(
return Ok(LifecycleStep::Await(after_deployment_success(
py,
self.call.kwargs(),
&self.response,
@ -366,7 +371,6 @@ impl PythonLifecycle for LegacyLogging {
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
match event {
LifecycleEvent::Started { .. } => Ok(LifecycleStep::Done),
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
let api_key = self
.context
@ -407,7 +411,7 @@ impl PythonLifecycle for LegacyLogging {
{
let error = self.error.as_ref().ok_or_else(missing_state)?;
self.pending = Some(Pending::DeploymentFailure);
return Ok(LifecycleStep::Await(DeploymentHooks::after_failure(
return Ok(LifecycleStep::Await(after_deployment_failure(
py,
self.call.kwargs(),
error,
@ -464,7 +468,11 @@ impl PythonLifecycle for LegacyLogging {
{
error.write_unraisable(py, None);
}
self.pending = None;
self.response = None;
self.error = None;
self.body = None;
self.headers = None;
self.context = None;
self.stream = None;
}
@ -482,7 +490,8 @@ impl PythonLifecycle for LegacyLogging {
visit.call(&stream.chunks)?;
visit.call(&stream.first_chunk)?;
}
visit.call(&self.body)
visit.call(&self.body)?;
visit.call(&self.headers)
}
}

View file

@ -3,16 +3,13 @@
//! lifetime. No other callback host has that obligation, which is why nothing outside
//! this crate holds them.
use litellm_host::{machine::Machine, route::Route};
use litellm_host_python::{RouteHost, lookup, run_call};
use litellm_host_python::lookup;
use pyo3::{
gc::{PyTraverseError, PyVisit},
prelude::*,
types::{PyDict, PyTuple},
};
use crate::{LegacyLogging, LegacySurface};
pub struct PublicCall {
args: Py<PyTuple>,
kwargs: Py<PyDict>,
@ -34,6 +31,11 @@ impl PublicCall {
})
}
/// The keyword view the call starts from, which the driver hands to `begin`.
pub fn arguments(&self, py: Python<'_>) -> Py<PyDict> {
self.kwargs.clone_ref(py)
}
pub(crate) fn args(&self) -> &Py<PyTuple> {
&self.args
}
@ -63,31 +65,6 @@ impl PublicCall {
}
}
/// Runs one native call under the legacy `Logging` contract: the route host projects from
/// the keyword view the contract prepares, and the contract observes the call.
pub fn run_legacy_call<H, M>(
py: Python<'_>,
surface: LegacySurface,
call: PublicCall,
machine: M,
route: H,
asynchronous: bool,
) -> PyResult<Py<PyAny>>
where
H: RouteHost + 'static,
M: Machine<Route = H::Route, Complete = <H::Route as Route>::Response> + 'static,
{
let arguments = call.kwargs.clone_ref(py);
run_call(
py,
machine,
route,
Box::new(LegacyLogging::new(py, surface, call, asynchronous)),
arguments,
asynchronous,
)
}
#[cfg(test)]
mod tests {
use super::*;

View file

@ -9,78 +9,10 @@ use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
use crate::logger::PythonLogger;
use crate::python::{Logging, Wrapper};
pub trait LegacyCallbacks {
impl PythonLogger {
/// `Logging.update_from_kwargs`: what the logger is told about the request it is
/// about to see, with consumed credentials redacted.
fn update_from_kwargs(
&self,
py: Python<'_>,
kwargs: &Py<PyDict>,
wire: &WireRequest,
context: &RequestContext,
) -> PyResult<()>;
/// `Logging.pre_call`.
fn pre_call(
&self,
py: Python<'_>,
input: &str,
api_key: Option<&str>,
body: &Bound<'_, PyDict>,
headers: &Bound<'_, PyDict>,
url: &str,
) -> PyResult<()>;
/// `Logging.post_call`.
fn post_call(
&self,
py: Python<'_>,
original_response: &str,
api_key: Option<&str>,
body: Option<&Py<PyDict>>,
headers: Option<&Py<PyDict>>,
) -> PyResult<()>;
fn defers_async_logging(&self, py: Python<'_>) -> bool;
fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()>;
fn sync_success_for_async_call(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
) -> PyResult<()>;
fn failure(
&self,
py: Python<'_>,
error: &Py<PyBaseException>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
asynchronous: bool,
) -> PyResult<Option<Py<PyAny>>>;
fn submit_success(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
) -> PyResult<()>;
fn enqueue_success(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
) -> PyResult<()>;
}
impl LegacyCallbacks for PythonLogger {
fn update_from_kwargs(
pub(crate) fn update_from_kwargs(
&self,
py: Python<'_>,
kwargs: &Py<PyDict>,
@ -128,7 +60,8 @@ impl LegacyCallbacks for PythonLogger {
Ok(())
}
fn pre_call(
/// `Logging.pre_call`.
pub(crate) fn pre_call(
&self,
py: Python<'_>,
input: &str,
@ -145,7 +78,8 @@ impl LegacyCallbacks for PythonLogger {
Ok(())
}
fn post_call(
/// `Logging.post_call`.
pub(crate) fn post_call(
&self,
py: Python<'_>,
original_response: &str,
@ -163,19 +97,19 @@ impl LegacyCallbacks for PythonLogger {
Ok(())
}
fn defers_async_logging(&self, py: Python<'_>) -> bool {
pub(crate) fn defers_async_logging(&self, py: Python<'_>) -> bool {
Logging::DefersAsync
.call(py, (self.object(py),))
.and_then(|value| value.extract())
.unwrap_or(false)
}
fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> {
pub(crate) fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> {
Logging::DeferSuccess.call(py, (self.object(py), pending))?;
Ok(())
}
fn sync_success_for_async_call(
pub(crate) fn sync_success_for_async_call(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
@ -186,7 +120,7 @@ impl LegacyCallbacks for PythonLogger {
Ok(())
}
fn failure(
pub(crate) fn failure(
&self,
py: Python<'_>,
error: &Py<PyBaseException>,
@ -199,7 +133,7 @@ impl LegacyCallbacks for PythonLogger {
Ok(asynchronous.then(|| value.unbind()))
}
fn submit_success(
pub(crate) fn submit_success(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
@ -210,7 +144,7 @@ impl LegacyCallbacks for PythonLogger {
Ok(())
}
fn enqueue_success(
pub(crate) fn enqueue_success(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
@ -227,6 +161,55 @@ impl LegacyCallbacks for PythonLogger {
}
enqueue.map(|_| ())
}
pub(crate) fn initialize_failure_context(
&self,
py: Python<'_>,
kwargs: &Py<PyDict>,
model: Option<String>,
custom_llm_provider: Option<String>,
) -> PyResult<()> {
let Some(model) = model else {
return Ok(());
};
let (provider, model) = match custom_llm_provider.filter(|provider| !provider.is_empty()) {
Some(provider) => {
let model = model
.strip_prefix(provider.as_str())
.and_then(|model| model.strip_prefix('/'))
.unwrap_or(&model)
.to_string();
(provider, model)
}
None => match model.split_once('/') {
Some((provider, model)) if !provider.is_empty() && !model.is_empty() => {
(provider.to_string(), model.to_string())
}
_ => return Ok(()),
},
};
let kwargs = kwargs.bind(py);
let metadata = PyDict::new(py);
for name in ["metadata", "litellm_metadata"] {
if let Some(value) = kwargs.get_item(name)? {
metadata.set_item(name, value)?;
}
}
let params = PyDict::new(py);
params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?;
Logging::Update.call(
py,
(
self.object(py),
metadata,
model,
PyDict::new(py),
params,
provider,
),
)?;
Ok(())
}
}
fn custom_pricing_fields(py: Python<'_>) -> PyResult<Vec<String>> {
@ -254,6 +237,6 @@ fn redact(
}
/// Proxy-internal calls skip the legacy success fan-out.
pub fn is_internal_call(py: Python<'_>) -> PyResult<bool> {
pub(crate) fn is_internal_call(py: Python<'_>) -> PyResult<bool> {
Wrapper::IsInternalCall.call(py, ())?.extract()
}

View file

@ -1,9 +1,10 @@
//! The proxy's deferred success release: the async success handler is queued only once
//! the proxy accepts the response, and at most once.
use pyo3::{exceptions::PyException, prelude::*};
use litellm_host_python::is_cancellation;
use pyo3::prelude::*;
use crate::{LegacyCallbacks, PythonLogger};
use crate::PythonLogger;
pub(crate) struct PendingSuccess {
pub(crate) logger: PythonLogger,
@ -37,7 +38,7 @@ impl PendingLogging {
&& success
{
match pending.asynchronous(py) {
Err(error) if error.is_instance_of::<PyException>(py) => {
Err(error) if !is_cancellation(py, &error) => {
error.write_unraisable(py, Some(pending.logger.object(py)));
}
result => return result,

View file

@ -6,8 +6,8 @@
//! core never learn which Python object is on the other end.
//!
//! Legacy callbacks receive the caller's own objects and may mutate them. [`PublicCall`]
//! is where those objects live, and [`run_legacy_call`] is how a route hands them over
//! without keeping a copy.
//! is where those objects live; the bridge hands it to [`LegacyPythonLifecycle::new`] and
//! keeps no copy.
mod adapter;
mod call;
@ -20,9 +20,11 @@ mod python;
#[path = "../tests/support.rs"]
mod test_support;
pub(crate) use adapter::LegacyLogging;
pub use adapter::{LegacySurface, PassThroughStream};
pub use call::{PublicCall, run_legacy_call};
pub(crate) use callbacks::{LegacyCallbacks, is_internal_call};
pub(crate) use logger::{DeploymentHooks, PythonLogger, finalize, setup};
pub use adapter::{LegacyPythonLifecycle, LegacyPythonSurface, PassThroughStream};
pub use call::PublicCall;
pub(crate) use callbacks::is_internal_call;
pub(crate) use logger::{
PythonLogger, after_deployment_failure, after_deployment_success, before_deployment_call,
finalize, setup,
};
pub(crate) use preparation::prepare;

View file

@ -1,3 +1,6 @@
//! One call's `Logging` instance (the logger) and the `@client` wrapper functions that
//! create and finish it: `function_setup`, `finalize` and the awaited deployment hooks.
use pyo3::{
exceptions::PyBaseException,
gc::{PyTraverseError, PyVisit},
@ -5,10 +8,10 @@ use pyo3::{
types::{PyDict, PyTuple},
};
use crate::python::{self, Wrapper};
use crate::python::{DeploymentHooks, Wrapper};
/// The `Logging` instance one call fans out through.
pub struct PythonLogger {
pub(crate) struct PythonLogger {
object: Py<PyAny>,
}
@ -21,17 +24,17 @@ impl PythonLogger {
self.object.bind(py)
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
object: self.object.clone_ref(py),
}
}
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.object)
}
pub fn restore_context(&self, py: Python<'_>) -> PyResult<()> {
pub(crate) fn restore_context(&self, py: Python<'_>) -> PyResult<()> {
Wrapper::RestoreContext.call(py, (self.object(py),))?;
Ok(())
}
@ -45,19 +48,19 @@ impl FromPyObject<'_, '_> for PythonLogger {
}
}
pub struct SetupResult<'py>(Bound<'py, PyAny>);
pub(crate) struct SetupResult<'py>(Bound<'py, PyAny>);
impl SetupResult<'_> {
pub fn logger(&self) -> PyResult<PythonLogger> {
pub(crate) fn logger(&self) -> PyResult<PythonLogger> {
Ok(PythonLogger::new(self.0.getattr("logger")?.unbind()))
}
pub fn kwargs(&self) -> PyResult<Py<PyDict>> {
pub(crate) fn kwargs(&self) -> PyResult<Py<PyDict>> {
Ok(self.0.getattr("kwargs")?.extract()?)
}
}
pub fn setup<'py>(
pub(crate) fn setup<'py>(
py: Python<'py>,
call_type: &str,
args: &Py<PyTuple>,
@ -70,7 +73,7 @@ pub fn setup<'py>(
.map(SetupResult)
}
pub fn finalize(
pub(crate) fn finalize(
py: Python<'_>,
response: &Option<Py<PyAny>>,
logger: &PythonLogger,
@ -82,40 +85,36 @@ pub fn finalize(
Ok(())
}
pub struct DeploymentHooks;
pub(crate) fn before_deployment_call(
py: Python<'_>,
kwargs: &Py<PyDict>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
DeploymentHooks::BeforeDeploymentCall
.call(py, (kwargs, call_type))
.map(Bound::unbind)
}
impl DeploymentHooks {
pub fn before_call(
py: Python<'_>,
kwargs: &Py<PyDict>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
python::DeploymentHooks::BeforeDeploymentCall
.call(py, (kwargs, call_type))
.map(Bound::unbind)
}
pub(crate) fn after_deployment_success(
py: Python<'_>,
kwargs: &Py<PyDict>,
response: &Option<Py<PyAny>>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
DeploymentHooks::AfterDeploymentSuccess
.call(py, (kwargs, response, call_type))
.map(Bound::unbind)
}
pub fn after_success(
py: Python<'_>,
kwargs: &Py<PyDict>,
response: &Option<Py<PyAny>>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
python::DeploymentHooks::AfterDeploymentSuccess
.call(py, (kwargs, response, call_type))
.map(Bound::unbind)
}
pub fn after_failure(
py: Python<'_>,
kwargs: &Py<PyDict>,
error: &Py<PyBaseException>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
python::DeploymentHooks::AfterDeploymentFailure
.call(py, (kwargs, error, call_type))
.map(Bound::unbind)
}
pub(crate) fn after_deployment_failure(
py: Python<'_>,
kwargs: &Py<PyDict>,
error: &Py<PyBaseException>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
DeploymentHooks::AfterDeploymentFailure
.call(py, (kwargs, error, call_type))
.map(Bound::unbind)
}
#[cfg(test)]

View file

@ -1,3 +1,6 @@
//! The kwargs rewrites the `@client` wrapper makes before the call: the logger is attached,
//! a named credential's values are inherited, and the budget and retry-count limits run.
use pyo3::{
prelude::*,
types::{PyDict, PyList},
@ -17,7 +20,7 @@ impl<'py> CredentialEntry<'py> {
}
}
pub fn prepare<'py>(
pub(crate) fn prepare<'py>(
py: Python<'py>,
kwargs: &Bound<'py, PyDict>,
logger: &crate::PythonLogger,

View file

@ -1,3 +1,6 @@
//! The Python contract of this crate: every function of
//! `litellm/rust_bridge/callbacks_legacy_python.py` that Rust calls, by name.
use pyo3::prelude::*;
use strum::{IntoStaticStr, VariantArray};
@ -153,7 +156,7 @@ mod tests {
use crate::test_support::PYTHON_CONTRACT;
#[test]
fn every_borrowed_function_is_in_the_python_contract() {
fn every_function_is_in_the_python_contract() {
let contract: serde_json::Map<String, serde_json::Value> =
serde_json::from_str(PYTHON_CONTRACT).unwrap();
let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect();
@ -177,7 +180,7 @@ mod tests {
)
.map(LegacyPython::name)
.collect();
assert_eq!(called.len(), declared.len(), "a function is borrowed twice");
assert_eq!(called.len(), declared.len(), "a function is listed twice");
assert_eq!(called.into_iter().collect::<BTreeSet<_>>(), declared);
}
}

View file

@ -7,7 +7,7 @@ use pyo3::prelude::*;
use pyo3::types::PyDict;
use rstest::rstest;
use super::LegacyLogging;
use super::LegacyPythonLifecycle;
use crate::test_support::{legacy_call, local, namespace, run};
const CALL: &CStr = c"
@ -24,7 +24,7 @@ fn begin<'py>(
py: Python<'py>,
locals: &Bound<'py, PyDict>,
asynchronous: bool,
) -> (LegacyLogging, LifecycleStep) {
) -> (LegacyPythonLifecycle, LifecycleStep) {
let mut logging = legacy_call(py, locals, asynchronous);
let kwargs = local(locals, "kwargs")
.cast_into::<PyDict>()
@ -63,6 +63,59 @@ fn deployment_pre_call_hook_runs_only_for_asynchronous_calls(#[case] asynchronou
});
}
#[test]
fn failure_context_is_initialized_before_argument_preparation() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
failure = RuntimeError('limit check failed')
metadata = {'request': 'ocr-1'}
class FailingLogger(StubLogger):
def update_from_kwargs(self, **update):
self.record('update', update)
self.update = update
def check_limits(self, arguments):
self.record('check_limits', arguments)
raise failure
logger = FailingLogger()
kwargs = {
'logger': logger,
'model': 'provider/model',
'metadata': metadata,
'litellm_call_id': 'call-1',
}
",
);
let mut logging = legacy_call(py, &locals, false);
let kwargs = local(&locals, "kwargs")
.cast_into::<PyDict>()
.unwrap()
.unbind();
let result = logging.begin(py, kwargs, 0.0);
let error = match result {
Err(error) => error,
Ok(_) => panic!("argument preparation succeeded"),
};
assert!(error.value(py).is(local(&locals, "failure")));
run(
py,
&locals,
c"
assert logger.names() == ['update', 'check_limits'], logger.calls
assert logger.update['model'] == 'model', logger.update
assert logger.update['custom_llm_provider'] == 'provider', logger.update
assert logger.update['kwargs']['metadata'] is metadata, logger.update
assert logger.update['litellm_params']['litellm_call_id'] == 'call-1', logger.update
",
);
});
}
#[test]
fn kwargs_returned_by_the_pre_call_hook_are_what_the_call_prepares() {
Python::initialize();

View file

@ -8,7 +8,7 @@ use pyo3::prelude::*;
use rstest::rstest;
use serde_json::{Map, Value, json};
use super::LegacyLogging;
use super::LegacyPythonLifecycle;
use crate::PythonLogger;
use crate::test_support::{legacy_call, local, namespace, run};
@ -77,7 +77,7 @@ fn before_send_bound(
locals.set_item(name, to_py(py, value).unwrap()).unwrap();
}
run(py, &locals, script);
let mut logging = LegacyLogging {
let mut logging = LegacyPythonLifecycle {
logger: Some(PythonLogger::new(local(&locals, "logger").unbind())),
..legacy_call(py, &locals, false)
};

View file

@ -3,7 +3,7 @@ use std::ffi::CStr;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyTuple};
use crate::{LegacyLogging, LegacySurface, PublicCall};
use crate::{LegacyPythonLifecycle, LegacyPythonSurface, PublicCall};
/// The parameters of every `callbacks_legacy_python` function, as the real module declares them.
/// `tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python
@ -125,6 +125,9 @@ class StubLogger:
self.hooks = {}
self.on_enqueue = lambda coroutine: None
def update_from_kwargs(self, **update):
self.update = update
def record(self, name, value):
self.calls.append((name, value))
@ -181,7 +184,7 @@ pub(crate) fn legacy_call(
py: Python<'_>,
locals: &Bound<'_, PyDict>,
asynchronous: bool,
) -> LegacyLogging {
) -> LegacyPythonLifecycle {
let request = locals
.get_item("request")
.unwrap()
@ -192,9 +195,9 @@ pub(crate) fn legacy_call(
.map(|kwargs| kwargs.cast_into::<PyDict>().unwrap())
.unwrap_or_else(|| PyDict::new(py));
let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap();
LegacyLogging::new(
LegacyPythonLifecycle::new(
py,
LegacySurface {
LegacyPythonSurface {
call_type: "test",
input_description: "test input",
stream: None,

View file

@ -8,7 +8,7 @@ use pyo3::prelude::*;
use pyo3::types::PyDict;
use rstest::rstest;
use super::LegacyLogging;
use super::LegacyPythonLifecycle;
use crate::PythonLogger;
use crate::test_support::{legacy_call, local, namespace, run};
@ -17,8 +17,8 @@ const TIMING: Timing = Timing {
end_time: 1.0,
};
fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> LegacyLogging {
LegacyLogging {
fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> LegacyPythonLifecycle {
LegacyPythonLifecycle {
logger: Some(PythonLogger::new(local(locals, "logger").unbind())),
..legacy_call(py, locals, asynchronous)
}
@ -27,7 +27,7 @@ fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> Leg
fn succeed(
py: Python<'_>,
locals: &Bound<'_, PyDict>,
logging: &mut LegacyLogging,
logging: &mut LegacyPythonLifecycle,
) -> LifecycleStep {
let response = local(locals, "response").unbind();
logging
@ -41,7 +41,11 @@ fn succeed(
.unwrap()
}
fn fail(py: Python<'_>, locals: &Bound<'_, PyDict>, logging: &mut LegacyLogging) -> LifecycleStep {
fn fail(
py: Python<'_>,
locals: &Bound<'_, PyDict>,
logging: &mut LegacyPythonLifecycle,
) -> LifecycleStep {
let failure = PyErr::from_value(local(locals, "failure"));
logging
.emit(
@ -105,7 +109,7 @@ fn internal_calls_skip_failure_callbacks_only_when_asynchronous(
Python::initialize();
Python::attach(|py| {
let locals = namespace(py, c"failure = ValueError('provider')");
let mut logging = LegacyLogging {
let mut logging = LegacyPythonLifecycle {
internal: true,
..logged(py, &locals, asynchronous)
};
@ -127,7 +131,7 @@ fn internal_async_calls_skip_the_async_success_fan_out() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(py, c"response = object()");
let mut logging = LegacyLogging {
let mut logging = LegacyPythonLifecycle {
internal: true,
..logged(py, &locals, true)
};

View file

@ -0,0 +1,27 @@
- Target invariants, not completion claims
- Keep this crate the Python runtime adapter for `litellm-callbacks-v1` and nothing else: the subscriber snapshot and one `PythonLifecycle` that delivers the core contract
- Envelopes, sequencing, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1`
- `litellm/rust_bridge/callbacks_v1_python.py` is the only Python module behind it; `python_contract.json` pins its functions on both sides
- Terms (contract terms are in `callbacks-v1/AGENTS.md`, lifecycle terms in `host-python/AGENTS.md`)
- Registry: the process-wide subscriber list; it lives in Python (`register`/`unregister`), never in Rust
- Snapshot: that list read once per call, with one observer and one interceptor handler already selected per subscriber for the call's mode (`subscribers.rs`)
- Projection: the public response turned into JSON by `project_response`; a failed projection becomes `response_error`, never a failed call
- Report: logging an observer's swallowed error through `report`
- Continuation: what a step returns to the driver once every observer of its envelope has run
- "Callback" in the Python module (`Callback` protocol) is the user's object; once registered it is a subscriber
- No Python object of the call reaches a callback; a callback influences a call only through a patch Rust validates
- `begin` returns the dict it was given by identity; `after_success` and `delivered` stay the trait's pass-through defaults
- Every callback gets a freshly built envelope per event; never cache or share a converted one
- Contract values cross into Python only through a fresh conversion per callback
- Schema v1 is owned by `litellm-callbacks-v1`; this crate only marshals it
- Everything rejectable is rejected at `register`, never during a call; the snapshot only parses and the lifecycle never re-validates a subscriber
- Observers cannot affect a call: ordinary exceptions are reported and swallowed at every event, awaited or not; an interceptor failure fails the call
- A cancellation is returned at once, with no further dispatch and no report
- Exactly one terminal envelope, last; none after a cancellation
- In a sync call no step suspends: handler selection happens once, at snapshot, from what registration recorded
- Every suspension is awaited inline in the caller's task; never `into_future`, never spawn
- The subscriber snapshot is taken once per call before the driver starts; later registrations do not reach that call
- `V1PythonLifecycle::subscribed` is the only constructor callers get; it returns `None` for an empty snapshot, so an unobserved call builds no envelope and projects no response
- This lifecycle may run after another one in a `LifecycleChain`; it observes the arguments and the wire as that one left them and never assumes it is alone
- `close` is idempotent; `traverse` visits every retained Python edge and calls no Python
- This crate has none of `callbacks-legacy-python`'s obligations and must not grow them; shared behaviour belongs to `PythonLifecycle`, not to either adapter

View file

@ -0,0 +1,18 @@
[package]
name = "litellm-callbacks-v1-python"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
autotests = false
[dependencies]
litellm-host.workspace = true
litellm-host-python.workspace = true
litellm-callbacks-v1.workspace = true
pyo3.workspace = true
serde_json.workspace = true
strum.workspace = true
[dev-dependencies]
litellm-auth.workspace = true

View file

@ -0,0 +1,6 @@
{
"snapshot": [],
"project_response": ["response"],
"new_call_id": [],
"report": ["name", "event", "error"]
}

View file

@ -0,0 +1,359 @@
//! The v1 lifecycle: each step builds one envelope, hands every subscribed observer a
//! fresh conversion of it, and, before the request is sent, folds every interceptor's
//! patch into the wire request.
use litellm_callbacks_v1::{
CallFacts, Envelope, ErrorFacts, Event, RequestFacts, Sequencer, WirePatch, apply,
};
use litellm_host::event::{MachineEvent, RequestContext, WireRequest};
use litellm_host_python::{
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py,
};
use pyo3::exceptions::PyValueError;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyString};
use serde_json::{Map, Value};
use crate::call::V1PythonSurface;
use crate::python::V1Python;
use crate::subscribers::Subscriber;
/// What a step hands back to the driver once every observer of its envelope has run.
enum Continuation {
Done,
Arguments(Py<PyDict>),
Wire(Box<WireRequest>),
}
/// The suspended step; `index` is the subscriber whose awaitable the driver is awaiting.
enum Pending {
Observe {
envelope: Envelope,
index: usize,
then: Continuation,
},
Intercept {
wire: Box<WireRequest>,
context: RequestContext,
index: usize,
},
}
pub struct V1PythonLifecycle {
surface: V1PythonSurface,
subscribers: Vec<Subscriber>,
asynchronous: bool,
sequencer: Option<Sequencer>,
streamed: bool,
pending: Option<Pending>,
closed: bool,
}
impl Continuation {
fn into_step(self) -> LifecycleStep {
match self {
Self::Done => LifecycleStep::Done,
Self::Arguments(arguments) => LifecycleStep::Arguments(arguments),
Self::Wire(wire) => LifecycleStep::Wire(wire),
}
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
match self {
Self::Arguments(arguments) => visit.call(arguments),
Self::Done | Self::Wire(_) => Ok(()),
}
}
}
impl V1PythonLifecycle {
pub(crate) fn new(
surface: V1PythonSurface,
subscribers: Vec<Subscriber>,
asynchronous: bool,
) -> Self {
Self {
surface,
subscribers,
asynchronous,
sequencer: None,
streamed: false,
pending: None,
closed: false,
}
}
fn envelope(&mut self, event: Event) -> PyResult<Envelope> {
self.sequencer
.as_mut()
.map(|sequencer| sequencer.envelope(event))
.ok_or_else(missing_state)
}
fn report(
&self,
py: Python<'_>,
index: usize,
envelope: &Envelope,
error: PyErr,
) -> PyResult<()> {
let name = &self.subscribers[index].name;
let event: &'static str = envelope.event.kind().into();
match V1Python::Report.call(py, (name, event, error.value(py))) {
Err(report_error) if is_cancellation(py, &report_error) => Err(report_error),
_ => Ok(()),
}
}
fn observe(
&mut self,
py: Python<'_>,
envelope: Envelope,
from: usize,
then: Continuation,
) -> PyResult<LifecycleStep> {
for index in from..self.subscribers.len() {
let Some(handler) = self.subscribers[index]
.observer(envelope.event.kind())
.map(|handler| handler.clone_ref(py))
else {
continue;
};
let outcome = to_py(py, &envelope).and_then(|argument| handler.call(py, argument));
match outcome {
Ok(awaitable) if handler.is_async() => {
self.pending = Some(Pending::Observe {
envelope,
index,
then,
});
return Ok(LifecycleStep::Await(awaitable.unbind()));
}
Ok(_) => {}
Err(error) if is_cancellation(py, &error) => return Err(error),
Err(error) => self.report(py, index, &envelope, error)?,
}
}
Ok(then.into_step())
}
fn apply_patch(
&self,
py: Python<'_>,
index: usize,
wire: WireRequest,
result: Py<PyAny>,
) -> PyResult<WireRequest> {
let patch = if result.bind(py).is_none() {
WirePatch::default()
} else {
from_py::<WirePatch>(result.bind(py))?
};
let name = &self.subscribers[index].name;
apply(wire, patch)
.map_err(|error| PyValueError::new_err(format!("callback {name}: {error}")))
}
fn intercept(
&mut self,
py: Python<'_>,
wire: Box<WireRequest>,
context: RequestContext,
from: usize,
) -> PyResult<LifecycleStep> {
let mut current = wire;
for index in from..self.subscribers.len() {
let Some(handler) = self.subscribers[index]
.intercept
.as_ref()
.map(|handler| handler.clone_ref(py))
else {
continue;
};
let request = to_py(py, &RequestFacts::new(&current, &context))?;
let result = handler.call(py, request)?;
if handler.is_async() {
self.pending = Some(Pending::Intercept {
wire: current,
context,
index,
});
return Ok(LifecycleStep::Await(result.unbind()));
}
current = Box::new(self.apply_patch(py, index, *current, result.unbind())?);
}
let envelope =
self.envelope(Event::RequestSending(RequestFacts::new(&current, &context)))?;
self.observe(py, envelope, 0, Continuation::Wire(current))
}
fn metadata(arguments: &Bound<'_, PyDict>) -> PyResult<(Map<String, Value>, Vec<String>)> {
let Some(value) = arguments.get_item("metadata")? else {
return Ok((Map::new(), Vec::new()));
};
let Ok(metadata) = value.cast::<PyDict>() else {
return Ok((Map::new(), vec!["metadata".to_string()]));
};
let mut included = Map::new();
let mut dropped = Vec::new();
for (key, value) in metadata.iter() {
match key.extract::<String>() {
Ok(name) => match from_py::<Value>(&value) {
Ok(value) => {
included.insert(name, value);
}
Err(_) => dropped.push(name),
},
Err(_) => dropped.push(key.str()?.to_string()),
}
}
Ok((included, dropped))
}
fn call_id(py: Python<'_>, arguments: &Bound<'_, PyDict>) -> PyResult<String> {
if let Some(value) = arguments.get_item("litellm_call_id")?
&& value.is_instance_of::<PyString>()
{
return value.extract();
}
V1Python::NewCallId.call(py, ())?.extract()
}
fn error_facts(py: Python<'_>, error: &PyErr) -> PyResult<ErrorFacts> {
let class = error.get_type(py);
let module = class.getattr("__module__")?.extract::<String>()?;
let qualified = class.getattr("__qualname__")?.extract::<String>()?;
let status_code = error
.value(py)
.getattr("status_code")
.ok()
.and_then(|value| value.extract::<u16>().ok());
Ok(ErrorFacts {
class: format!("{module}.{qualified}"),
message: error.value(py).str()?.to_string(),
status_code,
})
}
}
impl PythonLifecycle for V1PythonLifecycle {
fn begin(
&mut self,
py: Python<'_>,
arguments: Py<PyDict>,
start_time: f64,
) -> PyResult<LifecycleStep> {
let call_id = Self::call_id(py, arguments.bind(py))?;
let (metadata, metadata_dropped) = Self::metadata(arguments.bind(py))?;
self.sequencer = Some(Sequencer::new(call_id, self.surface.call_type));
let envelope = self.envelope(Event::CallStarted(CallFacts {
start_time,
asynchronous: self.asynchronous,
metadata,
metadata_dropped,
}))?;
self.observe(py, envelope, 0, Continuation::Arguments(arguments))
}
fn before_send(
&mut self,
py: Python<'_>,
wire: Box<WireRequest>,
context: &RequestContext,
) -> PyResult<LifecycleStep> {
self.intercept(py, wire, context.clone(), 0)
}
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
match event {
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
let envelope = self.envelope(Event::ResponseReceived {
body: raw.body.clone(),
})?;
self.observe(py, envelope, 0, Continuation::Done)
}
LifecycleEvent::Succeeded { timing, response } => {
let projected = V1Python::ProjectResponse
.call(py, (response,))
.and_then(|value| from_py::<Value>(&value));
let (response, response_error) = match projected {
Ok(response) => (response, None),
Err(error) if is_cancellation(py, &error) => return Err(error),
Err(error) => (Value::Null, Some(error.to_string())),
};
let envelope = self.envelope(Event::CallSucceeded {
timing: timing.into(),
streamed: self.streamed,
response,
response_error,
})?;
self.observe(py, envelope, 0, Continuation::Done)
}
LifecycleEvent::Failed {
timing,
origin,
error,
} => {
let facts = Self::error_facts(py, error)?;
let envelope = self.envelope(Event::CallFailed {
timing: timing.into(),
streamed: self.streamed,
origin: origin.into(),
error: facts,
})?;
self.observe(py, envelope, 0, Continuation::Done)
}
}
}
fn opened(&mut self, _py: Python<'_>) -> PyResult<()> {
self.streamed = true;
Ok(())
}
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
match self.pending.take().ok_or_else(missing_state)? {
Pending::Observe {
envelope,
index,
then,
} => {
if let Err(error) = result {
if is_cancellation(py, &error) {
return Err(error);
}
self.report(py, index, &envelope, error)?;
}
self.observe(py, envelope, index + 1, then)
}
Pending::Intercept {
wire,
context,
index,
} => {
let patched = self.apply_patch(py, index, *wire, result?)?;
self.intercept(py, Box::new(patched), context, index + 1)
}
}
}
fn close(&mut self, _py: Python<'_>) {
if self.closed {
return;
}
self.closed = true;
self.pending = None;
self.subscribers.clear();
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
for subscriber in &self.subscribers {
subscriber.traverse(visit)?;
}
match &self.pending {
Some(Pending::Observe { then, .. }) => then.traverse(visit),
Some(Pending::Intercept { .. }) | None => Ok(()),
}
}
}

View file

@ -0,0 +1,25 @@
//! What a route tells the v1 contract about itself, and the one constructor callers get.
use pyo3::prelude::*;
use crate::adapter::V1PythonLifecycle;
use crate::subscribers::snapshot;
#[derive(Clone, Copy, Debug)]
pub struct V1PythonSurface {
pub call_type: &'static str,
}
impl V1PythonLifecycle {
/// The lifecycle for one call, over the subscribers registered right now; later
/// registrations do not reach that call. `None` when nothing is subscribed, so an
/// unobserved call builds no envelope at all.
pub fn subscribed(
py: Python<'_>,
surface: V1PythonSurface,
asynchronous: bool,
) -> PyResult<Option<Self>> {
let subscribers = snapshot(py, asynchronous)?;
Ok((!subscribers.is_empty()).then(|| Self::new(surface, subscribers, asynchronous)))
}
}

View file

@ -0,0 +1,20 @@
//! The Python adapter of the v1 callback contract: the per-call subscriber snapshot and
//! one [`PythonLifecycle`](litellm_host_python::PythonLifecycle) that marshals
//! `litellm-callbacks-v1` envelopes and patches to Python handlers. The contract itself
//! lives in `litellm-callbacks-v1`.
mod adapter;
mod call;
mod python;
mod subscribers;
pub use adapter::V1PythonLifecycle;
pub use call::V1PythonSurface;
#[cfg(test)]
#[path = "../tests/lifecycle.rs"]
mod lifecycle_tests;
#[cfg(test)]
#[path = "../tests/support.rs"]
mod test_support;

View file

@ -0,0 +1,56 @@
//! The Python contract of this crate: every function of
//! `litellm/rust_bridge/callbacks_v1_python.py` that Rust calls, by name.
use pyo3::prelude::*;
use strum::{IntoStaticStr, VariantArray};
const MODULE: &str = "litellm.rust_bridge.callbacks_v1_python";
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)]
pub(crate) enum V1Python {
#[strum(serialize = "snapshot")]
Snapshot,
#[strum(serialize = "project_response")]
ProjectResponse,
#[strum(serialize = "new_call_id")]
NewCallId,
#[strum(serialize = "report")]
Report,
}
impl V1Python {
fn name(self) -> &'static str {
self.into()
}
pub(crate) fn call<'py, A>(self, py: Python<'py>, args: A) -> PyResult<Bound<'py, PyAny>>
where
A: pyo3::call::PyCallArgs<'py>,
{
py.import(MODULE)?.getattr(self.name())?.call1(args)
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use strum::VariantArray;
use super::V1Python;
use crate::test_support::PYTHON_CONTRACT;
#[test]
fn every_function_is_in_the_python_contract() {
let contract: serde_json::Map<String, serde_json::Value> =
serde_json::from_str(PYTHON_CONTRACT).unwrap();
let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect();
let called: Vec<&str> = V1Python::VARIANTS
.iter()
.copied()
.map(V1Python::name)
.collect();
assert_eq!(called.len(), declared.len(), "a function is listed twice");
assert_eq!(called.into_iter().collect::<BTreeSet<_>>(), declared);
}
}

View file

@ -0,0 +1,113 @@
//! The per-call subscriber snapshot: what Python's `register` recorded, read once, with
//! one handler chosen per role for the call's execution mode. `register` has already
//! rejected everything rejectable, so this only parses.
use std::collections::BTreeSet;
use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, select_handler};
use pyo3::exceptions::PyValueError;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use crate::python::V1Python;
/// One callable of a subscriber, tagged with whether calling it returns an awaitable.
pub enum Handler {
Sync(Py<PyAny>),
Async(Py<PyAny>),
}
pub struct Subscriber {
pub name: String,
pub events: BTreeSet<EventKind>,
/// `on_event` or `async_on_event`: cannot affect the call.
pub observe: Option<Handler>,
/// `before_send` or `async_before_send`: returns a patch; its failure fails the call.
pub intercept: Option<Handler>,
}
fn selected_handler(
subscriber: &Bound<'_, PyAny>,
mode: ExecutionMode,
sync_name: &str,
async_name: &str,
) -> PyResult<Option<Handler>> {
let sync = subscriber.getattr(sync_name)?;
let asynchronous = subscriber.getattr(async_name)?;
Ok(
match select_handler(mode, !sync.is_none(), !asynchronous.is_none()) {
HandlerSelection::Sync => Some(Handler::Sync(sync.unbind())),
HandlerSelection::Async => Some(Handler::Async(asynchronous.unbind())),
HandlerSelection::Skip => None,
},
)
}
pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<Subscriber>> {
let mode = if asynchronous {
ExecutionMode::Async
} else {
ExecutionMode::Sync
};
V1Python::Snapshot
.call(py, ())?
.try_iter()?
.map(|item| {
let subscriber = item?;
let events = subscriber
.getattr("events")?
.extract::<BTreeSet<String>>()?
.into_iter()
.map(|name| {
name.parse::<EventKind>().map_err(|_| {
PyValueError::new_err(format!("unknown callback event: {name}"))
})
})
.collect::<PyResult<BTreeSet<_>>>()?;
Ok(Subscriber {
name: subscriber.getattr("name")?.extract()?,
events,
observe: selected_handler(&subscriber, mode, "on_event", "async_on_event")?,
intercept: selected_handler(&subscriber, mode, "before_send", "async_before_send")?,
})
})
.collect()
}
impl Subscriber {
pub fn observer(&self, kind: EventKind) -> Option<&Handler> {
self.observe
.as_ref()
.filter(|_| self.events.contains(&kind))
}
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
for handler in self.observe.iter().chain(&self.intercept) {
visit.call(handler.object())?;
}
Ok(())
}
}
impl Handler {
fn object(&self) -> &Py<PyAny> {
match self {
Self::Sync(value) | Self::Async(value) => value,
}
}
pub fn is_async(&self) -> bool {
matches!(self, Self::Async(_))
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
match self {
Self::Sync(value) => Self::Sync(value.clone_ref(py)),
Self::Async(value) => Self::Async(value.clone_ref(py)),
}
}
pub fn call<'py>(&self, py: Python<'py>, argument: Py<PyAny>) -> PyResult<Bound<'py, PyAny>> {
self.object().bind(py).call1((argument,))
}
}

View file

@ -0,0 +1,497 @@
use std::collections::BTreeSet;
use litellm_auth::SecretValue;
use litellm_host::event::{MachineEvent, RawResponse, RequestContext, Timing, WireRequest};
use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle, from_py};
use pyo3::exceptions::asyncio::CancelledError;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use serde_json::{Value, json};
use crate::adapter::V1PythonLifecycle;
use crate::call::V1PythonSurface;
use crate::subscribers::{Handler, Subscriber};
use crate::test_support::{local, namespace};
use litellm_callbacks_v1::EventKind;
const TIMING: Timing = Timing {
start_time: 10.0,
end_time: 12.5,
};
fn subscriber(
locals: &Bound<'_, PyDict>,
name: &str,
events: &[EventKind],
observe: Option<(&str, bool)>,
intercept: Option<(&str, bool)>,
) -> Subscriber {
let handler = |spec: Option<(&str, bool)>| {
spec.map(|(local_name, asynchronous)| {
let callable = local(locals, local_name).unbind();
if asynchronous {
Handler::Async(callable)
} else {
Handler::Sync(callable)
}
})
};
Subscriber {
name: name.to_string(),
events: events.iter().copied().collect::<BTreeSet<_>>(),
observe: handler(observe),
intercept: handler(intercept),
}
}
fn lifecycle(subscribers: Vec<Subscriber>, asynchronous: bool) -> V1PythonLifecycle {
V1PythonLifecycle::new(
V1PythonSurface { call_type: "ocr" },
subscribers,
asynchronous,
)
}
fn context(secret: &str) -> RequestContext {
RequestContext {
model: "model".to_string(),
custom_llm_provider: "provider".to_string(),
optional_params: json!({"api_key": secret, "temperature": 0.2}),
secret_fields: vec!["api_key".to_string()],
api_key: Some(SecretValue::new(secret.to_string())),
}
}
fn wire(secret: &str) -> WireRequest {
WireRequest {
url: "https://provider.example/v1".to_string(),
headers: vec![
("Authorization".to_string(), secret.to_string()),
("X-Trace".to_string(), "one".to_string()),
],
body: json!({"input": "hello"}),
}
}
#[test]
fn full_sync_success_preserves_identity_sequence_and_secrets() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
import copy
seen = []
def observe(event):
seen.append(copy.deepcopy(event))
response = {'id': 'response'}
",
);
let events = [
EventKind::CallStarted,
EventKind::RequestSending,
EventKind::ResponseReceived,
EventKind::CallSucceeded,
];
let observer = subscriber(
&locals,
"sync-success",
&events,
Some(("observe", false)),
None,
);
let mut adapter = lifecycle(vec![observer], false);
let arguments = PyDict::new(py);
arguments.set_item("litellm_call_id", "call-123").unwrap();
let LifecycleStep::Arguments(returned) =
adapter.begin(py, arguments.clone().unbind(), 10.0).unwrap()
else {
panic!("begin did not return arguments");
};
assert!(returned.bind(py).is(&arguments));
let secret = "sentinel-secret-78e4";
let LifecycleStep::Wire(sent) = adapter
.before_send(py, Box::new(wire(secret)), &context(secret))
.unwrap()
else {
panic!("before_send did not return wire");
};
assert_eq!(sent.headers[0].1, secret);
let raw = MachineEvent::ResponseReceived {
raw: RawResponse {
body: "raw response".to_string(),
},
};
assert!(matches!(
adapter.emit(py, LifecycleEvent::Machine(&raw)).unwrap(),
LifecycleStep::Done
));
let response = local(&locals, "response").unbind();
let LifecycleStep::Response(returned) = adapter
.after_success(py, response.clone_ref(py), TIMING)
.unwrap()
else {
panic!("after_success did not return response");
};
assert!(returned.bind(py).is(response.bind(py)));
assert!(matches!(
adapter
.emit(
py,
LifecycleEvent::Succeeded {
timing: TIMING,
response: &response,
},
)
.unwrap(),
LifecycleStep::Done
));
let seen = from_py::<Vec<Value>>(&local(&locals, "seen")).unwrap();
assert_eq!(
seen.iter()
.map(|envelope| envelope["event"]["type"].as_str().unwrap())
.collect::<Vec<_>>(),
[
"call.started",
"request.sending",
"response.received",
"call.succeeded",
]
);
assert_eq!(
seen.iter()
.map(|envelope| envelope["seq"].as_u64().unwrap())
.collect::<Vec<_>>(),
[0, 1, 2, 3]
);
assert!(!serde_json::to_string(&seen).unwrap().contains(secret));
});
}
#[test]
fn every_observer_gets_a_fresh_envelope() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
import copy
seen = []
def mutate(event):
event['event']['body']['input'] = 'changed'
def record(event):
seen.append(copy.deepcopy(event))
",
);
let event = [EventKind::RequestSending];
let subscribers = vec![
subscriber(&locals, "mutator", &event, Some(("mutate", false)), None),
subscriber(&locals, "recorder", &event, Some(("record", false)), None),
];
let mut adapter = lifecycle(subscribers, false);
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
let LifecycleStep::Wire(sent) = adapter
.before_send(py, Box::new(wire("secret")), &context("secret"))
.unwrap()
else {
panic!("before_send did not return wire");
};
assert_eq!(sent.body, json!({"input": "hello"}));
let seen = from_py::<Vec<Value>>(&local(&locals, "seen")).unwrap();
assert_eq!(seen[0]["event"]["body"], json!({"input": "hello"}));
});
}
#[test]
fn ordinary_observer_errors_are_reported_and_swallowed() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
seen = []
def fail(event):
raise ValueError('observer failed')
def record(event):
seen.append(event['event']['type'])
",
);
let event = [EventKind::CallStarted];
let subscribers = vec![
subscriber(
&locals,
"observer-error-401",
&event,
Some(("fail", false)),
None,
),
subscriber(
&locals,
"after-error",
&event,
Some(("record", false)),
None,
),
];
let mut adapter = lifecycle(subscribers, false);
assert!(matches!(
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(),
LifecycleStep::Arguments(_)
));
assert_eq!(
local(&locals, "seen").extract::<Vec<String>>().unwrap(),
["call.started"]
);
let reports = py
.import("litellm.rust_bridge.callbacks_v1_python")
.unwrap()
.getattr("reports")
.unwrap();
let names = reports
.try_iter()
.unwrap()
.filter_map(Result::ok)
.filter_map(|report| report.get_item(0).ok()?.extract::<String>().ok())
.collect::<Vec<_>>();
assert!(names.contains(&"observer-error-401".to_string()));
});
}
#[test]
fn cancellation_stops_observer_fanout() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
import asyncio
seen = []
def cancel(event):
raise asyncio.CancelledError()
def record(event):
seen.append(event)
",
);
let event = [EventKind::CallStarted];
let subscribers = vec![
subscriber(&locals, "cancel", &event, Some(("cancel", false)), None),
subscriber(
&locals,
"after-cancel",
&event,
Some(("record", false)),
None,
),
];
let mut adapter = lifecycle(subscribers, false);
let error = match adapter.begin(py, PyDict::new(py).unbind(), 10.0) {
Err(error) => error,
Ok(_) => panic!("cancellation was swallowed"),
};
assert!(error.is_instance_of::<CancelledError>(py));
assert_eq!(local(&locals, "seen").len().unwrap(), 0);
});
}
#[test]
fn async_observers_resume_in_order_and_return_arguments() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
calls = []
def first(event):
calls.append('first')
return object()
def second(event):
calls.append('second')
return object()
",
);
let event = [EventKind::CallStarted];
let subscribers = vec![
subscriber(&locals, "async-first", &event, Some(("first", true)), None),
subscriber(
&locals,
"async-second",
&event,
Some(("second", true)),
None,
),
];
let mut adapter = lifecycle(subscribers, true);
let arguments = PyDict::new(py).unbind();
assert!(matches!(
adapter.begin(py, arguments.clone_ref(py), 10.0).unwrap(),
LifecycleStep::Await(_)
));
assert!(matches!(
adapter.resume(py, Ok(py.None())).unwrap(),
LifecycleStep::Await(_)
));
let LifecycleStep::Arguments(returned) = adapter.resume(py, Ok(py.None())).unwrap() else {
panic!("resume did not return arguments");
};
assert!(returned.bind(py).is(arguments.bind(py)));
assert_eq!(
local(&locals, "calls").extract::<Vec<String>>().unwrap(),
["first", "second"]
);
});
}
#[test]
fn interceptors_chain_and_observers_see_the_final_wire() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
seen = []
def first(request):
return {'headers': {'set': [['X-Added', 'yes']]}, 'body': {'step': 1}}
def second(request):
seen.append(request)
return None
def observe(event):
seen.append(event)
",
);
let subscribers = vec![
subscriber(&locals, "first", &[], None, Some(("first", false))),
subscriber(&locals, "second", &[], None, Some(("second", false))),
subscriber(
&locals,
"observer",
&[EventKind::RequestSending],
Some(("observe", false)),
None,
),
];
let mut adapter = lifecycle(subscribers, false);
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
let LifecycleStep::Wire(sent) = adapter
.before_send(py, Box::new(wire("secret")), &context("secret"))
.unwrap()
else {
panic!("before_send did not return wire");
};
assert_eq!(sent.body, json!({"step": 1}));
assert_eq!(
sent.headers.last().unwrap(),
&("X-Added".to_string(), "yes".to_string())
);
let seen = from_py::<Vec<Value>>(&local(&locals, "seen")).unwrap();
assert_eq!(seen[0]["body"], json!({"step": 1}));
assert_eq!(seen[1]["event"]["body"], json!({"step": 1}));
});
}
#[test]
fn invalid_interceptor_patch_stops_later_handlers() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
seen = []
def invalid(request):
return {'headers': {'set': [['Authorization', 'replacement']]}}
def later(request):
seen.append('later')
def observe(event):
seen.append('observe')
",
);
let subscribers = vec![
subscriber(
&locals,
"invalid-patch",
&[],
None,
Some(("invalid", false)),
),
subscriber(&locals, "later", &[], None, Some(("later", false))),
subscriber(
&locals,
"observer",
&[EventKind::RequestSending],
Some(("observe", false)),
None,
),
];
let mut adapter = lifecycle(subscribers, false);
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
let error = match adapter.before_send(py, Box::new(wire("secret")), &context("secret")) {
Err(error) => error,
Ok(_) => panic!("invalid patch was accepted"),
};
assert!(error.to_string().contains("invalid-patch"));
assert_eq!(local(&locals, "seen").len().unwrap(), 0);
});
}
#[test]
fn projection_failure_is_observed_without_failing_terminal_dispatch() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
py,
c"
seen = []
def observe(event):
seen.append(event)
class Broken:
project_error = True
response = Broken()
",
);
let observer = subscriber(
&locals,
"terminal",
&[EventKind::CallSucceeded],
Some(("observe", false)),
None,
);
let mut adapter = lifecycle(vec![observer], false);
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
adapter.opened(py).unwrap();
let response = local(&locals, "response").unbind();
assert!(matches!(
adapter
.emit(
py,
LifecycleEvent::Succeeded {
timing: TIMING,
response: &response,
},
)
.unwrap(),
LifecycleStep::Done
));
let seen = from_py::<Vec<Value>>(&local(&locals, "seen")).unwrap();
assert_eq!(seen[0]["event"]["streamed"], true);
assert!(seen[0]["event"]["response"].is_null());
assert!(
seen[0]["event"]["response_error"]
.as_str()
.unwrap()
.contains("projection failed")
);
});
}
#[test]
fn resume_without_pending_fails_and_close_is_idempotent() {
Python::initialize();
Python::attach(|py| {
let _locals = namespace(py, c"");
let mut adapter = lifecycle(Vec::new(), false);
assert!(adapter.resume(py, Ok(py.None())).is_err());
adapter.close(py);
adapter.close(py);
});
}

View file

@ -0,0 +1,58 @@
use std::ffi::CStr;
use pyo3::prelude::*;
use pyo3::types::PyDict;
pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
const STUBS: &CStr = c"
import inspect
import json
import sys
import types
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1_python'):
sys.modules.setdefault(name, types.ModuleType(name))
module = sys.modules['litellm.rust_bridge.callbacks_v1_python']
contract = json.loads(python_contract)
def contracted(name, fake):
signature = inspect.Signature([
inspect.Parameter(parameter, inspect.Parameter.POSITIONAL_OR_KEYWORD)
for parameter in contract[name]
])
def checked(*args, **kwargs):
signature.bind(*args, **kwargs)
return fake(*args, **kwargs)
return checked
if not hasattr(module, 'reports'):
module.reports = []
def project_response(response):
if getattr(response, 'project_error', False):
raise ValueError('projection failed')
return response
functions = {
'snapshot': lambda: (),
'project_response': project_response,
'new_call_id': lambda: 'generated-call-id',
'report': lambda name, event, error: module.reports.append((name, event, error)),
}
for name, function in functions.items():
setattr(module, name, contracted(name, function))
";
pub(crate) fn namespace<'py>(py: Python<'py>, script: &CStr) -> Bound<'py, PyDict> {
let locals = PyDict::new(py);
locals.set_item("python_contract", PYTHON_CONTRACT).unwrap();
py.run(STUBS, Some(&locals), Some(&locals)).unwrap();
py.run(script, Some(&locals), Some(&locals)).unwrap();
locals
}
pub(crate) fn local<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py, PyAny> {
locals.get_item(name).unwrap().unwrap()
}

View file

@ -0,0 +1,17 @@
- Target invariants, not completion claims
- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction, handler selection and validated wire patches
- This crate may depend on `litellm-host`, but never on `litellm-host-python`, PyO3 or a runtime registry
- `golden/v1` pins every serialized envelope shape; every `EventKind` has a golden file by construction, and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are tested against those files
- Terms (shared ones are in `host-python/AGENTS.md`)
- Subscriber: one registered consumer of the contract: a name, a schema, the event kinds it wants and its handlers
- Handler: one callable of a subscriber; an observer receives envelopes and cannot affect the call; an interceptor receives `RequestFacts` and returns a patch
- Handler selection (`select_handler`): an async call prefers the async handler and falls back to the sync one; a sync call never selects an async handler
- Event: one thing that happened in a call; `EventKind` is its dotted name (`call.started`, `request.sending`, `response.received`, `call.succeeded`, `call.failed`), the serde tag and the subscription name at once
- Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; `Sequencer` numbers a call's envelopes from zero
- Facts (`CallFacts`, `RequestFacts`, `TimingFacts`, `ErrorFacts`): a redacted, serializable projection of runtime state; never a live object
- Patch (`WirePatch`): an interceptor's edit of the wire: header `remove`, then header `set`, optional `body` replacement; `apply` validates it
- Redaction: credential headers by name (`CREDENTIAL_HEADERS`) and optional params by the route's secret fields become `[REDACTED]`; shape never changes
- No runtime object of a call reaches a callback contract value
- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction; its fields are private so no other crate can build one
- A patch cannot change the URL or set or remove a credential header
- Schema v1 is additive-only; a rename, removal or retype requires a new contract version

View file

@ -0,0 +1,19 @@
[package]
name = "litellm-callbacks-v1"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
autotests = false
[dependencies]
litellm-host.workspace = true
serde.workspace = true
serde_json.workspace = true
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]
litellm-auth.workspace = true
proptest.workspace = true
rstest.workspace = true

View file

@ -0,0 +1,13 @@
{
"schema": 1,
"call_id": "call-123",
"call_type": "ocr",
"seq": 4,
"event": {
"type": "call.failed",
"timing": {"start_time": 10.0, "end_time": 11.0, "duration": 1.0},
"streamed": true,
"origin": "call",
"error": {"class": "builtins.ValueError", "message": "bad request", "status_code": 400}
}
}

View file

@ -0,0 +1,13 @@
{
"schema": 1,
"call_id": "call-123",
"call_type": "ocr",
"seq": 0,
"event": {
"type": "call.started",
"start_time": 10.0,
"asynchronous": true,
"metadata": {"team": "core"},
"metadata_dropped": ["broken"]
}
}

View file

@ -0,0 +1,13 @@
{
"schema": 1,
"call_id": "call-123",
"call_type": "ocr",
"seq": 3,
"event": {
"type": "call.succeeded",
"timing": {"start_time": 10.0, "end_time": 12.5, "duration": 2.5},
"streamed": false,
"response": {"id": "response"},
"response_error": null
}
}

View file

@ -0,0 +1,15 @@
{
"schema": 1,
"call_id": "call-123",
"call_type": "ocr",
"seq": 1,
"event": {
"type": "request.sending",
"model": "model",
"custom_llm_provider": "provider",
"optional_params": {"temperature": 0.2},
"url": "https://provider.example/v1",
"headers": [["Authorization", "[REDACTED]"]],
"body": {"input": "hello"}
}
}

View file

@ -0,0 +1,10 @@
{
"schema": 1,
"call_id": "call-123",
"call_type": "ocr",
"seq": 2,
"event": {
"type": "response.received",
"body": "raw response"
}
}

View file

@ -0,0 +1,315 @@
//! The v1 envelope: what a subscriber receives. An [`Envelope`] is the schema version, the
//! call's identity, a sequence number and one [`Event`]; every `*Facts` type is a
//! redacted, serializable projection that never holds a runtime object of the call.
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
use serde::Serialize;
use serde_json::{Map, Value};
use strum::{EnumString, IntoStaticStr, VariantArray};
use crate::redact;
pub const SCHEMA_V1: u32 = 1;
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Envelope {
pub schema: u32,
pub call_id: String,
pub call_type: &'static str,
pub seq: u32,
pub event: Event,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
#[serde(tag = "type")]
pub enum Event {
#[serde(rename = "call.started")]
CallStarted(CallFacts),
#[serde(rename = "request.sending")]
RequestSending(RequestFacts),
#[serde(rename = "response.received")]
ResponseReceived { body: String },
#[serde(rename = "call.succeeded")]
CallSucceeded {
timing: TimingFacts,
streamed: bool,
response: Value,
response_error: Option<String>,
},
#[serde(rename = "call.failed")]
CallFailed {
timing: TimingFacts,
streamed: bool,
origin: Origin,
error: ErrorFacts,
},
}
#[derive(
EnumString,
IntoStaticStr,
VariantArray,
Clone,
Copy,
PartialEq,
Eq,
Hash,
PartialOrd,
Ord,
Debug,
)]
pub enum EventKind {
#[strum(serialize = "call.started")]
CallStarted,
#[strum(serialize = "request.sending")]
RequestSending,
#[strum(serialize = "response.received")]
ResponseReceived,
#[strum(serialize = "call.succeeded")]
CallSucceeded,
#[strum(serialize = "call.failed")]
CallFailed,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct CallFacts {
pub start_time: f64,
pub asynchronous: bool,
pub metadata: Map<String, Value>,
pub metadata_dropped: Vec<String>,
}
/// The request as a subscriber may see it. The fields are private so that
/// [`RequestFacts::new`], which redacts, is the only way to build one.
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct RequestFacts {
model: String,
custom_llm_provider: String,
optional_params: Value,
url: String,
headers: Vec<(String, String)>,
body: Value,
}
impl RequestFacts {
pub fn new(wire: &WireRequest, context: &RequestContext) -> Self {
Self {
model: context.model.clone(),
custom_llm_provider: context.custom_llm_provider.clone(),
optional_params: redact::params(&context.optional_params, &context.secret_fields),
url: wire.url.clone(),
headers: redact::headers(&wire.headers),
body: wire.body.clone(),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
pub struct TimingFacts {
pub start_time: f64,
pub end_time: f64,
pub duration: f64,
}
impl From<Timing> for TimingFacts {
fn from(timing: Timing) -> Self {
Self {
start_time: timing.start_time,
end_time: timing.end_time,
duration: timing.end_time - timing.start_time,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Origin {
Call,
Host,
}
impl From<FailureOrigin> for Origin {
fn from(origin: FailureOrigin) -> Self {
match origin {
FailureOrigin::Call => Self::Call,
FailureOrigin::Host => Self::Host,
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct ErrorFacts {
pub class: String,
pub message: String,
pub status_code: Option<u16>,
}
impl Event {
pub fn kind(&self) -> EventKind {
match self {
Self::CallStarted(_) => EventKind::CallStarted,
Self::RequestSending(_) => EventKind::RequestSending,
Self::ResponseReceived { .. } => EventKind::ResponseReceived,
Self::CallSucceeded { .. } => EventKind::CallSucceeded,
Self::CallFailed { .. } => EventKind::CallFailed,
}
}
}
pub struct Sequencer {
call_id: String,
call_type: &'static str,
next: u32,
}
impl Sequencer {
pub fn new(call_id: String, call_type: &'static str) -> Self {
Self {
call_id,
call_type,
next: 0,
}
}
pub fn envelope(&mut self, event: Event) -> Envelope {
let envelope = Envelope {
schema: SCHEMA_V1,
call_id: self.call_id.clone(),
call_type: self.call_type,
seq: self.next,
event,
};
self.next += 1;
envelope
}
}
#[cfg(test)]
mod tests {
use litellm_auth::SecretValue;
use proptest::prelude::*;
use serde_json::json;
use strum::VariantArray;
use super::*;
fn event(kind: EventKind) -> Event {
match kind {
EventKind::CallStarted => Event::CallStarted(CallFacts {
start_time: 10.0,
asynchronous: true,
metadata: Map::from_iter([("team".to_string(), json!("core"))]),
metadata_dropped: vec!["broken".to_string()],
}),
EventKind::RequestSending => Event::RequestSending(RequestFacts {
model: "model".to_string(),
custom_llm_provider: "provider".to_string(),
optional_params: json!({"temperature": 0.2}),
url: "https://provider.example/v1".to_string(),
headers: vec![("Authorization".to_string(), redact::REDACTED.to_string())],
body: json!({"input": "hello"}),
}),
EventKind::ResponseReceived => Event::ResponseReceived {
body: "raw response".to_string(),
},
EventKind::CallSucceeded => Event::CallSucceeded {
timing: TimingFacts {
start_time: 10.0,
end_time: 12.5,
duration: 2.5,
},
streamed: false,
response: json!({"id": "response"}),
response_error: None,
},
EventKind::CallFailed => Event::CallFailed {
timing: TimingFacts {
start_time: 10.0,
end_time: 11.0,
duration: 1.0,
},
streamed: true,
origin: Origin::Call,
error: ErrorFacts {
class: "builtins.ValueError".to_string(),
message: "bad request".to_string(),
status_code: Some(400),
},
},
}
}
fn golden(kind: EventKind, seq: u32) -> Envelope {
Envelope {
schema: SCHEMA_V1,
call_id: "call-123".to_string(),
call_type: "ocr",
seq,
event: event(kind),
}
}
/// Exhaustive on purpose: a new event kind does not compile without a golden file,
/// and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are pinned to the golden files.
fn golden_json(kind: EventKind) -> &'static str {
match kind {
EventKind::CallStarted => include_str!("../golden/v1/call.started.json"),
EventKind::RequestSending => include_str!("../golden/v1/request.sending.json"),
EventKind::ResponseReceived => include_str!("../golden/v1/response.received.json"),
EventKind::CallSucceeded => include_str!("../golden/v1/call.succeeded.json"),
EventKind::CallFailed => include_str!("../golden/v1/call.failed.json"),
}
}
#[test]
fn serialized_v1_envelopes_match_golden_contract() {
for (seq, &kind) in EventKind::VARIANTS.iter().enumerate() {
assert_eq!(
serde_json::to_value(golden(kind, seq as u32)).unwrap(),
serde_json::from_str::<Value>(golden_json(kind)).unwrap()
);
}
}
#[test]
fn event_tags_equal_subscription_names() {
for &kind in EventKind::VARIANTS {
let value = serde_json::to_value(event(kind)).unwrap();
let expected: &'static str = kind.into();
assert_eq!(value["type"], expected);
}
}
#[test]
fn request_facts_never_serialize_credentials() {
let sentinel = "sentinel-secret-78e4";
let wire = WireRequest {
url: "https://provider.example".to_string(),
headers: vec![("Authorization".to_string(), sentinel.to_string())],
body: json!({"input": "safe"}),
};
let context = RequestContext {
model: "model".to_string(),
custom_llm_provider: "provider".to_string(),
optional_params: json!({"api_key": sentinel}),
secret_fields: vec!["api_key".to_string()],
api_key: Some(SecretValue::new(sentinel.to_string())),
};
assert!(
!serde_json::to_string(&RequestFacts::new(&wire, &context))
.unwrap()
.contains(sentinel)
);
}
proptest! {
#[test]
fn sequence_numbers_increase_from_zero(length in 0usize..100) {
let mut sequencer = Sequencer::new("call".to_string(), "ocr");
let sequence: Vec<u32> = (0..length)
.map(|_| sequencer.envelope(event(EventKind::ResponseReceived)).seq)
.collect();
prop_assert_eq!(sequence, (0..length as u32).collect::<Vec<_>>());
}
}
}

View file

@ -0,0 +1,15 @@
//! The runtime-neutral v1 callback contract: envelopes and their sequencing, credential
//! redaction, handler selection and validated wire patches. No PyO3 and no registry; a
//! language host marshals these values to its own subscribers.
mod envelope;
mod patch;
mod policy;
mod redact;
pub use envelope::{
CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, Sequencer,
TimingFacts,
};
pub use patch::{HeaderPatch, PatchError, WirePatch, apply};
pub use policy::{ExecutionMode, HandlerSelection, select_handler};

View file

@ -0,0 +1,209 @@
//! The v1 patch: the only way an interceptor changes a request. A [`WirePatch`] removes
//! and sets headers and may replace the body; it cannot name the URL, and [`apply`]
//! rejects any credential header.
use litellm_host::event::WireRequest;
use serde::Deserialize;
use serde_json::Value;
use thiserror::Error;
use crate::redact::is_credential_header;
#[derive(Debug, Default, PartialEq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WirePatch {
#[serde(default)]
pub headers: HeaderPatch,
#[serde(default)]
pub body: Option<Value>,
}
#[derive(Debug, Default, PartialEq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HeaderPatch {
#[serde(default)]
pub remove: Vec<String>,
#[serde(default)]
pub set: Vec<(String, String)>,
}
#[derive(Debug, PartialEq, Error)]
pub enum PatchError {
#[error("protected credential header: {0}")]
ProtectedHeader(String),
#[error("invalid header name: {0}")]
InvalidHeaderName(String),
#[error("invalid header value for: {0}")]
InvalidHeaderValue(String),
}
fn valid_header_name(name: &str) -> bool {
!name.is_empty()
&& name.bytes().all(|byte| {
byte.is_ascii_alphanumeric()
|| matches!(
byte,
b'!' | b'#'
| b'$'
| b'%'
| b'&'
| b'\''
| b'*'
| b'+'
| b'-'
| b'.'
| b'^'
| b'_'
| b'`'
| b'|'
| b'~'
)
})
}
fn validate_name(name: &str) -> Result<(), PatchError> {
if is_credential_header(name) {
return Err(PatchError::ProtectedHeader(name.to_string()));
}
if !valid_header_name(name) {
return Err(PatchError::InvalidHeaderName(name.to_string()));
}
Ok(())
}
pub fn apply(wire: WireRequest, patch: WirePatch) -> Result<WireRequest, PatchError> {
for name in &patch.headers.remove {
validate_name(name)?;
}
for (name, value) in &patch.headers.set {
validate_name(name)?;
if value
.bytes()
.any(|byte| matches!(byte, b'\r' | b'\n' | b'\0'))
{
return Err(PatchError::InvalidHeaderValue(name.clone()));
}
}
let HeaderPatch { remove, set } = patch.headers;
let retained: Vec<(String, String)> = wire
.headers
.into_iter()
.filter(|(name, _)| {
!remove
.iter()
.chain(set.iter().map(|(name, _)| name))
.any(|changed| name.eq_ignore_ascii_case(changed))
})
.collect();
let headers = retained.into_iter().chain(set).collect();
Ok(WireRequest {
url: wire.url,
headers,
body: patch.body.unwrap_or(wire.body),
})
}
#[cfg(test)]
mod tests {
use proptest::prelude::*;
use rstest::rstest;
use serde_json::json;
use super::*;
fn wire() -> WireRequest {
WireRequest {
url: "https://provider.example/v1".to_string(),
headers: vec![
("Authorization".to_string(), "secret".to_string()),
("X-Trace".to_string(), "one".to_string()),
("x-trace".to_string(), "two".to_string()),
("Keep".to_string(), "yes".to_string()),
],
body: json!({"old": true}),
}
}
#[test]
fn default_patch_is_identity() {
let original = wire();
assert_eq!(apply(original.clone(), WirePatch::default()), Ok(original));
}
#[test]
fn remove_precedes_set_and_names_match_case_insensitively() {
let patch = WirePatch {
headers: HeaderPatch {
remove: vec!["keep".to_string()],
set: vec![("X-TRACE".to_string(), "three".to_string())],
},
body: Some(json!({"new": true})),
};
let result = apply(wire(), patch).unwrap();
assert_eq!(
result.headers,
vec![
("Authorization".to_string(), "secret".to_string()),
("X-TRACE".to_string(), "three".to_string()),
]
);
assert_eq!(result.body, json!({"new": true}));
assert_eq!(result.url, "https://provider.example/v1");
}
#[rstest]
#[case(json!({"url": "https://attacker.example"}))]
#[case(json!({"headers": {"append": []}}))]
fn unknown_patch_fields_are_rejected(#[case] value: Value) {
assert!(serde_json::from_value::<WirePatch>(value).is_err());
}
#[rstest]
#[case("Authorization", "value", PatchError::ProtectedHeader("Authorization".to_string()))]
#[case("bad name", "value", PatchError::InvalidHeaderName("bad name".to_string()))]
#[case("X-Test", "bad\r\nvalue", PatchError::InvalidHeaderValue("X-Test".to_string()))]
fn invalid_set_is_rejected(
#[case] name: &str,
#[case] value: &str,
#[case] expected: PatchError,
) {
let patch = WirePatch {
headers: HeaderPatch {
remove: vec![],
set: vec![(name.to_string(), value.to_string())],
},
body: None,
};
assert_eq!(apply(wire(), patch), Err(expected));
}
#[test]
fn credential_headers_cannot_be_removed() {
let patch = WirePatch {
headers: HeaderPatch {
remove: vec!["AUTHORIZATION".to_string()],
set: vec![],
},
body: None,
};
assert_eq!(
apply(wire(), patch),
Err(PatchError::ProtectedHeader("AUTHORIZATION".to_string()))
);
}
proptest! {
#[test]
fn successful_patches_preserve_url(value in "[A-Za-z0-9]{0,20}") {
let original = wire();
let patch = WirePatch {
headers: HeaderPatch { remove: vec![], set: vec![("X-Safe".to_string(), value)] },
body: None,
};
let result = apply(original.clone(), patch).unwrap();
prop_assert_eq!(result.url, original.url);
prop_assert_eq!(result.headers[0].clone(), original.headers[0].clone());
}
}
}

View file

@ -0,0 +1,47 @@
//! Which of a subscriber's handlers a call uses: an async call prefers the async handler
//! and falls back to the sync one; a sync call never selects an async handler.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ExecutionMode {
Sync,
Async,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum HandlerSelection {
Sync,
Async,
Skip,
}
pub fn select_handler(mode: ExecutionMode, has_sync: bool, has_async: bool) -> HandlerSelection {
match (mode, has_sync, has_async) {
(ExecutionMode::Async, _, true) => HandlerSelection::Async,
(_, true, _) => HandlerSelection::Sync,
_ => HandlerSelection::Skip,
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case(ExecutionMode::Async, true, true, HandlerSelection::Async)]
#[case(ExecutionMode::Async, false, true, HandlerSelection::Async)]
#[case(ExecutionMode::Async, true, false, HandlerSelection::Sync)]
#[case(ExecutionMode::Sync, true, true, HandlerSelection::Sync)]
#[case(ExecutionMode::Sync, true, false, HandlerSelection::Sync)]
#[case(ExecutionMode::Sync, false, true, HandlerSelection::Skip)]
#[case(ExecutionMode::Sync, false, false, HandlerSelection::Skip)]
fn handler_selection_never_selects_async_for_sync_calls(
#[case] mode: ExecutionMode,
#[case] has_sync: bool,
#[case] has_async: bool,
#[case] expected: HandlerSelection,
) {
assert_eq!(select_handler(mode, has_sync, has_async), expected);
}
}

View file

@ -0,0 +1,104 @@
//! The contract's credential redaction: credential headers by name, optional params by
//! the route's secret fields. Redaction replaces values and never changes shape.
use serde_json::{Map, Value};
pub const REDACTED: &str = "[REDACTED]";
pub const CREDENTIAL_HEADERS: &[&str] = &[
"authorization",
"proxy-authorization",
"x-api-key",
"api-key",
"x-goog-api-key",
"x-amz-security-token",
"cookie",
];
pub fn is_credential_header(name: &str) -> bool {
CREDENTIAL_HEADERS
.iter()
.any(|credential| name.eq_ignore_ascii_case(credential))
}
pub fn headers(values: &[(String, String)]) -> Vec<(String, String)> {
values
.iter()
.map(|(name, value)| {
let redacted = if is_credential_header(name) {
REDACTED.to_string()
} else {
value.clone()
};
(name.clone(), redacted)
})
.collect()
}
pub fn params(value: &Value, secret_fields: &[String]) -> Value {
let Value::Object(object) = value else {
return value.clone();
};
Value::Object(
object
.iter()
.map(|(name, value)| {
let redacted = if secret_fields.iter().any(|secret| secret == name) {
Value::String(REDACTED.to_string())
} else {
value.clone()
};
(name.clone(), redacted)
})
.collect::<Map<_, _>>(),
)
}
#[cfg(test)]
mod tests {
use proptest::prelude::*;
use serde_json::json;
use super::*;
#[test]
fn credential_header_matching_is_ascii_case_insensitive() {
assert!(is_credential_header("AuThOrIzAtIoN"));
assert!(!is_credential_header("x-request-id"));
}
#[test]
fn redaction_replaces_values_without_changing_shape() {
let input = vec![
("X-Request-ID".to_string(), "abc".to_string()),
("Authorization".to_string(), "secret".to_string()),
("authorization".to_string(), "other".to_string()),
];
assert_eq!(
headers(&input),
vec![
("X-Request-ID".to_string(), "abc".to_string()),
("Authorization".to_string(), REDACTED.to_string()),
("authorization".to_string(), REDACTED.to_string()),
]
);
assert_eq!(
params(
&json!({"api_key": "secret", "model": "m"}),
&["api_key".to_string()]
),
json!({"api_key": REDACTED, "model": "m"})
);
}
proptest! {
#[test]
fn redaction_is_idempotent(
name in "[A-Za-z0-9-]{1,20}",
value in "[A-Za-z0-9]{0,20}"
) {
let input = vec![(name, value)];
prop_assert_eq!(headers(&headers(&input)), headers(&input));
}
}
}

View file

@ -8,6 +8,7 @@ repository.workspace = true
[dependencies]
fancy-regex.workspace = true
litellm-types.workspace = true
percent-encoding.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_path_to_error = "0.1"

View file

@ -1,5 +1,6 @@
use std::marker::PhantomData;
use percent_encoding::{AsciiSet, utf8_percent_encode};
use url::Url;
#[derive(Debug, thiserror::Error)]
@ -64,6 +65,35 @@ impl ApiUrl<Complete> {
self
}
/// Like [`Self::append_query_pairs`], but percent-encodes keys and values with `escape`
/// instead of form encoding, so callers choose which reserved characters stay literal.
/// An existing query is kept; an empty result leaves no dangling `?`.
pub fn append_query_pairs_escaped<'a>(
mut self,
pairs: impl IntoIterator<Item = (&'a str, &'a str)>,
escape: &'static AsciiSet,
) -> Self {
let appended = pairs.into_iter().map(|(key, value)| {
format!(
"{}={}",
utf8_percent_encode(key, escape),
utf8_percent_encode(value, escape)
)
});
let query = self
.url
.query()
.filter(|existing| !existing.is_empty())
.map(str::to_owned)
.into_iter()
.chain(appended)
.collect::<Vec<_>>()
.join("&");
self.url
.set_query(Some(query.as_str()).filter(|query| !query.is_empty()));
self
}
pub fn into_string(self) -> String {
self.url.into()
}
@ -111,4 +141,41 @@ mod tests {
"https://example.test/analyze?model=name+with+spaces"
);
}
const KEEP_COMMAS: &AsciiSet = &percent_encoding::NON_ALPHANUMERIC.remove(b',');
#[rstest::rstest]
#[case::fresh_query("https://example.test", &[("pages", "1,2")], "?pages=1,2")]
#[case::existing_query_kept(
"https://example.test?tenant=a",
&[("pages", "1,2")],
"?tenant=a&pages=1,2"
)]
#[case::bare_question_mark("https://example.test?", &[("pages", "1")], "?pages=1")]
#[case::no_pairs_no_query("https://example.test", &[], "")]
#[case::no_pairs_keeps_query("https://example.test?tenant=a", &[], "?tenant=a")]
#[case::outside_set_escaped(
"https://example.test",
&[("q", "a b&c=d+e%2C#")],
"?q=a%20b%26c%3Dd%2Be%252C%23"
)]
#[case::keys_escaped_too("https://example.test", &[("a&b", "1")], "?a%26b=1")]
#[case::non_ascii_as_utf8("https://example.test", &[("q", "é")], "?q=%C3%A9")]
fn escaped_query_pairs(
#[case] base: &str,
#[case] pairs: &[(&str, &str)],
#[case] expected_query: &str,
) {
let actual = ApiUrl::parse(base)
.and_then(|url| url.complete_path(&["analyze"]))
.map(|url| {
url.append_query_pairs_escaped(pairs.iter().copied(), KEEP_COMMAS)
.into_string()
})
.expect("url builds");
assert_eq!(
actual,
format!("https://example.test/analyze{expected_query}")
);
}
}

View file

@ -1,4 +1,4 @@
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
use litellm_llms::{
aws_textract::ocr::{
analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation,
@ -168,11 +168,8 @@ pub(crate) fn resolve_provider_config(
model: &str,
custom_llm_provider: Option<&str>,
) -> Result<(String, OcrConfigKind), Error> {
let provider =
get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider {
model,
custom_llm_provider: OcrProvider::Mistral.into(),
});
let provider = get_custom_llm_provider(model, custom_llm_provider)
.ok_or_else(|| Error::ProviderNotProvided(model.to_string()))?;
let ocr_provider = provider
.custom_llm_provider
.parse::<OcrProvider>()
@ -482,6 +479,15 @@ mod tests {
);
}
#[rstest]
#[case::bare("mistral-ocr-latest")]
#[case::other_route_model("gpt-4o")]
fn ocr_contract_model_without_a_provider_is_rejected_like_python(#[case] model: &str) {
let error = resolve_provider_config(model, None).unwrap_err();
assert!(matches!(&error, Error::ProviderNotProvided(passed) if passed == model));
assert_eq!(error.http_status_code(), Some(400));
}
#[rstest]
#[case::prefix("not_a_provider/model", None)]
#[case::explicit("model", Some("not_a_provider"))]

View file

@ -96,7 +96,17 @@ pub fn decode_request_input<D: Into<OcrDocumentInput>>(
}
pub fn decode_document(value: Value) -> Result<OcrDocument, Error> {
let kind = value.get("type").and_then(Value::as_str);
let Some(fields) = value.as_object() else {
return Err(Error::DocumentNotObject);
};
let kind = match fields.get("type") {
Some(Value::String(kind)) if matches!(kind.as_str(), "document_url" | "image_url") => {
Some(kind.as_str())
}
Some(Value::String(kind)) => return Err(Error::InvalidDocumentType(kind.clone())),
Some(other) => return Err(Error::InvalidDocumentType(other.to_string())),
None => return Err(Error::InvalidDocumentType("None".into())),
};
if matches!(kind, Some("document_url")) && value.get("document_url").is_none()
|| matches!(kind, Some("image_url")) && value.get("image_url").is_none()
{
@ -122,9 +132,12 @@ mod tests {
}
#[rstest]
#[case::non_object(json!([]), "document")]
#[case::missing_type(json!({"document_url":"https://example.com/a.pdf"}), "document")]
#[case::unsupported_type(json!({"type":"text"}), "type")]
#[case::non_object(json!([]), "document must be a dict")]
#[case::missing_type(
json!({"document_url":"https://example.com/a.pdf"}),
"Invalid document type: None"
)]
#[case::unsupported_type(json!({"type":"text"}), "Invalid document type: text")]
#[case::missing_document_url(json!({"type":"document_url"}), "Document URL")]
#[case::missing_image_url(json!({"type":"image_url"}), "Document URL")]
fn ocr_contract_malformed_document_is_bad_request(
@ -134,7 +147,7 @@ mod tests {
let error = decode_document(document).unwrap_err();
assert!(matches!(
error,
Error::RequestField { .. } | Error::MissingDocumentUrl
Error::DocumentNotObject | Error::InvalidDocumentType(_) | Error::MissingDocumentUrl
));
assert_eq!(error.http_status_code(), Some(400));
assert!(error.to_string().contains(field), "{error}");

View file

@ -5,6 +5,26 @@
- `RouteHost::invoke` receives the keyword view the adapter's `begin` returned, not the caller's dict; a route host that projects from it inherits that adapter's rewrites (for the legacy adapter: setup, deployment hooks, credential inheritance)
- A native failure, including one a host op returns as `InvokeError::Native`, is classified exactly once through the route's `classify`; a Python exception raised inside the call, and a failure in `begin` or `after_success`, is raised as is
- A failing `classify` is raised with the native error's text as its `__context__`, never swallowed
- Terms; the callback crates build on these and never redefine them
- Machine: the native call; it never calls out, it yields a `HostOp` (`Route`, `BeforeSend`, `Emit`, `Open`, `Deliver`) and waits for the `HostResult`
- Host: whatever answers those ops; the Python host is two capabilities the driver composes: `RouteHost` (route ops, argument and response projection) and `PythonLifecycle` (callbacks)
- Host language = callback language: a lifecycle speaks `Py<...>` and `PyErr` directly; another host language gets its own crate of this shape and reuses only `litellm-host` and `litellm-callbacks-v1`
- Driver: runs one call; owns the machine, one route host and exactly one lifecycle
- Lifecycle: one implementation of `PythonLifecycle`, one callback contract's view of a call; chain: several lifecycles behind one, in a fixed order
- Step: one driver-to-lifecycle method call, answered with a `LifecycleStep`; a threading step (`begin`, `before_send`, `after_success`) returns the value the call continues with and defaults to returning what it was given; an observing step (`emit`, `opened`, `delivered`) returns nothing to the call
- Suspension: a step answered with `Await`; the driver awaits it in the caller's task and continues the same step through `resume`
- Terminal event: `Succeeded` or `Failed`, exactly once, last
- Cancellation: a raised `BaseException` that is not an `Exception`; `is_cancellation` is the only definition; never reported, never swallowed, never followed by dispatch
- Keyword view: the kwargs dict as rewritten so far; `begin` threads it and the route projects from the result
- Wire: the `WireRequest` (URL, headers, body) about to be sent; `before_send` threads it
- Surface: the static facts a route declares about itself to a contract (`call_type`, ...)
- Python contract: a crate's `python.rs` enum, its `python_contract.json` and the one `litellm/rust_bridge/*.py` module behind them, tested from both sides
- `PythonLifecycle` is the whole callback boundary: the driver holds exactly one and never learns how many contracts are behind it
- `LifecycleChain` composes several in a fixed order: `begin`, `before_send` and `after_success` thread their value forward, events reach each in turn, any error ends the step with no further dispatch
- `begin` carries the call's start time and is the first thing a lifecycle hears of a call; there is no separate started event
- A lifecycle whose `begin` was never reached receives nothing of that call; the terminal event goes to the ones `begin` reached
- A suspension resumes the lifecycle that suspended, then continues the same step with the ones after it; a retained failure is the raised exception object itself
- Ordering policy (which contract runs first) belongs to whoever builds the chain, never to this crate
- Use standard PyO3 ownership and conversion APIs
- Prefer `Bound<'py, T>` for attached operations/results, `Py<T>` for retention; binding/unbinding does not copy payloads
- Use `pythonize` for selected Serde data, never a JSON-text round trip; share conversion with `Pythonized<T>`

View file

@ -1,15 +1,22 @@
use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest};
use litellm_host::route::Route;
use pyo3::exceptions::PyRuntimeError;
use pyo3::exceptions::{PyException, PyRuntimeError};
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::PyDict;
/// Whether an error ends the call outright: anything that is not a `PyException`, such
/// as `CancelledError` or `KeyboardInterrupt`. A cancellation is never reported, never
/// swallowed and never followed by further dispatch.
pub fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool {
!error.is_instance_of::<PyException>(py)
}
pub fn missing_state() -> PyErr {
PyRuntimeError::new_err("missing native call state")
}
/// What an adapter step produced: either the value the driver asked for, or a Python
/// What a lifecycle step produced: either the value the driver asked for, or a Python
/// awaitable the driver hands back to the caller's task before asking again.
pub enum LifecycleStep {
Await(Py<PyAny>),
@ -19,12 +26,10 @@ pub enum LifecycleStep {
Done,
}
/// What a lifecycle observes: the driver's start, the machine's own events, and one
/// terminal event carrying the public value the caller receives.
/// What a lifecycle observes: the machine's own events, and one terminal event
/// carrying the public value the caller receives.
#[derive(Clone, Copy)]
pub enum LifecycleEvent<'a> {
Started {
start_time: f64,
},
Machine(&'a MachineEvent),
Succeeded {
timing: Timing,
@ -37,46 +42,73 @@ pub enum LifecycleEvent<'a> {
},
}
/// One consumer of a call's lifecycle on the Python side. The driver calls the steps in
/// order: `begin` before the machine starts, `before_send` and `emit` while it runs,
/// `after_success` and one terminal `emit` after it completes. Whenever a step returns
/// [`LifecycleStep::Await`], the driver awaits it in the caller's task and continues the
/// same step through `resume`.
/// One consumer of a call's lifecycle on the Python side: the callback capability of the
/// Python host, next to [`RouteHost`], the route capability.
///
/// The steps come in two kinds. A threading step receives a value and returns the value
/// the call continues with, so a lifecycle may rewrite it; each defaults to returning
/// what it was given. An observing step receives an event or a notice and returns
/// nothing to the call. Two steps answer an operation the machine raised
/// ([`HostOp`](litellm_host::host::HostOp)); the rest the driver originates around it:
///
/// | step | kind | raised by |
/// |---|---|---|
/// | `begin` | threads the keyword view | driver, before the machine starts |
/// | `before_send` | threads the wire request | machine, `HostOp::BeforeSend` |
/// | `emit(Machine)` | observes | machine, `HostOp::Emit` |
/// | `opened`, `delivered` | observe | driver, answering `HostOp::Open` and `Deliver` |
/// | `after_success` | threads the public response | driver, after the machine completes |
/// | `emit(Succeeded \| Failed)` | observes | driver, exactly once, last |
///
/// Whenever a step returns [`LifecycleStep::Await`], the driver awaits it in the caller's
/// task and continues the same step through `resume`.
///
/// A step that fails with an ordinary exception fails the call with that exception,
/// except on a terminal event, where the adapter is expected to report and swallow its
/// own errors. An exception that is not a `PyException`, such as a cancellation, ends
/// the call without further dispatch.
/// except on a terminal event, where the lifecycle is expected to report and swallow its
/// own errors. A cancellation ([`is_cancellation`]) ends the call without further
/// dispatch.
pub trait PythonLifecycle: Send + Sync {
/// `start_time` is the call's start in epoch seconds, the same value the terminal
/// event's `Timing` carries.
fn begin(
&mut self,
py: Python<'_>,
_py: Python<'_>,
arguments: Py<PyDict>,
started_at: f64,
) -> PyResult<LifecycleStep>;
_start_time: f64,
) -> PyResult<LifecycleStep> {
Ok(LifecycleStep::Arguments(arguments))
}
fn before_send(
&mut self,
py: Python<'_>,
_py: Python<'_>,
wire: Box<WireRequest>,
context: &RequestContext,
) -> PyResult<LifecycleStep>;
_context: &RequestContext,
) -> PyResult<LifecycleStep> {
Ok(LifecycleStep::Wire(wire))
}
fn after_success(
&mut self,
py: Python<'_>,
_py: Python<'_>,
response: Py<PyAny>,
timing: Timing,
) -> PyResult<LifecycleStep>;
_timing: Timing,
) -> PyResult<LifecycleStep> {
Ok(LifecycleStep::Response(response))
}
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep>;
/// The call streams and its stream was handed to the caller. The caller is not
/// inside an await here, so this step and `delivered` cannot suspend.
fn opened(&mut self, py: Python<'_>) -> PyResult<()>;
fn opened(&mut self, _py: Python<'_>) -> PyResult<()> {
Ok(())
}
/// One chunk of an open stream is about to reach the caller.
fn delivered(&mut self, py: Python<'_>, chunk: &Py<PyAny>) -> PyResult<()>;
fn delivered(&mut self, _py: Python<'_>, _chunk: &Py<PyAny>) -> PyResult<()> {
Ok(())
}
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep>;

View file

@ -0,0 +1,560 @@
//! Several lifecycles consuming one call as one [`PythonLifecycle`]. The driver keeps a
//! single consumer; which contracts a call runs under is decided where the chain is
//! built, never by the driver and never by a route.
use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest};
use pyo3::exceptions::PyBaseException;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::PyDict;
use crate::adapter::{LifecycleEvent, LifecycleStep, PythonLifecycle, missing_state};
/// An event kept across a suspension, so the lifecycles after the suspended one still
/// receive it. A failure keeps the raised exception object itself.
enum RetainedEvent {
Machine(MachineEvent),
Succeeded {
timing: Timing,
response: Py<PyAny>,
},
Failed {
timing: Timing,
origin: FailureOrigin,
error: Py<PyBaseException>,
},
}
/// The step a suspended lifecycle was answering, with what the later lifecycles need to
/// answer it too. The value being threaded stays with the suspended lifecycle.
enum Step {
Begin { start_time: f64 },
BeforeSend { context: RequestContext },
AfterSuccess { timing: Timing },
Emit(RetainedEvent),
}
struct Suspended {
index: usize,
step: Step,
}
/// Runs every step through its lifecycles in the order they were given.
///
/// `begin`, `before_send` and `after_success` thread their value: each lifecycle
/// receives what the one before it returned, so a later lifecycle sees an earlier one's
/// rewrites and never the reverse. Events reach each lifecycle in turn. An error from
/// any lifecycle ends the step at once, with no dispatch to the ones after it.
///
/// A lifecycle observes a call only once its `begin` was reached: when an earlier
/// lifecycle fails the call in `begin`, the later ones never learn of the call, and the
/// terminal event goes to those that did.
pub struct LifecycleChain {
lifecycles: Vec<Box<dyn PythonLifecycle>>,
/// How many lifecycles `begin` has reached.
entered: usize,
pending: Option<Suspended>,
}
impl RetainedEvent {
fn retain(py: Python<'_>, event: LifecycleEvent<'_>) -> Self {
match event {
LifecycleEvent::Machine(event) => Self::Machine(event.clone()),
LifecycleEvent::Succeeded { timing, response } => Self::Succeeded {
timing,
response: response.clone_ref(py),
},
LifecycleEvent::Failed {
timing,
origin,
error,
} => Self::Failed {
timing,
origin,
error: error.clone_ref(py).into_value(py),
},
}
}
fn emit(&self, py: Python<'_>, lifecycle: &mut dyn PythonLifecycle) -> PyResult<LifecycleStep> {
match self {
Self::Machine(event) => lifecycle.emit(py, LifecycleEvent::Machine(event)),
Self::Succeeded { timing, response } => lifecycle.emit(
py,
LifecycleEvent::Succeeded {
timing: *timing,
response,
},
),
Self::Failed {
timing,
origin,
error,
} => {
let error = PyErr::from_value(error.bind(py).clone().into_any());
lifecycle.emit(
py,
LifecycleEvent::Failed {
timing: *timing,
origin: *origin,
error: &error,
},
)
}
}
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
match self {
Self::Succeeded { response, .. } => visit.call(response),
Self::Failed { error, .. } => visit.call(error),
Self::Machine(_) => Ok(()),
}
}
}
impl LifecycleChain {
pub fn new(lifecycles: Vec<Box<dyn PythonLifecycle>>) -> Self {
Self {
lifecycles,
entered: 0,
pending: None,
}
}
/// How many lifecycles a step reaches: `begin` tries all of them, every later step
/// reaches the ones `begin` reached.
fn audience(&self, step: &Step) -> usize {
match step {
Step::Begin { .. } => self.lifecycles.len(),
Step::BeforeSend { .. } | Step::AfterSuccess { .. } | Step::Emit(_) => self.entered,
}
}
fn call(
&mut self,
py: Python<'_>,
index: usize,
step: &Step,
carried: LifecycleStep,
) -> PyResult<LifecycleStep> {
let lifecycle = self.lifecycles[index].as_mut();
match (step, carried) {
(Step::Begin { start_time }, LifecycleStep::Arguments(arguments)) => {
self.entered = index + 1;
lifecycle.begin(py, arguments, *start_time)
}
(Step::BeforeSend { context }, LifecycleStep::Wire(wire)) => {
lifecycle.before_send(py, wire, context)
}
(Step::AfterSuccess { timing }, LifecycleStep::Response(response)) => {
lifecycle.after_success(py, response, *timing)
}
(Step::Emit(event), LifecycleStep::Done) => event.emit(py, lifecycle),
_ => Err(missing_state()),
}
}
/// Continues `step` from the lifecycle at `next`, handing each one what the previous
/// produced, until one suspends or the last has answered.
fn proceed(
&mut self,
py: Python<'_>,
mut next: usize,
mut carried: LifecycleStep,
step: Step,
) -> PyResult<LifecycleStep> {
while next < self.audience(&step) {
carried = match self.call(py, next, &step, carried)? {
LifecycleStep::Await(awaitable) => {
self.pending = Some(Suspended { index: next, step });
return Ok(LifecycleStep::Await(awaitable));
}
produced => produced,
};
next += 1;
}
Ok(carried)
}
}
impl PythonLifecycle for LifecycleChain {
fn begin(
&mut self,
py: Python<'_>,
arguments: Py<PyDict>,
start_time: f64,
) -> PyResult<LifecycleStep> {
self.proceed(
py,
0,
LifecycleStep::Arguments(arguments),
Step::Begin { start_time },
)
}
fn before_send(
&mut self,
py: Python<'_>,
wire: Box<WireRequest>,
context: &RequestContext,
) -> PyResult<LifecycleStep> {
self.proceed(
py,
0,
LifecycleStep::Wire(wire),
Step::BeforeSend {
context: context.clone(),
},
)
}
fn after_success(
&mut self,
py: Python<'_>,
response: Py<PyAny>,
timing: Timing,
) -> PyResult<LifecycleStep> {
self.proceed(
py,
0,
LifecycleStep::Response(response),
Step::AfterSuccess { timing },
)
}
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
for index in 0..self.entered {
match self.lifecycles[index].emit(py, event)? {
LifecycleStep::Await(awaitable) => {
self.pending = Some(Suspended {
index,
step: Step::Emit(RetainedEvent::retain(py, event)),
});
return Ok(LifecycleStep::Await(awaitable));
}
LifecycleStep::Done => {}
_ => return Err(missing_state()),
}
}
Ok(LifecycleStep::Done)
}
fn opened(&mut self, py: Python<'_>) -> PyResult<()> {
self.lifecycles[..self.entered]
.iter_mut()
.try_for_each(|lifecycle| lifecycle.opened(py))
}
fn delivered(&mut self, py: Python<'_>, chunk: &Py<PyAny>) -> PyResult<()> {
self.lifecycles[..self.entered]
.iter_mut()
.try_for_each(|lifecycle| lifecycle.delivered(py, chunk))
}
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
let Suspended { index, step } = self.pending.take().ok_or_else(missing_state)?;
match self.lifecycles[index].resume(py, result)? {
LifecycleStep::Await(awaitable) => {
self.pending = Some(Suspended { index, step });
Ok(LifecycleStep::Await(awaitable))
}
produced => self.proceed(py, index + 1, produced, step),
}
}
fn close(&mut self, py: Python<'_>) {
self.pending = None;
for lifecycle in &mut self.lifecycles {
lifecycle.close(py);
}
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
for lifecycle in &self.lifecycles {
lifecycle.traverse(visit)?;
}
match &self.pending {
Some(Suspended {
step: Step::Emit(event),
..
}) => event.traverse(visit),
_ => Ok(()),
}
}
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use pyo3::exceptions::PyValueError;
use super::*;
type Log = Arc<Mutex<Vec<String>>>;
#[derive(Clone, Copy, PartialEq)]
enum Script {
Plain,
FailBegin,
SuspendBegin,
SuspendTerminal,
}
struct Recorder {
name: &'static str,
script: Script,
log: Log,
suspended: Option<LifecycleStep>,
}
impl Recorder {
fn record(&self, entry: &str) {
self.log
.lock()
.unwrap()
.push(format!("{}.{entry}", self.name));
}
fn suspend(&mut self, py: Python<'_>, then: LifecycleStep) -> PyResult<LifecycleStep> {
self.suspended = Some(then);
Ok(LifecycleStep::Await(py.None()))
}
}
impl PythonLifecycle for Recorder {
fn begin(
&mut self,
py: Python<'_>,
arguments: Py<PyDict>,
_: f64,
) -> PyResult<LifecycleStep> {
let seen: Vec<String> = arguments
.bind(py)
.keys()
.iter()
.map(|key| key.extract().unwrap())
.collect();
self.record(&format!("begin{seen:?}"));
arguments.bind(py).set_item(self.name, true)?;
match self.script {
Script::FailBegin => Err(PyValueError::new_err("begin failed")),
Script::SuspendBegin => self.suspend(py, LifecycleStep::Arguments(arguments)),
_ => Ok(LifecycleStep::Arguments(arguments)),
}
}
fn before_send(
&mut self,
_: Python<'_>,
mut wire: Box<WireRequest>,
_: &RequestContext,
) -> PyResult<LifecycleStep> {
self.record(&format!("before_send:{}", wire.url));
wire.url.push_str(self.name);
Ok(LifecycleStep::Wire(wire))
}
fn after_success(
&mut self,
_: Python<'_>,
response: Py<PyAny>,
_: Timing,
) -> PyResult<LifecycleStep> {
self.record("after_success");
Ok(LifecycleStep::Response(response))
}
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
let terminal = match event {
LifecycleEvent::Machine(_) => {
self.record("machine");
false
}
LifecycleEvent::Succeeded { response, .. } => {
self.record(&format!("succeeded:{}", response.bind(py)));
true
}
LifecycleEvent::Failed { error, .. } => {
self.record(&format!("failed:{}", error.value(py)));
true
}
};
if terminal && self.script == Script::SuspendTerminal {
return self.suspend(py, LifecycleStep::Done);
}
Ok(LifecycleStep::Done)
}
fn opened(&mut self, _: Python<'_>) -> PyResult<()> {
self.record("opened");
Ok(())
}
fn delivered(&mut self, _: Python<'_>, _: &Py<PyAny>) -> PyResult<()> {
self.record("delivered");
Ok(())
}
fn resume(&mut self, _: Python<'_>, _: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
self.record("resume");
self.suspended.take().ok_or_else(missing_state)
}
fn close(&mut self, _: Python<'_>) {
self.record("close");
}
fn traverse(&self, _: &PyVisit<'_>) -> Result<(), PyTraverseError> {
Ok(())
}
}
fn chain(scripts: [(&'static str, Script); 2]) -> (LifecycleChain, Log) {
let log = Log::default();
let lifecycles = scripts
.into_iter()
.map(|(name, script)| {
Box::new(Recorder {
name,
script,
log: Arc::clone(&log),
suspended: None,
}) as Box<dyn PythonLifecycle>
})
.collect();
(LifecycleChain::new(lifecycles), log)
}
fn entries(log: &Log) -> Vec<String> {
std::mem::take(&mut *log.lock().unwrap())
}
fn timing() -> Timing {
Timing {
start_time: 1.0,
end_time: 2.0,
}
}
fn context() -> RequestContext {
RequestContext {
model: "model".to_string(),
custom_llm_provider: "provider".to_string(),
optional_params: serde_json::Value::Null,
secret_fields: Vec::new(),
api_key: None,
}
}
#[test]
fn a_later_lifecycle_receives_what_an_earlier_one_returned() {
crate::initialize_python();
Python::attach(|py| {
let (mut chain, log) = chain([("first", Script::Plain), ("second", Script::Plain)]);
let arguments = PyDict::new(py).unbind();
let step = chain.begin(py, arguments.clone_ref(py), 1.0).unwrap();
assert!(matches!(step, LifecycleStep::Arguments(returned) if returned.is(&arguments)));
let wire = Box::new(WireRequest {
url: "url:".to_string(),
headers: Vec::new(),
body: serde_json::Value::Null,
});
let step = chain.before_send(py, wire, &context()).unwrap();
assert!(matches!(step, LifecycleStep::Wire(wire) if wire.url == "url:firstsecond"));
assert_eq!(
entries(&log),
[
"first.begin[]",
"second.begin[\"first\"]",
"first.before_send:url:",
"second.before_send:url:first",
]
);
});
}
#[test]
fn a_lifecycle_begin_never_reached_observes_nothing_of_the_call() {
crate::initialize_python();
Python::attach(|py| {
let (mut chain, log) = chain([("first", Script::FailBegin), ("second", Script::Plain)]);
let error = chain
.begin(py, PyDict::new(py).unbind(), 1.0)
.err()
.expect("begin fails");
let failed = LifecycleEvent::Failed {
timing: timing(),
origin: FailureOrigin::Host,
error: &error,
};
assert!(matches!(
chain.emit(py, failed).unwrap(),
LifecycleStep::Done
));
chain.close(py);
assert_eq!(
entries(&log),
[
"first.begin[]",
"first.failed:begin failed",
"first.close",
"second.close",
]
);
});
}
#[test]
fn a_suspended_step_resumes_its_lifecycle_and_then_reaches_the_later_ones() {
crate::initialize_python();
Python::attach(|py| {
let (mut chain, log) = chain([
("first", Script::SuspendBegin),
("second", Script::SuspendTerminal),
]);
let step = chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap();
assert!(matches!(step, LifecycleStep::Await(_)));
assert_eq!(entries(&log), ["first.begin[]"]);
let step = chain.resume(py, Ok(py.None())).unwrap();
assert!(matches!(step, LifecycleStep::Arguments(_)));
assert_eq!(entries(&log), ["first.resume", "second.begin[\"first\"]"]);
});
}
#[test]
fn an_event_kept_across_a_suspension_reaches_the_later_lifecycles_unchanged() {
crate::initialize_python();
Python::attach(|py| {
let (mut chain, log) = chain([
("first", Script::SuspendTerminal),
("second", Script::Plain),
]);
chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap();
entries(&log);
let raised = PyValueError::new_err("provider failed");
let failed = LifecycleEvent::Failed {
timing: timing(),
origin: FailureOrigin::Call,
error: &raised,
};
assert!(matches!(
chain.emit(py, failed).unwrap(),
LifecycleStep::Await(_)
));
assert!(matches!(
chain.resume(py, Ok(py.None())).unwrap(),
LifecycleStep::Done
));
assert_eq!(
entries(&log),
[
"first.failed:provider failed",
"first.resume",
"second.failed:provider failed",
]
);
});
}
}

View file

@ -46,7 +46,6 @@ enum Stage {
#[derive(Clone, Copy)]
enum Expect {
Started,
Arguments,
Wire,
Emitted,
@ -159,13 +158,7 @@ where
match (self.pending.take(), result) {
(None, None) => {
self.started_at = epoch_seconds();
let started = LifecycleEvent::Started {
start_time: self.started_at,
};
match self.adapter.emit(py, started) {
Ok(step) => self.on_adapter(py, step, Expect::Started),
Err(error) => self.adapter_failed(py, error),
}
self.begin(py)
}
(Some(Pending::Native), Some(Ok(_))) => {
let result = self.take_native_result()?;
@ -201,7 +194,6 @@ where
self.pending = Some(Pending::Adapter(expect));
Ok(ExecutionStep::Await(awaitable))
}
(Expect::Started, LifecycleStep::Done) => self.begin(py),
(Expect::Arguments, LifecycleStep::Arguments(arguments)) => {
self.arguments = Some(arguments);
self.stage = Stage::Call;
@ -789,7 +781,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
self.log.push(match event {
LifecycleEvent::Started { .. } => "started".into(),
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
format!("response:{}", raw.body)
}
@ -921,7 +912,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
assert_eq!(
log,
[
"started",
"begin",
"route:project",
"before_send",
@ -967,7 +957,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
assert_eq!(
log,
[
"started",
"begin",
"route:project",
"classify:provider exploded",
@ -1001,7 +990,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
assert_eq!(
log,
[
"started",
"begin",
"route:project",
"classify:op rejected",
@ -1033,7 +1021,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
assert_eq!(
log,
[
"started",
"begin",
"route:project",
"failed:Call:op failed",
@ -1071,7 +1058,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
assert_eq!(
log,
[
"started",
"begin",
"route:project",
"classify:provider exploded",
@ -1102,7 +1088,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
assert_eq!(
log,
[
"started",
"begin",
"failed:Host:begin failed",
"adapter.close",
@ -1225,10 +1210,7 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
)
.unwrap_err();
assert!(!error.is_instance_of::<pyo3::exceptions::PyException>(py));
assert_eq!(
log.entries(),
["started", "begin", "route", "adapter.close"]
);
assert_eq!(log.entries(), ["begin", "route", "adapter.close"]);
});
}

View file

@ -6,6 +6,7 @@
mod adapter;
mod argument;
mod callable;
mod chain;
mod driver;
mod execution;
mod fork_gate;
@ -14,10 +15,12 @@ mod handle;
mod marshal;
pub use adapter::{
InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, missing_state,
InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, is_cancellation,
missing_state,
};
pub use argument::lookup;
pub use callable::wrap_failure;
pub use chain::LifecycleChain;
pub use driver::run_call;
pub use execution::{
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value,

View file

@ -22,6 +22,7 @@ base64.workspace = true
bytes.workspace = true
data-url = "0.3.2"
futures-util.workspace = true
percent-encoding.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }

View file

@ -8,6 +8,7 @@ use litellm_core_utils::{
serde_compat::{FiniteF64, LaxI64},
url_utils::ApiUrl,
};
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC};
use reqwest::Url;
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::{Map, Value};
@ -549,6 +550,17 @@ async fn poll_operation(
}
}
/// Python parity: the Python provider builds this query with
/// `urllib.parse.quote(value, safe=",")`, which escapes everything except RFC 3986 unreserved
/// characters and `,`. The literal commas are also what the service documents for the
/// `pages` and `features` lists; form encoding would send `%2C`.
const PYTHON_QUOTE_KEEP_COMMAS: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'.')
.remove(b'_')
.remove(b'~')
.remove(b',');
impl AzureDocumentIntelligenceOcrConfig {
fn build_ocr_url(
&self,
@ -561,16 +573,15 @@ impl AzureDocumentIntelligenceOcrConfig {
ApiUrl::parse(endpoint)
.and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model]))
.map(|url| {
url.append_query_pairs(
[("api-version", api_version)]
.into_iter()
.chain(params.pages.iter().map(|pages| ("pages", pages.as_str())))
.chain(
params
.features
.iter()
.map(|features| ("features", features.as_str())),
),
url.append_query_pairs_escaped(
[
("api-version", Some(api_version)),
("pages", params.pages.as_deref()),
("features", params.features.as_deref()),
]
.into_iter()
.filter_map(|(key, value)| Some((key, value?))),
PYTHON_QUOTE_KEEP_COMMAS,
)
.into_string()
})
@ -649,6 +660,54 @@ mod tests {
use super::*;
#[test]
fn query_escape_set_matches_python_quote() {
// Expected output from Python:
// quote("".join(map(chr, range(0x20, 0x7f))) + "é", safe=",")
let input = (0x20u8..0x7f)
.map(char::from)
.chain(['é'])
.collect::<String>();
assert_eq!(
percent_encoding::utf8_percent_encode(&input, PYTHON_QUOTE_KEEP_COMMAS).to_string(),
"%20%21%22%23%24%25%26%27%28%29%2A%2B,-.%2F0123456789%3A%3B%3C%3D%3E%3F%40\
ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%C3%A9"
);
}
#[rstest]
#[case::no_lists(None, None, "")]
#[case::pages_only(Some("1,3-4"), None, "&pages=1,3-4")]
#[case::features_only(
None,
Some("keyValuePairs,languages"),
"&features=keyValuePairs,languages"
)]
#[case::both_lists(
Some("1,3-4"),
Some("keyValuePairs,languages"),
"&pages=1,3-4&features=keyValuePairs,languages"
)]
fn ocr_url_query_matches_python(
#[case] pages: Option<&str>,
#[case] features: Option<&str>,
#[case] expected_lists: &str,
) {
let params = DocumentIntelligenceParams {
pages: pages.map(Into::into),
features: features.map(Into::into),
};
assert_eq!(
AzureDocumentIntelligenceOcrConfig
.build_ocr_url("https://di.test", "prebuilt-layout", &params, "2024-11-30")
.unwrap(),
format!(
"https://di.test/documentintelligence/documentModels/prebuilt-layout:analyze\
?api-version=2024-11-30{expected_lists}"
)
);
}
fn map(value: Value) -> Result<DocumentIntelligenceParams, Error> {
let arguments = serde_json::from_value(value).unwrap();
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model")

View file

@ -198,7 +198,7 @@ mod tests {
client.document_fetcher(),
OcrDocument::ImageUrl {
image_url: format!("http://{address}/image"),
extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]),
extra_fields: Map::from_iter([("detail".into(), "high".into())]),
},
&OcrConnection::default(),
)
@ -210,7 +210,7 @@ mod tests {
converted,
OcrDocument::ImageUrl {
image_url: "data:image/png;base64,YWJj".into(),
extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]),
extra_fields: Map::from_iter([("detail".into(), "high".into())]),
}
);
assert!(!request.to_ascii_lowercase().contains("authorization"));

View file

@ -30,6 +30,10 @@ pub enum Error {
MissingField(&'static str),
#[error("Document URL is required")]
MissingDocumentUrl,
#[error("document must be a dict with 'type' and URL/file field")]
DocumentNotObject,
#[error("Invalid document type: {0}. Must be 'document_url', 'image_url', or 'file'")]
InvalidDocumentType(String),
#[error("invalid OCR document data URI")]
InvalidDataUri,
#[error(
@ -76,6 +80,10 @@ pub enum Error {
Unsupported(&'static str),
#[error("invalid provider: {0}")]
InvalidProvider(String),
#[error(
"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={0}"
)]
ProviderNotProvided(String),
#[error("invalid model: {provider} has no model {model:?} - use one of: {}", supported.join(", "))]
InvalidModel {
provider: &'static str,
@ -149,6 +157,8 @@ impl Error {
| Self::RequestField { .. }
| Self::MissingField(_)
| Self::MissingDocumentUrl
| Self::DocumentNotObject
| Self::InvalidDocumentType(_)
| Self::InvalidDataUri
| Self::ReductoSource
| Self::InlineDocumentTooLarge
@ -161,6 +171,7 @@ impl Error {
| Self::DotModel
| Self::InvalidRequest(_)
| Self::InvalidProvider(_)
| Self::ProviderNotProvided(_)
| Self::InvalidModel { .. }
| Self::Params(_)
| Self::Headers(_)

View file

@ -32,16 +32,31 @@ pub enum OcrDocument {
DocumentUrl {
document_url: String,
#[serde(flatten)]
extra_fields: BTreeMap<String, Option<String>>,
extra_fields: BTreeMap<String, Value>,
},
#[serde(rename = "image_url")]
ImageUrl {
#[serde(deserialize_with = "image_url_source")]
image_url: String,
#[serde(flatten)]
extra_fields: BTreeMap<String, Option<String>>,
extra_fields: BTreeMap<String, Value>,
},
}
/// Mistral accepts `image_url` as a string or as `{"url": ...}`; providers only consume the
/// source, so the object form is read down to its URL.
fn image_url_source<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<String, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum ImageUrl {
Source(String),
Object { url: String },
}
Ok(match ImageUrl::deserialize(deserializer)? {
ImageUrl::Source(url) | ImageUrl::Object { url } => url,
})
}
impl OcrDocument {
pub fn source(&self) -> &str {
match self {
@ -346,8 +361,13 @@ pub struct DecodedOcrResponse<T> {
pub fn decode_request_value<T: DeserializeOwned>(value: Value, prefix: &str) -> Result<T, Error> {
serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| {
let path = error.path().to_string();
Error::RequestField {
path: format!("{prefix}.{}", error.path()),
path: if path == "." {
prefix.to_string()
} else {
format!("{prefix}.{path}")
},
}
})
}
@ -639,10 +659,8 @@ mod tests {
assert!(serde_json::from_value::<LiteLLMOcrResponse>(Value::Object(payload)).is_err());
}
assert!(
serde_json::from_value::<OcrDocument>(json!({
"type":"image_url", "image_url":"https://example.com/image", "detail":42
}))
.is_err()
serde_json::from_value::<OcrDocument>(json!({"type":"image_url", "image_url":42}))
.is_err()
);
}
@ -694,6 +712,20 @@ mod tests {
);
}
#[test]
fn documents_accept_what_python_forwards_to_the_provider() {
let document: OcrDocument = serde_json::from_value(json!({
"type":"image_url", "image_url":{"url":"https://example.com/image", "detail":"high"}
}))
.unwrap();
assert_eq!(document.source(), "https://example.com/image");
let document = json!({
"type":"document_url", "document_url":"https://example.com/a.pdf", "n":3
});
let decoded: OcrDocument = serde_json::from_value(document.clone()).unwrap();
assert_eq!(serde_json::to_value(decoded).unwrap(), document);
}
#[test]
fn response_serialization_flattens_extra_fields_and_omits_absent_native_response() {
let response = LiteLLMOcrResponse {

View file

@ -1,7 +1,11 @@
- Target invariants, not completion claims; these supersede the crate guidance below where they conflict
- Keep this crate the product-specific PyO3 consumer of `litellm-host-python`
- Own registration, input projection, the route host and the caller callables it answers operations with (file readers, token providers), public response/error construction and the per-call composition of machine, route host and callback contract
- Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `run_legacy_call`; the bridge hands the public call over and keeps no copy
- Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `LegacyPythonLifecycle`; the bridge hands the public call over and keeps no copy
- Routes never select a callback contract: `callbacks::run_python_call` is the only way a route starts a call, and `src/callbacks.rs` is the only place that names a contract crate
- Every call runs the legacy `Logging` lifecycle, then the v1 lifecycle when anything is subscribed; with no subscriber the call is exactly the legacy-only call
- A route describes itself once as a `CallSurface`; fields only the legacy contract reads are marked so and go when it goes
- Retiring legacy for a route, or for all of them, is a change to that one composition, never to a route, the driver or an adapter
- Value-oriented execution, sync waiting, nested-runtime checks, signal polling and panic containment live in `litellm-host-python`; native async work uses `pyo3-async-runtimes`, Serde output uses `Pythonized<T>`
- Core owns typed native state, the route machine, provider preparation/I/O and normalization; the host driver owns terminal events; the legacy adapter in `litellm-callbacks-legacy-python` owns `Logging` dispatch policy
- Python, Rust SDK and gateway use one lifecycle-bearing core route entrypoint; provider helpers stay private, never bridge-accessible transport drivers

View file

@ -19,10 +19,12 @@ panic-test = []
bytes.workspace = true
litellm-auth.workspace = true
litellm-callbacks-legacy-python.workspace = true
litellm-callbacks-v1-python.workspace = true
litellm-core.workspace = true
litellm-core-utils.workspace = true
litellm-auth-gcp.workspace = true
litellm-http.workspace = true
litellm-host.workspace = true
litellm-llms.workspace = true
litellm-types.workspace = true
litellm-host-python.workspace = true

View file

@ -0,0 +1,79 @@
//! The callback contracts a Python call runs under, composed here once so that no route
//! selects one. Every call runs the legacy `Logging` contract, for as long as Python's
//! `Logging` exists, and then the v1 contract whenever anything is subscribed to it.
//! Both are a [`PythonLifecycle`]; that trait is the whole boundary the driver knows.
use litellm_callbacks_legacy_python::{LegacyPythonLifecycle, LegacyPythonSurface};
pub(crate) use litellm_callbacks_legacy_python::{PassThroughStream, PublicCall};
use litellm_callbacks_v1_python::{V1PythonLifecycle, V1PythonSurface};
use litellm_host::{machine::Machine, route::Route};
use litellm_host_python::{LifecycleChain, PythonLifecycle, RouteHost, run_call};
use pyo3::prelude::*;
/// What a route tells the callback contracts about itself.
#[derive(Clone, Copy, Debug)]
pub(crate) struct CallSurface {
pub call_type: &'static str,
/// Legacy only: what `Logging.pre_call` is told the input was.
pub input_description: &'static str,
/// Legacy only: how a streamed response is billed; `None` for a route that never
/// streams.
pub stream: Option<PassThroughStream>,
}
impl CallSurface {
fn legacy(self) -> LegacyPythonSurface {
LegacyPythonSurface {
call_type: self.call_type,
input_description: self.input_description,
stream: self.stream,
}
}
fn v1(self) -> V1PythonSurface {
V1PythonSurface {
call_type: self.call_type,
}
}
}
/// Legacy first: v1 subscribers observe the keyword view and the wire request as the
/// legacy contract left them, and share the `litellm_call_id` its setup assigned.
fn lifecycle(
py: Python<'_>,
surface: CallSurface,
call: PublicCall,
asynchronous: bool,
) -> PyResult<Box<dyn PythonLifecycle>> {
let legacy = Box::new(LegacyPythonLifecycle::new(
py,
surface.legacy(),
call,
asynchronous,
));
Ok(
match V1PythonLifecycle::subscribed(py, surface.v1(), asynchronous)? {
Some(v1) => Box::new(LifecycleChain::new(vec![legacy, Box::new(v1)])),
None => legacy,
},
)
}
/// Runs one native call for a Python caller. The route host projects from the keyword
/// view the lifecycle's `begin` prepares.
pub(crate) fn run_python_call<H, M>(
py: Python<'_>,
surface: CallSurface,
call: PublicCall,
machine: M,
route: H,
asynchronous: bool,
) -> PyResult<Py<PyAny>>
where
H: RouteHost + 'static,
M: Machine<Route = H::Route, Complete = <H::Route as Route>::Response> + 'static,
{
let arguments = call.arguments(py);
let lifecycle = lifecycle(py, surface, call, asynchronous)?;
run_call(py, machine, route, lifecycle, arguments, asynchronous)
}

View file

@ -1,3 +1,4 @@
mod callbacks;
mod credentials;
mod diagnostics;
mod errors;

View file

@ -1,18 +1,18 @@
mod host;
use host::MessagesRouteHost;
use litellm_callbacks_legacy_python::{
LegacySurface, PassThroughStream, PublicCall, run_legacy_call,
};
use litellm_core::messages::route::{messages_machine, supports};
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use crate::errors::RustBridgeDeclined;
use crate::{
callbacks::{CallSurface, PassThroughStream, PublicCall, run_python_call},
errors::RustBridgeDeclined,
};
const SURFACE: LegacySurface = LegacySurface {
const SURFACE: CallSurface = CallSurface {
call_type: "anthropic_messages",
input_description: "Messages",
stream: Some(PassThroughStream {
@ -39,7 +39,7 @@ fn run_messages(
"the Rust Messages route does not serve this provider",
));
}
run_legacy_call(
run_python_call(
py,
SURFACE,
PublicCall::capture(&request, &args, &kwargs)?,

View file

@ -26,6 +26,14 @@ pub(super) fn to_pyerr(error: Error) -> PyErr {
.ok();
error
}
Error::InvalidProvider(provider) => {
let error = core_error_to_pyerr(Error::InvalidProvider(provider.clone()).into());
error
.value(py)
.setattr("ocr_invalid_provider", provider)
.ok();
error
}
Error::FileRead { path, source } if source.kind() == std::io::ErrorKind::NotFound => {
PyFileNotFoundError::new_err(format!("File not found: {}", path.display()))
}

View file

@ -7,7 +7,6 @@ use std::sync::{Arc, LazyLock};
use host::OcrRouteHost;
use litellm_auth_gcp::VertexAuth;
use litellm_callbacks_legacy_python::{LegacySurface, PublicCall, run_legacy_call};
use litellm_core::ocr::route::ocr_machine;
use litellm_core_utils::settings::ProcessEnvironment;
use litellm_llms::base_llm::ocr::{
@ -19,15 +18,20 @@ use pyo3::{
types::{PyDict, PyTuple},
};
use crate::{errors::RustBridgeDeclined, http, python_settings::PythonSettings};
use crate::{
callbacks::{CallSurface, PublicCall, run_python_call},
errors::RustBridgeDeclined,
http,
python_settings::PythonSettings,
};
const SURFACE: LegacySurface = LegacySurface {
const SURFACE: CallSurface = CallSurface {
call_type: "ocr",
input_description: "OCR document processing",
stream: None,
};
const ASYNC_SURFACE: LegacySurface = LegacySurface {
const ASYNC_SURFACE: CallSurface = CallSurface {
call_type: "aocr",
..SURFACE
};
@ -52,7 +56,7 @@ fn run_ocr(
secrets,
)
.map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?;
run_legacy_call(
run_python_call(
py,
if asynchronous { ASYNC_SURFACE } else { SURFACE },
PublicCall::capture(&request, &args, &kwargs)?,

View file

@ -83,21 +83,27 @@ enum ProjectedDocument {
impl ProjectedDocument {
fn project(document: &Bound<'_, PyAny>) -> PyResult<Self> {
let kind: String = document
.get_item("type")
.and_then(|value| value.extract())
.map_err(|error| {
let py = document.py();
if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py)
|| error.is_instance_of::<pyo3::exceptions::PyTypeError>(py)
{
ocr_error_to_pyerr(Error::RequestField {
path: "document.type".into(),
})
} else {
error
}
})?;
let py = document.py();
if !document.is_instance_of::<PyDict>() && !document.hasattr("__getitem__")? {
return Err(ocr_error_to_pyerr(Error::DocumentNotObject));
}
let kind = match document.get_item("type") {
Ok(value) => value,
Err(error) if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py) => {
return Err(ocr_error_to_pyerr(Error::InvalidDocumentType(
"None".into(),
)));
}
Err(error) if error.is_instance_of::<pyo3::exceptions::PyTypeError>(py) => {
return Err(ocr_error_to_pyerr(Error::DocumentNotObject));
}
Err(error) => return Err(error),
};
let Ok(kind) = kind.extract::<String>() else {
return Err(ocr_error_to_pyerr(Error::InvalidDocumentType(
kind.str()?.to_string(),
)));
};
if kind != "file" {
return Ok(Self::Other(from_py(document)?));
}
@ -471,21 +477,25 @@ document = Document()
}
#[rstest::rstest]
#[case::missing(c"{}")]
#[case::non_string(c"{'type': 1}")]
#[case::list(c"[]")]
fn malformed_document_discriminators_are_bad_requests_naming_the_field(
#[case::missing(
c"{}",
"Invalid document type: None. Must be 'document_url', 'image_url', or 'file'"
)]
#[case::non_string(
c"{'type': 1}",
"Invalid document type: 1. Must be 'document_url', 'image_url', or 'file'"
)]
#[case::list(c"[]", "document must be a dict with 'type' and URL/file field")]
fn malformed_document_discriminators_are_bad_requests_with_the_python_message(
#[case] document: &std::ffi::CStr,
#[case] expected: &str,
) {
Python::initialize();
Python::attach(|py| {
let error = project_document(&py.eval(document, None, None).unwrap()).unwrap_err();
let value = error.value(py);
assert!(error.is_instance_of::<PyValueError>(py));
assert_eq!(
value.to_string(),
"invalid OCR request field: document.type"
);
assert_eq!(value.to_string(), expected);
assert_eq!(
value
.getattr("status_code")

View file

@ -73,7 +73,7 @@ def _prepare_ocr_request(
raise litellm.BadRequestError(
message="document must be a dict with 'type' and URL/file field",
model=model,
llm_provider=_error_provider(model, custom_llm_provider) or "",
llm_provider=resolve_error_provider(model, custom_llm_provider) or "",
)
normalized_document: Final = (
@ -85,13 +85,13 @@ def _prepare_ocr_request(
raise litellm.BadRequestError(
message=f"Invalid document type: {doc_type}. Must be 'document_url', 'image_url', or 'file'",
model=model,
llm_provider=_error_provider(model, custom_llm_provider) or "",
llm_provider=resolve_error_provider(model, custom_llm_provider) or "",
)
if not normalized_document.get(doc_type):
raise litellm.BadRequestError(
message="Document URL is required",
model=model,
llm_provider=_error_provider(model, custom_llm_provider) or "",
llm_provider=resolve_error_provider(model, custom_llm_provider) or "",
)
(
@ -182,7 +182,7 @@ def _prepare_ocr_request(
)
def _error_provider(model: str, custom_llm_provider: str | None) -> str | None:
def resolve_error_provider(model: str, custom_llm_provider: str | None) -> str | None:
if custom_llm_provider is not None:
return custom_llm_provider
prefix: Final = model.partition("/")[0]
@ -252,7 +252,7 @@ async def aocr(
return response
except Exception as e:
error_provider: Final = _error_provider(model, custom_llm_provider)
error_provider: Final = resolve_error_provider(model, custom_llm_provider)
error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model
raise litellm.exception_type(
model=error_model,
@ -412,7 +412,7 @@ def ocr(
return response
except Exception as e:
error_provider: Final = _error_provider(model, custom_llm_provider)
error_provider: Final = resolve_error_provider(model, custom_llm_provider)
error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model
raise litellm.exception_type(
model=error_model,

View file

@ -0,0 +1,272 @@
import inspect
import json
import threading
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Final, Literal, Protocol, TypeAlias, TypeGuard # noqa: TID251 # recursive JSON validation
from typing_extensions import ReadOnly, TypeAliasType, TypedDict
JSONScalar: TypeAlias = None | bool | int | float | str
JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation
"JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"]
)
SUPPORTED_SCHEMAS: Final = frozenset({1})
EVENTS: Final = frozenset(
{
"call.started",
"request.sending",
"response.received",
"call.succeeded",
"call.failed",
}
)
class TimingFactsV1(TypedDict):
start_time: ReadOnly[float]
end_time: ReadOnly[float]
duration: ReadOnly[float]
class CallFactsV1(TypedDict):
start_time: ReadOnly[float]
asynchronous: ReadOnly[bool]
metadata: ReadOnly[Mapping[str, JSONValue]]
metadata_dropped: ReadOnly[Sequence[str]]
class RequestFactsV1(TypedDict):
model: ReadOnly[str]
custom_llm_provider: ReadOnly[str]
optional_params: ReadOnly[JSONValue]
url: ReadOnly[str]
headers: ReadOnly[Sequence[tuple[str, str]]]
body: ReadOnly[JSONValue]
ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field
"ErrorFactsV1",
{ # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field
"class": ReadOnly[str],
"message": ReadOnly[str],
"status_code": ReadOnly[int | None],
},
)
class CallStartedV1(CallFactsV1):
type: ReadOnly[Literal["call.started"]]
class RequestSendingV1(RequestFactsV1):
type: ReadOnly[Literal["request.sending"]]
class ResponseReceivedV1(TypedDict):
type: ReadOnly[Literal["response.received"]]
body: ReadOnly[str]
class CallSucceededV1(TypedDict):
type: ReadOnly[Literal["call.succeeded"]]
timing: ReadOnly[TimingFactsV1]
streamed: ReadOnly[bool]
response: ReadOnly[JSONValue]
response_error: ReadOnly[str | None]
class CallFailedV1(TypedDict):
type: ReadOnly[Literal["call.failed"]]
timing: ReadOnly[TimingFactsV1]
streamed: ReadOnly[bool]
origin: ReadOnly[Literal["call", "host"]]
error: ReadOnly[ErrorFactsV1]
EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1
class EnvelopeV1(TypedDict):
schema: ReadOnly[Literal[1]]
call_id: ReadOnly[str]
call_type: ReadOnly[str]
seq: ReadOnly[int]
event: ReadOnly[EventV1]
class HeaderPatchV1(TypedDict, total=False):
remove: ReadOnly[Sequence[str]]
set: ReadOnly[Sequence[tuple[str, str]]]
class WirePatchV1(TypedDict, total=False):
headers: ReadOnly[HeaderPatchV1]
body: ReadOnly[JSONValue]
class Callback(Protocol):
name: str
schema: int
events: frozenset[str]
def on_event(self, event: EnvelopeV1) -> None: ...
async def async_on_event(self, event: EnvelopeV1) -> None: ...
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
class RegistrationError(ValueError):
pass
class UnsupportedSchema(RegistrationError):
pass
class UnknownEvent(RegistrationError):
pass
class MissingHandler(RegistrationError):
pass
class EmptyObserverEvents(RegistrationError):
pass
class HandlerKindMismatch(RegistrationError):
pass
class DuplicateName(RegistrationError):
pass
@dataclass(frozen=True, slots=True)
class Subscriber:
name: str
schema: int
events: frozenset[str]
on_event: object | None
async_on_event: object | None
before_send: object | None
async_before_send: object | None
class _Registry:
def __init__(self) -> None:
self.subscribers: tuple[Subscriber, ...] = ()
self.lock: Final = threading.Lock()
_REGISTRY: Final = _Registry()
def _handler(callback: Callback, name: str) -> object | None:
return getattr(callback, name, None)
def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None:
if handler is None:
return
if not callable(handler):
raise HandlerKindMismatch(f"{name} must be callable")
is_async: Final = inspect.iscoroutinefunction(handler)
if is_async != asynchronous:
expected: Final = "async" if asynchronous else "sync"
raise HandlerKindMismatch(f"{name} must be {expected}")
def register(callback: Callback) -> None:
schema: Final = callback.schema
if schema not in SUPPORTED_SCHEMAS:
raise UnsupportedSchema(f"unsupported callback schema: {schema}")
events: Final = frozenset(callback.events)
unknown: Final = events - EVENTS
if unknown:
raise UnknownEvent(f"unknown callback events: {sorted(unknown)}")
on_event: Final = _handler(callback, "on_event")
async_on_event: Final = _handler(callback, "async_on_event")
before_send: Final = _handler(callback, "before_send")
async_before_send: Final = _handler(callback, "async_before_send")
handlers: Final = (on_event, async_on_event, before_send, async_before_send)
if all(handler is None for handler in handlers):
raise MissingHandler("callback must define at least one handler")
if (on_event is not None or async_on_event is not None) and not events:
raise EmptyObserverEvents("observer callback events cannot be empty")
_validate_kind("on_event", on_event, False)
_validate_kind("async_on_event", async_on_event, True)
_validate_kind("before_send", before_send, False)
_validate_kind("async_before_send", async_before_send, True)
subscriber: Final = Subscriber(
name=callback.name,
schema=schema,
events=events,
on_event=on_event,
async_on_event=async_on_event,
before_send=before_send,
async_before_send=async_before_send,
)
with _REGISTRY.lock:
if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers):
raise DuplicateName(f"duplicate callback name: {subscriber.name}")
_REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber)
def unregister(callback: Callback) -> None:
with _REGISTRY.lock:
_REGISTRY.subscribers = tuple(
subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != callback.name
)
def snapshot() -> tuple[Subscriber, ...]:
return _REGISTRY.subscribers
def _is_json_value(
value: object,
) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value
if value is None or isinstance(value, (bool, int, float, str)):
return True
if isinstance(value, list):
return all(
_is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively
for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively
)
if isinstance(value, dict):
return all(
isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively
for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively
)
return False
def project_response(response: object) -> JSONValue:
model_dump: Final = getattr(response, "model_dump", None)
projected: Final = model_dump(mode="json") if callable(model_dump) else response
json.dumps(projected, allow_nan=False)
if _is_json_value(projected):
return projected
raise TypeError(f"unsupported response type: {type(response).__qualname__}")
def new_call_id() -> str:
return str(uuid.uuid4())
def report(name: str, event: str, error: BaseException) -> None:
from litellm._logging import verbose_logger
exception: Final = (type(error), error, error.__traceback__)
verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception)

View file

@ -17,8 +17,8 @@ _UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]])
class UpstreamFailure(Exception):
def __init__(self, response: httpx.Response, cause: Exception) -> None:
super().__init__(str(cause))
self.message: Final = str(cause)
super().__init__(response.text)
self.message: Final = response.text
self.response: Final = response
self.status_code: Final = response.status_code
self.__cause__ = cause
@ -49,13 +49,13 @@ class ExceptionMapper(Protocol):
) -> Exception: ...
def map_failure(error: Exception, model: str, request_provider: str, kwargs: Mapping[str, object]) -> Exception:
def map_failure(error: Exception, model: str, request_provider: str | None, kwargs: Mapping[str, object]) -> Exception:
mapper: Final = cast( # cast-ok: bounded adapter for the legacy public exception mapper
ExceptionMapper, litellm.exception_type
)
try:
return mapper(
model=model.removeprefix(f"{request_provider}/"),
model=model.removeprefix(f"{request_provider}/") if request_provider else model,
custom_llm_provider=request_provider,
original_exception=error,
completion_kwargs=dict(kwargs), # mutable-ok: exception mapper requires owned kwargs
@ -67,7 +67,11 @@ def map_failure(error: Exception, model: str, request_provider: str, kwargs: Map
def map_native_failure(
error: Exception, model: str, request_provider: str, kwargs: Mapping[str, object], api_base: str | None = None
error: Exception,
model: str,
request_provider: str | None,
kwargs: Mapping[str, object],
api_base: str | None = None,
) -> Exception:
"""`map_failure`, reading a native `(status, body)` provider failure as the HTTP response it was."""
original: Final = _upstream_failure(error, api_base)

View file

@ -8,6 +8,7 @@ from pydantic import TypeAdapter
import litellm
from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse
from litellm.ocr.main import resolve_error_provider
from litellm.rust_bridge import failures
from litellm.rust_bridge.failures import UpstreamFailure
from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest
@ -31,11 +32,31 @@ def arguments(request: LiteLLMOcrRequest) -> Mapping[str, object]:
return request.kwargs
def map_failure(error: Exception, request: LiteLLMOcrRequest, request_provider: str) -> Exception:
def map_failure(error: Exception, request: LiteLLMOcrRequest, projected_provider: str) -> Exception:
request_provider: Final = projected_provider or resolve_error_provider(request.model, request.custom_llm_provider)
if getattr(error, "ocr_request_format_error", False):
return litellm.UnsupportedParamsError(
message=f"Invalid `req_format`: {request.kwargs.get('req_format')!r}. Expected 'native' or 'litellm'.",
model=request.model.removeprefix(f"{request_provider}/"),
llm_provider=request_provider,
message=f"Invalid `req_format`: {request.kwargs.get('req_format')!r}. Expected one of litellm, native.",
model=request.model.removeprefix(f"{request_provider}/") if request_provider else request.model,
llm_provider=request_provider or "",
)
invalid_provider: Final = getattr(error, "ocr_invalid_provider", None)
if isinstance(invalid_provider, str):
return _invalid_provider_failure(invalid_provider, request)
return failures.map_native_failure(error, request.model, request_provider, arguments(request), request.api_base)
def _invalid_provider_failure(provider: str, request: LiteLLMOcrRequest) -> Exception:
"""Python rejects a LiteLLM provider without OCR support differently from an unknown provider prefix."""
if provider in litellm.provider_list:
return failures.map_failure(
ValueError(f"OCR is not supported for provider: {provider}"), request.model, None, arguments(request)
)
return litellm.BadRequestError(
message=(
"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. "
f"You passed model={request.model}"
),
model=request.model,
llm_provider="",
)

View file

@ -68,6 +68,7 @@ IGNORE_FUNCTIONS = [
"_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side.
"_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
"completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None.
"_is_json_value", # walks a tree that already passed json.dumps(allow_nan=False): finite, acyclic, and bounded by JSON nesting depth.
]

View file

@ -52,13 +52,21 @@ class ExecutionVariant:
class SubprocessWorker:
def __init__(self, runner: SubprocessRunner, provider: ReplayServer, variant: ExecutionVariant) -> None:
def __init__(
self,
runner: SubprocessRunner,
provider: ReplayServer,
variant: ExecutionVariant,
*,
baseline: bool,
) -> None:
project_root: Final = str(PROJECT_ROOT)
existing_pythonpath: Final = os.environ.get("PYTHONPATH")
env: Final = {
**os.environ,
**dict(variant.environment),
"LITELLM_USER_AGENT": runner.baseline_user_agent,
# Both implementations honor LITELLM_USER_AGENT, so only the baseline carries the sentinel.
**({"LITELLM_USER_AGENT": runner.baseline_user_agent} if baseline else {}),
"PYTHONPATH": os.pathsep.join(path for path in (project_root, existing_pythonpath) if path),
}
self.mode: Final = variant.name
@ -159,9 +167,11 @@ class SubprocessWorker:
def execution_worker(
runner: SubprocessRunner,
variant: ExecutionVariant,
*,
baseline: bool,
) -> Generator[SubprocessWorker]:
with replay_server() as provider:
worker: Final = SubprocessWorker(runner, provider, variant)
worker: Final = SubprocessWorker(runner, provider, variant, baseline=baseline)
try:
yield worker
finally:
@ -174,8 +184,8 @@ def execution_worker_pair(
baseline: ExecutionVariant,
candidate: ExecutionVariant,
) -> Generator[tuple[SubprocessWorker, SubprocessWorker]]:
with execution_worker(runner, baseline) as baseline_worker:
with execution_worker(runner, candidate) as candidate_worker:
with execution_worker(runner, baseline, baseline=True) as baseline_worker:
with execution_worker(runner, candidate, baseline=False) as candidate_worker:
yield baseline_worker, candidate_worker

View file

@ -234,14 +234,6 @@ INVALID_OCR_CASES: Final = (
expected_status_code=500,
expected_message="OCR is not supported for provider: openai",
),
InvalidOcrCase(
name="unsupported_reducto_model",
model="reducto/parse-v4",
document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
expected_exception_type="litellm.exceptions.APIConnectionError",
expected_status_code=500,
expected_message="OCR is not supported for provider: reducto",
),
InvalidOcrCase(
name="unknown_provider_prefix",
model="not_a_provider/model",
@ -254,40 +246,40 @@ INVALID_OCR_CASES: Final = (
name="non_object_document",
model="mistral/mistral-ocr-latest",
document=[],
expected_exception_type="litellm.exceptions.APIConnectionError",
expected_status_code=500,
expected_exception_type="litellm.exceptions.BadRequestError",
expected_status_code=400,
expected_message="document must be a dict",
),
InvalidOcrCase(
name="missing_document_type",
model="mistral/mistral-ocr-latest",
document={},
expected_exception_type="litellm.exceptions.APIConnectionError",
expected_status_code=500,
expected_exception_type="litellm.exceptions.BadRequestError",
expected_status_code=400,
expected_message="Invalid document type: None",
),
InvalidOcrCase(
name="unsupported_document_type",
model="mistral/mistral-ocr-latest",
document={"type": "text", "text": "not a document"},
expected_exception_type="litellm.exceptions.APIConnectionError",
expected_status_code=500,
expected_exception_type="litellm.exceptions.BadRequestError",
expected_status_code=400,
expected_message="Invalid document type: text",
),
InvalidOcrCase(
name="missing_document_url",
model="azure_ai/doc-intelligence/prebuilt-read",
document={"type": "document_url"},
expected_exception_type="litellm.exceptions.APIConnectionError",
expected_status_code=500,
expected_exception_type="litellm.exceptions.BadRequestError",
expected_status_code=400,
expected_message="Document URL is required",
),
InvalidOcrCase(
name="missing_image_url",
model="azure_ai/doc-intelligence/prebuilt-read",
document={"type": "image_url"},
expected_exception_type="litellm.exceptions.APIConnectionError",
expected_status_code=500,
expected_exception_type="litellm.exceptions.BadRequestError",
expected_status_code=400,
expected_message="Document URL is required",
),
InvalidOcrCase(
@ -303,8 +295,8 @@ INVALID_OCR_CASES: Final = (
name="invalid_document_intelligence_pages",
model="azure_ai/doc-intelligence/prebuilt-read",
document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
expected_exception_type="litellm.exceptions.APIConnectionError",
expected_status_code=500,
expected_exception_type="litellm.exceptions.BadRequestError",
expected_status_code=400,
expected_message="`pages` integers must be >= 0",
extra_kwargs=(("pages", [-1]),),
),
@ -312,8 +304,8 @@ INVALID_OCR_CASES: Final = (
name="invalid_document_intelligence_features",
model="azure_ai/doc-intelligence/prebuilt-read",
document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
expected_exception_type="litellm.exceptions.APIConnectionError",
expected_status_code=500,
expected_exception_type="litellm.exceptions.BadRequestError",
expected_status_code=400,
expected_message="Invalid `features` for Azure Document Intelligence",
extra_kwargs=(("features", [1]),),
),

View file

@ -83,3 +83,33 @@ def test_map_failure_leaves_non_upstream_errors_unwrapped() -> None:
def test_map_failure_reports_invalid_request_format_as_unsupported_params() -> None:
with pytest.raises(litellm.UnsupportedParamsError, match="Invalid `req_format`: 'markdown'"):
raise map_failure(RustFormatError(), REQUEST, "mistral")
def test_failure_before_projection_reports_the_provider_python_reports() -> None:
error: Final = ValueError("Document URL is required")
error.status_code = 400 # type: ignore[attr-defined]
mapped: Final = map_failure(error, REQUEST, "")
assert isinstance(mapped, litellm.BadRequestError)
assert mapped.llm_provider == "mistral"
assert "BadRequestError: - " not in str(mapped)
@pytest.mark.parametrize(
("provider", "expected_type", "expected_message"),
[
("openai", litellm.APIConnectionError, "OCR is not supported for provider: openai"),
("not_a_provider", litellm.BadRequestError, "LLM Provider NOT provided"),
],
)
def test_invalid_provider_failures_follow_the_python_route(
provider: str, expected_type: type[Exception], expected_message: str
) -> None:
error: Final = ValueError(f"invalid provider: {provider}")
error.ocr_invalid_provider = provider # type: ignore[attr-defined]
mapped: Final = map_failure(error, REQUEST, "")
assert isinstance(mapped, expected_type)
assert expected_message in str(mapped)

View file

@ -10,7 +10,7 @@ from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.rust_bridge import callbacks_legacy_python as legacy
from litellm.rust_bridge import callbacks_legacy_python as callbacks
from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup
_OCR_KWARGS: Final = MappingProxyType(
@ -81,12 +81,10 @@ def test_setup_builds_a_logger_when_none_is_supplied(call_type: str, kwargs: Map
assert result.logger.litellm_call_id == result.kwargs["litellm_call_id"]
CONTRACT_PATH: Final = (
Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json"
)
CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json"
def test_the_rust_contract_matches_the_shim_signatures() -> None:
contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text())
assert contract == {name: list(inspect.signature(getattr(legacy, name)).parameters) for name in contract}
assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract}

View file

@ -0,0 +1,178 @@
import inspect
import warnings
from collections.abc import Generator
from pathlib import Path
from typing import Final, cast
import pytest
from pydantic import BaseModel, TypeAdapter
from litellm.rust_bridge import callbacks_v1_python as callbacks
ROOT: Final = Path(__file__).parents[3]
CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json"
GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1"
def _clear_registry() -> None:
for subscriber in callbacks.snapshot():
callbacks.unregister(cast(callbacks.Callback, subscriber))
@pytest.fixture(autouse=True)
def clear_registry() -> Generator[None, None, None]:
_clear_registry()
yield
_clear_registry()
def test_the_rust_contract_matches_the_shim_signatures() -> None:
contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text())
assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract}
def test_v1_golden_envelopes_match_the_typed_contract() -> None:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning)
adapter: Final = TypeAdapter(callbacks.EnvelopeV1)
envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")]
assert {envelope["event"]["type"] for envelope in envelopes} == callbacks.EVENTS
assert {envelope["schema"] for envelope in envelopes} == callbacks.SUPPORTED_SCHEMAS
class ValidCallback:
name: str = "valid"
schema: int = 1
events: frozenset[str] = frozenset({"call.started"})
def on_event(self, event: callbacks.EnvelopeV1) -> None:
del event
@pytest.mark.parametrize(
("callback", "error"),
[
(
type(
"Unsupported",
(),
{"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None},
)(),
callbacks.UnsupportedSchema,
),
(
type(
"Unknown",
(),
{"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None},
)(),
callbacks.UnknownEvent,
),
(
type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(),
callbacks.MissingHandler,
),
(
type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(),
callbacks.EmptyObserverEvents,
),
(
type(
"AsyncFunction",
(),
{
"name": "async-function",
"schema": 1,
"events": frozenset({"call.started"}),
"async_on_event": lambda *_: None,
},
)(),
callbacks.HandlerKindMismatch,
),
],
ids=["schema", "event", "missing", "empty-events", "async-kind"],
)
def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None:
with pytest.raises(error):
callbacks.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test
class WrongSyncKind:
name: str = "wrong-sync-kind"
schema: int = 1
events: frozenset[str] = frozenset({"call.started"})
async def on_event(self, event: callbacks.EnvelopeV1) -> None:
del event
def test_register_rejects_coroutine_in_sync_slot() -> None:
with pytest.raises(callbacks.HandlerKindMismatch):
callbacks.register(cast(callbacks.Callback, WrongSyncKind()))
class AsyncObserver:
name: str = "async-observer"
schema: int = 1
events: frozenset[str] = frozenset({"call.started"})
async def async_on_event(self, event: callbacks.EnvelopeV1) -> None:
del event
class SyncInterceptor:
name: str = "sync-interceptor"
schema: int = 1
events: frozenset[str] = frozenset()
def before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None:
del request
return None
class AsyncInterceptor:
name: str = "async-interceptor"
schema: int = 1
events: frozenset[str] = frozenset()
async def async_before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None:
del request
return None
@pytest.mark.parametrize("callback", [ValidCallback(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()])
def test_register_accepts_each_handler_family(callback: callbacks.Callback) -> None:
callbacks.register(callback)
assert len(callbacks.snapshot()) == 1
def test_register_rejects_duplicate_names() -> None:
callbacks.register(cast(callbacks.Callback, ValidCallback()))
with pytest.raises(callbacks.DuplicateName):
callbacks.register(cast(callbacks.Callback, ValidCallback()))
def test_snapshot_is_stable_across_later_registry_writes() -> None:
first: Final = ValidCallback()
second: Final = AsyncObserver()
callbacks.register(cast(callbacks.Callback, first))
captured: Final = callbacks.snapshot()
callbacks.register(cast(callbacks.Callback, second))
callbacks.unregister(cast(callbacks.Callback, first))
assert callbacks.snapshot() is not captured
assert tuple(subscriber.name for subscriber in captured) == ("valid",)
class ResponseModel(BaseModel):
value: int
def test_project_response_supports_pydantic_and_json_values() -> None:
assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3}
original: Final = {"nested": [1, True, None]}
assert callbacks.project_response(original) is original
def test_project_response_rejects_unsupported_objects() -> None:
with pytest.raises(TypeError):
callbacks.project_response(object())

View file

@ -52,3 +52,18 @@ def test_kwargs_are_handed_to_the_mapper_as_owned_copies(monkeypatch: pytest.Mon
assert seen[0]["completion_kwargs"] is not request_kwargs
assert seen[0]["model"] == "gpt-4o"
assert seen[0]["custom_llm_provider"] == "openai"
def test_native_provider_failure_message_is_the_provider_body_not_the_args_tuple() -> None:
class NativeUpstream(Exception):
headers = [("retry-after", "7")]
native_error: Final = NativeUpstream(429, '{"message": "slow down"}')
mapped: Final = failures.map_native_failure(
native_error, "mistral/mistral-ocr-latest", "mistral", MappingProxyType({}), "https://api.mistral.ai/v1"
)
assert isinstance(mapped, litellm.RateLimitError)
assert '{"message": "slow down"}' in str(mapped)
assert "(429," not in str(mapped)

View file

@ -0,0 +1,141 @@
from collections.abc import Generator
from typing import Final
import pytest
from litellm.integrations.custom_logger import CustomLogger
from litellm.rust_bridge import callbacks_v1_python
from litellm.rust_bridge.callbacks_v1_python import EVENTS, EnvelopeV1, RequestFactsV1, WirePatchV1
from tests.test_litellm_rust.support.callback_recorder import drain_logging
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
from tests.test_litellm_rust.support.requests import (
OCR_RESPONSE,
call_native_aocr,
call_native_ocr,
request_body,
)
pytestmark = pytest.mark.requires_rust_extension
@pytest.fixture
def ocr_server(recording_server: RecordingServer) -> RecordingServer:
recording_server.default_response = ResponseSpec(body=OCR_RESPONSE)
return recording_server
class Observer:
name: Final = "observer"
schema: Final = 1
events: Final = EVENTS
def __init__(self) -> None:
self.envelopes: Final[list[EnvelopeV1]] = []
def on_event(self, event: EnvelopeV1) -> None:
self.envelopes.append(event)
class Interceptor:
name: Final = "interceptor"
schema: Final = 1
events: Final[frozenset[str]] = frozenset()
def __init__(self) -> None:
self.bodies: Final[list[object]] = []
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None:
self.bodies.append(request["body"])
return {"headers": {"set": [("x-v1-interceptor", "seen")]}}
@pytest.fixture
def observer() -> Generator[Observer]:
subscriber: Final = Observer()
callbacks_v1_python.register(subscriber)
try:
yield subscriber
finally:
callbacks_v1_python.unregister(subscriber)
@pytest.fixture
def interceptor() -> Generator[Interceptor]:
subscriber: Final = Interceptor()
callbacks_v1_python.register(subscriber)
try:
yield subscriber
finally:
callbacks_v1_python.unregister(subscriber)
class LegacyCallId(CustomLogger):
def __init__(self) -> None:
self.call_ids: Final[list[str]] = []
def log_pre_api_call(self, model, messages, kwargs):
self.call_ids.append(kwargs["litellm_call_id"])
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_native_ocr_runs_legacy_and_v1_callbacks_on_the_same_call(
ocr_server: RecordingServer, observer: Observer, asynchronous: bool
) -> None:
legacy: Final = LegacyCallId()
if asynchronous:
await call_native_aocr(ocr_server, callbacks=[legacy])
else:
call_native_ocr(ocr_server, callbacks=[legacy])
await drain_logging()
assert [envelope["event"]["type"] for envelope in observer.envelopes] == [
"call.started",
"request.sending",
"response.received",
"call.succeeded",
]
assert [envelope["seq"] for envelope in observer.envelopes] == sorted(
envelope["seq"] for envelope in observer.envelopes
)
assert len(legacy.call_ids) == 1
assert {envelope["call_id"] for envelope in observer.envelopes} == set(legacy.call_ids)
def test_native_ocr_v1_interceptor_sees_the_wire_as_the_legacy_callback_left_it(
ocr_server: RecordingServer, interceptor: Interceptor
) -> None:
class Edit(CustomLogger):
def log_pre_api_call(self, model, messages, kwargs):
request_body(kwargs)["include_image_base64"] = True
call_native_ocr(ocr_server, callbacks=[Edit()])
assert len(interceptor.bodies) == 1
body: Final = interceptor.bodies[0]
assert isinstance(body, dict)
assert body["include_image_base64"] is True
request: Final = ocr_server.requests[-1]
assert request.headers["x-v1-interceptor"] == "seen"
def test_native_ocr_failure_reaches_both_contracts_once(
recording_server: RecordingServer, observer: Observer
) -> None:
recording_server.default_response = ResponseSpec(status=400, body={"message": "bad document"})
failures: Final[list[object]] = []
class Failure(CustomLogger):
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
failures.append(kwargs["exception"])
with pytest.raises(Exception) as raised: # noqa: PT011 # the mapped provider error class is covered elsewhere
call_native_ocr(recording_server, callbacks=[Failure()])
assert failures == [raised.value]
terminal: Final = [
envelope for envelope in observer.envelopes if envelope["event"]["type"] in {"call.succeeded", "call.failed"}
]
assert [envelope["event"]["type"] for envelope in terminal] == ["call.failed"]
assert observer.envelopes[-1] is terminal[0]