mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
refactor(rust): compose callback lifecycles in a chain and tighten both contracts
Compose the legacy and v1 lifecycles behind one PythonLifecycle through LifecycleChain, built once in python-bridge, and simplify the boundary: - drop LifecycleEvent::Started; begin carries the start time and is the first thing a lifecycle hears of a call - give PythonLifecycle pass-through defaults for the threading steps and the stream notices, and document which steps thread and which observe - export one is_cancellation from litellm-host-python - callbacks-v1: private RequestFacts fields, a golden file per EventKind by construction, Python EVENTS and SUPPORTED_SCHEMAS pinned to the goldens - callbacks-v1-python: the snapshot only parses (register is the only place that rejects), Handler owns call/is_async, Pending keeps a subscriber index - callbacks-legacy-python: LegacyCallbacks becomes inherent PythonLogger methods, deployment hooks become free functions, retained headers are traversed and cleared - define the shared terms in host-python/AGENTS.md and each contract's own terms in its crate - remove test_loader.py, whose fork hook was never committed Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
36c5056972
commit
d187fe8b2c
39 changed files with 1288 additions and 583 deletions
2
litellm-rust/Cargo.lock
generated
2
litellm-rust/Cargo.lock
generated
|
|
@ -2222,8 +2222,10 @@ dependencies = [
|
|||
"litellm-auth",
|
||||
"litellm-auth-gcp",
|
||||
"litellm-callbacks-legacy-python",
|
||||
"litellm-callbacks-v1-python",
|
||||
"litellm-core",
|
||||
"litellm-core-utils",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,20 @@
|
|||
- Target invariants, not completion claims
|
||||
- Keep this crate the legacy `@client` wrapper as the native call sees it, and nothing else: the `Logging` contract (`function_setup`, the deployment hooks, `pre_call`/`post_call`, the sync and async success and failure fan-out, the deferred proxy release, the argument sharing those callbacks rely on) plus the kwargs rewrites the wrapper makes on the way in (credential-name inheritance, the budget and retry-count limits)
|
||||
- The driver in `litellm-host-python`, the routes and core see one `PythonLifecycle`; they never learn which Python objects consume a call
|
||||
- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`)
|
||||
- Terms (lifecycle terms are in `host-python/AGENTS.md`)
|
||||
- `Logging`: litellm's Python class; logger: the one `Logging` instance of this call (`PythonLogger`), reused from `litellm_logging_obj` or built by `function_setup`
|
||||
- Borrowed function: a litellm Python internal Rust still calls, one variant of `LegacyPython`; a user's own callback is never one
|
||||
- Fan-out: `Logging` dispatching to every registered callback; family: the sync or the async set of success and failure handlers
|
||||
- Deployment hooks: the awaited pre-call, post-call and failure hooks on each callback; async calls only, as in `@client`
|
||||
- Re-aliasing: replacing body values with the caller's equal objects before `pre_call`, so a callback mutates what the caller holds
|
||||
- Deferred release: the proxy holds back async success logging until it accepts the response (`PendingLogging.release`)
|
||||
- Internal call: a proxy-internal call; when it is async, the async success dispatch and both failure families are skipped, and the sync success handlers still run
|
||||
- Pass-through billing (`PassThroughStream`): end-of-stream cost logging from the delivered chunks; a failure after the stream opened bills them as partial usage, and a sync call falls back to the plain failure handler
|
||||
- Redaction here is `****`, matching what the Python path logs; `[REDACTED]` belongs to the v1 contract
|
||||
- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`, `Streaming`)
|
||||
- The enum only shrinks: when Rust owns a subsystem, delete its group rather than adding a Rust path beside it
|
||||
- Calling a user's own callback directly is permanent Python surface and gets its own type outside `LegacyPython`
|
||||
- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes hand it over through `run_legacy_python_call` and keep no copy
|
||||
- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes capture it and the bridge's `run_python_call` hands it to `LegacyPythonLifecycle::new`, keeping no copy
|
||||
- `setup` reuses a `Logging` the caller passed as `litellm_logging_obj` (the proxy and Router are the live cases) and otherwise builds one through `function_setup`, as `@client` does
|
||||
- Either way every phase calls the same `Logging` method the Python path calls; which callbacks run is `Logging`'s decision, never this crate's
|
||||
- Callbacks receive the caller's own objects and may mutate them; this crate alone carries that obligation
|
||||
|
|
|
|||
|
|
@ -9,13 +9,11 @@ autotests = false
|
|||
[dependencies]
|
||||
litellm-host.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
|
||||
pyo3.workspace = true
|
||||
strum.workspace = true
|
||||
serde_json.workspace = true
|
||||
strum.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-auth.workspace = true
|
||||
proptest.workspace = true
|
||||
rstest.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ use litellm_host::event::{
|
|||
FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest, epoch_seconds,
|
||||
};
|
||||
use litellm_host_python::{
|
||||
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, missing_state, to_py,
|
||||
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::{PyBaseException, PyException},
|
||||
exceptions::PyBaseException,
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
prelude::*,
|
||||
types::{PyDict, PyList},
|
||||
|
|
@ -17,7 +17,8 @@ use pyo3::{
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger,
|
||||
PublicCall, PythonLogger, after_deployment_failure, after_deployment_success,
|
||||
before_deployment_call,
|
||||
deferred::{PendingLogging, PendingSuccess},
|
||||
finalize, is_internal_call, prepare,
|
||||
python::Streaming,
|
||||
|
|
@ -79,10 +80,6 @@ fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult<Py<PyAny>> {
|
|||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool {
|
||||
!error.is_instance_of::<PyException>(py)
|
||||
}
|
||||
|
||||
impl LegacyPythonLifecycle {
|
||||
pub fn new(
|
||||
py: Python<'_>,
|
||||
|
|
@ -151,7 +148,7 @@ impl LegacyPythonLifecycle {
|
|||
|
||||
fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
|
||||
match self.try_dispatch_success(py) {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
Err(error) if !is_cancellation(py, &error) => {
|
||||
error.write_unraisable(py, self.logger.as_ref().map(|logger| logger.object(py)));
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -210,7 +207,7 @@ impl LegacyPythonLifecycle {
|
|||
),
|
||||
);
|
||||
match billed {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
Err(error) if !is_cancellation(py, &error) => {
|
||||
error.write_unraisable(py, Some(logger.object(py)));
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -285,10 +282,10 @@ impl PythonLifecycle for LegacyPythonLifecycle {
|
|||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
started_at: f64,
|
||||
start_time: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.call.set_kwargs(arguments);
|
||||
self.start = datetime(py, started_at)?;
|
||||
self.start = datetime(py, start_time)?;
|
||||
self.internal = is_internal_call(py)?;
|
||||
let result = setup(
|
||||
py,
|
||||
|
|
@ -302,7 +299,7 @@ impl PythonLifecycle for LegacyPythonLifecycle {
|
|||
self.call.set_kwargs(result.kwargs()?);
|
||||
if self.runs_deployment_hooks() {
|
||||
self.pending = Some(Pending::DeploymentPreCall);
|
||||
return Ok(LifecycleStep::Await(DeploymentHooks::before_call(
|
||||
return Ok(LifecycleStep::Await(before_deployment_call(
|
||||
py,
|
||||
self.call.kwargs(),
|
||||
self.surface.call_type,
|
||||
|
|
@ -365,7 +362,7 @@ impl PythonLifecycle for LegacyPythonLifecycle {
|
|||
self.response = Some(response);
|
||||
if self.runs_deployment_hooks() {
|
||||
self.pending = Some(Pending::DeploymentPostCall);
|
||||
return Ok(LifecycleStep::Await(DeploymentHooks::after_success(
|
||||
return Ok(LifecycleStep::Await(after_deployment_success(
|
||||
py,
|
||||
self.call.kwargs(),
|
||||
&self.response,
|
||||
|
|
@ -377,7 +374,6 @@ impl PythonLifecycle for LegacyPythonLifecycle {
|
|||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
match event {
|
||||
LifecycleEvent::Started { .. } => Ok(LifecycleStep::Done),
|
||||
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
|
||||
let api_key = self
|
||||
.context
|
||||
|
|
@ -418,7 +414,7 @@ impl PythonLifecycle for LegacyPythonLifecycle {
|
|||
{
|
||||
let error = self.error.as_ref().ok_or_else(missing_state)?;
|
||||
self.pending = Some(Pending::DeploymentFailure);
|
||||
return Ok(LifecycleStep::Await(DeploymentHooks::after_failure(
|
||||
return Ok(LifecycleStep::Await(after_deployment_failure(
|
||||
py,
|
||||
self.call.kwargs(),
|
||||
error,
|
||||
|
|
@ -475,7 +471,11 @@ impl PythonLifecycle for LegacyPythonLifecycle {
|
|||
{
|
||||
error.write_unraisable(py, None);
|
||||
}
|
||||
self.pending = None;
|
||||
self.response = None;
|
||||
self.error = None;
|
||||
self.body = None;
|
||||
self.headers = None;
|
||||
self.context = None;
|
||||
self.stream = None;
|
||||
}
|
||||
|
|
@ -493,7 +493,8 @@ impl PythonLifecycle for LegacyPythonLifecycle {
|
|||
visit.call(&stream.chunks)?;
|
||||
visit.call(&stream.first_chunk)?;
|
||||
}
|
||||
visit.call(&self.body)
|
||||
visit.call(&self.body)?;
|
||||
visit.call(&self.headers)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,16 +3,13 @@
|
|||
//! lifetime. No other callback host has that obligation, which is why nothing outside
|
||||
//! this crate holds them.
|
||||
|
||||
use litellm_host::{machine::Machine, route::Route};
|
||||
use litellm_host_python::{RouteHost, lookup, run_call};
|
||||
use litellm_host_python::lookup;
|
||||
use pyo3::{
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
prelude::*,
|
||||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::{LegacyPythonLifecycle, LegacyPythonSurface};
|
||||
|
||||
pub struct PublicCall {
|
||||
args: Py<PyTuple>,
|
||||
kwargs: Py<PyDict>,
|
||||
|
|
@ -34,6 +31,11 @@ impl PublicCall {
|
|||
})
|
||||
}
|
||||
|
||||
/// The keyword view the call starts from, which the driver hands to `begin`.
|
||||
pub fn arguments(&self, py: Python<'_>) -> Py<PyDict> {
|
||||
self.kwargs.clone_ref(py)
|
||||
}
|
||||
|
||||
pub(crate) fn args(&self) -> &Py<PyTuple> {
|
||||
&self.args
|
||||
}
|
||||
|
|
@ -63,31 +65,6 @@ impl PublicCall {
|
|||
}
|
||||
}
|
||||
|
||||
/// Runs one native call under the legacy `Logging` contract: the route host projects from
|
||||
/// the keyword view the contract prepares, and the contract observes the call.
|
||||
pub fn run_legacy_python_call<H, M>(
|
||||
py: Python<'_>,
|
||||
surface: LegacyPythonSurface,
|
||||
call: PublicCall,
|
||||
machine: M,
|
||||
route: H,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Py<PyAny>>
|
||||
where
|
||||
H: RouteHost + 'static,
|
||||
M: Machine<Route = H::Route, Complete = <H::Route as Route>::Response> + 'static,
|
||||
{
|
||||
let arguments = call.kwargs.clone_ref(py);
|
||||
run_call(
|
||||
py,
|
||||
machine,
|
||||
route,
|
||||
Box::new(LegacyPythonLifecycle::new(py, surface, call, asynchronous)),
|
||||
arguments,
|
||||
asynchronous,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -9,78 +9,10 @@ use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
|
|||
use crate::logger::PythonLogger;
|
||||
use crate::python::{Logging, Wrapper};
|
||||
|
||||
pub trait LegacyCallbacks {
|
||||
impl PythonLogger {
|
||||
/// `Logging.update_from_kwargs`: what the logger is told about the request it is
|
||||
/// about to see, with consumed credentials redacted.
|
||||
fn update_from_kwargs(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
wire: &WireRequest,
|
||||
context: &RequestContext,
|
||||
) -> PyResult<()>;
|
||||
|
||||
/// `Logging.pre_call`.
|
||||
fn pre_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
input: &str,
|
||||
api_key: Option<&str>,
|
||||
body: &Bound<'_, PyDict>,
|
||||
headers: &Bound<'_, PyDict>,
|
||||
url: &str,
|
||||
) -> PyResult<()>;
|
||||
|
||||
/// `Logging.post_call`.
|
||||
fn post_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
original_response: &str,
|
||||
api_key: Option<&str>,
|
||||
body: Option<&Py<PyDict>>,
|
||||
headers: Option<&Py<PyDict>>,
|
||||
) -> PyResult<()>;
|
||||
|
||||
fn defers_async_logging(&self, py: Python<'_>) -> bool;
|
||||
|
||||
fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()>;
|
||||
|
||||
fn sync_success_for_async_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()>;
|
||||
|
||||
fn failure(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
error: &Py<PyBaseException>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Option<Py<PyAny>>>;
|
||||
|
||||
fn submit_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()>;
|
||||
|
||||
fn enqueue_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()>;
|
||||
}
|
||||
|
||||
impl LegacyCallbacks for PythonLogger {
|
||||
fn update_from_kwargs(
|
||||
pub(crate) fn update_from_kwargs(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
|
|
@ -128,7 +60,8 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn pre_call(
|
||||
/// `Logging.pre_call`.
|
||||
pub(crate) fn pre_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
input: &str,
|
||||
|
|
@ -145,7 +78,8 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn post_call(
|
||||
/// `Logging.post_call`.
|
||||
pub(crate) fn post_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
original_response: &str,
|
||||
|
|
@ -163,19 +97,19 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn defers_async_logging(&self, py: Python<'_>) -> bool {
|
||||
pub(crate) fn defers_async_logging(&self, py: Python<'_>) -> bool {
|
||||
Logging::DefersAsync
|
||||
.call(py, (self.object(py),))
|
||||
.and_then(|value| value.extract())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> {
|
||||
pub(crate) fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> {
|
||||
Logging::DeferSuccess.call(py, (self.object(py), pending))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sync_success_for_async_call(
|
||||
pub(crate) fn sync_success_for_async_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
|
|
@ -186,7 +120,7 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn failure(
|
||||
pub(crate) fn failure(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
error: &Py<PyBaseException>,
|
||||
|
|
@ -199,7 +133,7 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(asynchronous.then(|| value.unbind()))
|
||||
}
|
||||
|
||||
fn submit_success(
|
||||
pub(crate) fn submit_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
|
|
@ -210,7 +144,7 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn enqueue_success(
|
||||
pub(crate) fn enqueue_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
|
|
@ -227,9 +161,7 @@ impl LegacyCallbacks for PythonLogger {
|
|||
}
|
||||
enqueue.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonLogger {
|
||||
pub(crate) fn initialize_failure_context(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
|
|
@ -305,6 +237,6 @@ fn redact(
|
|||
}
|
||||
|
||||
/// Proxy-internal calls skip the legacy success fan-out.
|
||||
pub fn is_internal_call(py: Python<'_>) -> PyResult<bool> {
|
||||
pub(crate) fn is_internal_call(py: Python<'_>) -> PyResult<bool> {
|
||||
Wrapper::IsInternalCall.call(py, ())?.extract()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
//! The proxy's deferred success release: the async success handler is queued only once
|
||||
//! the proxy accepts the response, and at most once.
|
||||
|
||||
use pyo3::{exceptions::PyException, prelude::*};
|
||||
use litellm_host_python::is_cancellation;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::{LegacyCallbacks, PythonLogger};
|
||||
use crate::PythonLogger;
|
||||
|
||||
pub(crate) struct PendingSuccess {
|
||||
pub(crate) logger: PythonLogger,
|
||||
|
|
@ -37,7 +38,7 @@ impl PendingLogging {
|
|||
&& success
|
||||
{
|
||||
match pending.asynchronous(py) {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
Err(error) if !is_cancellation(py, &error) => {
|
||||
error.write_unraisable(py, Some(pending.logger.object(py)));
|
||||
}
|
||||
result => return result,
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
//! core never learn which Python object is on the other end.
|
||||
//!
|
||||
//! Legacy callbacks receive the caller's own objects and may mutate them. [`PublicCall`]
|
||||
//! is where those objects live, and [`run_legacy_python_call`] is how a route hands them over
|
||||
//! without keeping a copy.
|
||||
//! is where those objects live; the bridge hands it to [`LegacyPythonLifecycle::new`] and
|
||||
//! keeps no copy.
|
||||
|
||||
mod adapter;
|
||||
mod call;
|
||||
|
|
@ -20,9 +20,11 @@ mod python;
|
|||
#[path = "../tests/support.rs"]
|
||||
mod test_support;
|
||||
|
||||
pub(crate) use adapter::LegacyPythonLifecycle;
|
||||
pub use adapter::{LegacyPythonSurface, PassThroughStream};
|
||||
pub use call::{PublicCall, run_legacy_python_call};
|
||||
pub(crate) use callbacks::{LegacyCallbacks, is_internal_call};
|
||||
pub(crate) use logger::{DeploymentHooks, PythonLogger, finalize, setup};
|
||||
pub use adapter::{LegacyPythonLifecycle, LegacyPythonSurface, PassThroughStream};
|
||||
pub use call::PublicCall;
|
||||
pub(crate) use callbacks::is_internal_call;
|
||||
pub(crate) use logger::{
|
||||
PythonLogger, after_deployment_failure, after_deployment_success, before_deployment_call,
|
||||
finalize, setup,
|
||||
};
|
||||
pub(crate) use preparation::prepare;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! One call's `Logging` instance (the logger) and the `@client` wrapper functions that
|
||||
//! create and finish it: `function_setup`, `finalize` and the awaited deployment hooks.
|
||||
|
||||
use pyo3::{
|
||||
exceptions::PyBaseException,
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
|
|
@ -5,10 +8,10 @@ use pyo3::{
|
|||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::python::{self, Wrapper};
|
||||
use crate::python::{DeploymentHooks, Wrapper};
|
||||
|
||||
/// The `Logging` instance one call fans out through.
|
||||
pub struct PythonLogger {
|
||||
pub(crate) struct PythonLogger {
|
||||
object: Py<PyAny>,
|
||||
}
|
||||
|
||||
|
|
@ -21,17 +24,17 @@ impl PythonLogger {
|
|||
self.object.bind(py)
|
||||
}
|
||||
|
||||
pub fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
Self {
|
||||
object: self.object.clone_ref(py),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
visit.call(&self.object)
|
||||
}
|
||||
|
||||
pub fn restore_context(&self, py: Python<'_>) -> PyResult<()> {
|
||||
pub(crate) fn restore_context(&self, py: Python<'_>) -> PyResult<()> {
|
||||
Wrapper::RestoreContext.call(py, (self.object(py),))?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -45,19 +48,19 @@ impl FromPyObject<'_, '_> for PythonLogger {
|
|||
}
|
||||
}
|
||||
|
||||
pub struct SetupResult<'py>(Bound<'py, PyAny>);
|
||||
pub(crate) struct SetupResult<'py>(Bound<'py, PyAny>);
|
||||
|
||||
impl SetupResult<'_> {
|
||||
pub fn logger(&self) -> PyResult<PythonLogger> {
|
||||
pub(crate) fn logger(&self) -> PyResult<PythonLogger> {
|
||||
Ok(PythonLogger::new(self.0.getattr("logger")?.unbind()))
|
||||
}
|
||||
|
||||
pub fn kwargs(&self) -> PyResult<Py<PyDict>> {
|
||||
pub(crate) fn kwargs(&self) -> PyResult<Py<PyDict>> {
|
||||
Ok(self.0.getattr("kwargs")?.extract()?)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn setup<'py>(
|
||||
pub(crate) fn setup<'py>(
|
||||
py: Python<'py>,
|
||||
call_type: &str,
|
||||
args: &Py<PyTuple>,
|
||||
|
|
@ -70,7 +73,7 @@ pub fn setup<'py>(
|
|||
.map(SetupResult)
|
||||
}
|
||||
|
||||
pub fn finalize(
|
||||
pub(crate) fn finalize(
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
logger: &PythonLogger,
|
||||
|
|
@ -82,40 +85,36 @@ pub fn finalize(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub struct DeploymentHooks;
|
||||
pub(crate) fn before_deployment_call(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
DeploymentHooks::BeforeDeploymentCall
|
||||
.call(py, (kwargs, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
impl DeploymentHooks {
|
||||
pub fn before_call(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
python::DeploymentHooks::BeforeDeploymentCall
|
||||
.call(py, (kwargs, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
pub(crate) fn after_deployment_success(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
DeploymentHooks::AfterDeploymentSuccess
|
||||
.call(py, (kwargs, response, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
pub fn after_success(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
python::DeploymentHooks::AfterDeploymentSuccess
|
||||
.call(py, (kwargs, response, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
pub fn after_failure(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
error: &Py<PyBaseException>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
python::DeploymentHooks::AfterDeploymentFailure
|
||||
.call(py, (kwargs, error, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
pub(crate) fn after_deployment_failure(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
error: &Py<PyBaseException>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
DeploymentHooks::AfterDeploymentFailure
|
||||
.call(py, (kwargs, error, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! The kwargs rewrites the `@client` wrapper makes before the call: the logger is attached,
|
||||
//! a named credential's values are inherited, and the budget and retry-count limits run.
|
||||
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyDict, PyList},
|
||||
|
|
@ -17,7 +20,7 @@ impl<'py> CredentialEntry<'py> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn prepare<'py>(
|
||||
pub(crate) fn prepare<'py>(
|
||||
py: Python<'py>,
|
||||
kwargs: &Bound<'py, PyDict>,
|
||||
logger: &crate::PythonLogger,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! The Python contract of this crate: every function of
|
||||
//! `litellm/rust_bridge/callbacks_legacy_python.py` that Rust calls, by name.
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use strum::{IntoStaticStr, VariantArray};
|
||||
|
||||
|
|
@ -153,7 +156,7 @@ mod tests {
|
|||
use crate::test_support::PYTHON_CONTRACT;
|
||||
|
||||
#[test]
|
||||
fn every_borrowed_function_is_in_the_python_contract() {
|
||||
fn every_function_is_in_the_python_contract() {
|
||||
let contract: serde_json::Map<String, serde_json::Value> =
|
||||
serde_json::from_str(PYTHON_CONTRACT).unwrap();
|
||||
let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect();
|
||||
|
|
@ -177,7 +180,7 @@ mod tests {
|
|||
)
|
||||
.map(LegacyPython::name)
|
||||
.collect();
|
||||
assert_eq!(called.len(), declared.len(), "a function is borrowed twice");
|
||||
assert_eq!(called.len(), declared.len(), "a function is listed twice");
|
||||
assert_eq!(called.into_iter().collect::<BTreeSet<_>>(), declared);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,17 +2,26 @@
|
|||
- Keep this crate the Python runtime adapter for `litellm-callbacks-v1` and nothing else: the subscriber snapshot and one `PythonLifecycle` that delivers the core contract
|
||||
- Envelopes, sequencing, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1`
|
||||
- `litellm/rust_bridge/callbacks_v1_python.py` is the only Python module behind it; `python_contract.json` pins its functions on both sides
|
||||
- Terms (contract terms are in `callbacks-v1/AGENTS.md`, lifecycle terms in `host-python/AGENTS.md`)
|
||||
- Registry: the process-wide subscriber list; it lives in Python (`register`/`unregister`), never in Rust
|
||||
- Snapshot: that list read once per call, with one observer and one interceptor handler already selected per subscriber for the call's mode (`subscribers.rs`)
|
||||
- Projection: the public response turned into JSON by `project_response`; a failed projection becomes `response_error`, never a failed call
|
||||
- Report: logging an observer's swallowed error through `report`
|
||||
- Continuation: what a step returns to the driver once every observer of its envelope has run
|
||||
- "Callback" in the Python module (`Callback` protocol) is the user's object; once registered it is a subscriber
|
||||
- No Python object of the call reaches a callback; a callback influences a call only through a patch Rust validates
|
||||
- `begin` returns the dict it was given and `after_success` the response it was given, by identity
|
||||
- `begin` returns the dict it was given by identity; `after_success` and `delivered` stay the trait's pass-through defaults
|
||||
- Every callback gets a freshly built envelope per event; never cache or share a converted one
|
||||
- Contract values cross into Python only through a fresh conversion per callback
|
||||
- Schema v1 is owned by `litellm-callbacks-v1`; this crate only marshals it
|
||||
- Everything rejectable is rejected at `register`, never during a call
|
||||
- Everything rejectable is rejected at `register`, never during a call; the snapshot only parses and the lifecycle never re-validates a subscriber
|
||||
- Observers cannot affect a call: ordinary exceptions are reported and swallowed at every event, awaited or not; an interceptor failure fails the call
|
||||
- A cancellation (anything that is not a `PyException`) is returned at once, with no further dispatch and no report
|
||||
- A cancellation is returned at once, with no further dispatch and no report
|
||||
- Exactly one terminal envelope, last; none after a cancellation
|
||||
- In a sync call no step suspends: handler selection happens once, at snapshot, from what registration recorded
|
||||
- Every suspension is awaited inline in the caller's task; never `into_future`, never spawn
|
||||
- The subscriber snapshot is taken once per call before the driver starts; later registrations do not reach that call
|
||||
- `V1PythonLifecycle::subscribed` is the only constructor callers get; it returns `None` for an empty snapshot, so an unobserved call builds no envelope and projects no response
|
||||
- This lifecycle may run after another one in a `LifecycleChain`; it observes the arguments and the wire as that one left them and never assumes it is alone
|
||||
- `close` is idempotent; `traverse` visits every retained Python edge and calls no Python
|
||||
- This crate has none of `callbacks-legacy-python`'s obligations and must not grow them; shared behaviour belongs to `PythonLifecycle`, not to either adapter
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
//! The v1 lifecycle: each step builds one envelope, hands every subscribed observer a
|
||||
//! fresh conversion of it, and, before the request is sent, folds every interceptor's
|
||||
//! patch into the wire request.
|
||||
|
||||
use litellm_callbacks_v1::{
|
||||
CallFacts, Envelope, ErrorFacts, Event, RequestFacts, SCHEMA_V1, Sequencer, WirePatch, apply,
|
||||
CallFacts, Envelope, ErrorFacts, Event, RequestFacts, Sequencer, WirePatch, apply,
|
||||
};
|
||||
use litellm_host::event::{MachineEvent, RequestContext, Timing, WireRequest};
|
||||
use litellm_host::event::{MachineEvent, RequestContext, WireRequest};
|
||||
use litellm_host_python::{
|
||||
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, missing_state, to_py,
|
||||
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py,
|
||||
};
|
||||
use pyo3::exceptions::{PyException, PyValueError};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyString};
|
||||
|
|
@ -13,26 +17,26 @@ use serde_json::{Map, Value};
|
|||
|
||||
use crate::call::V1PythonSurface;
|
||||
use crate::python::V1Python;
|
||||
use crate::registry::{Handler, Subscriber};
|
||||
use crate::subscribers::Subscriber;
|
||||
|
||||
enum Then {
|
||||
/// What a step hands back to the driver once every observer of its envelope has run.
|
||||
enum Continuation {
|
||||
Done,
|
||||
Arguments(Py<PyDict>),
|
||||
Wire(Box<WireRequest>),
|
||||
}
|
||||
|
||||
/// The suspended step; `index` is the subscriber whose awaitable the driver is awaiting.
|
||||
enum Pending {
|
||||
Observe {
|
||||
envelope: Envelope,
|
||||
next: usize,
|
||||
then: Then,
|
||||
awaiting: String,
|
||||
index: usize,
|
||||
then: Continuation,
|
||||
},
|
||||
Intercept {
|
||||
wire: Box<WireRequest>,
|
||||
context: RequestContext,
|
||||
next: usize,
|
||||
awaiting: String,
|
||||
index: usize,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -40,18 +44,13 @@ pub struct V1PythonLifecycle {
|
|||
surface: V1PythonSurface,
|
||||
subscribers: Vec<Subscriber>,
|
||||
asynchronous: bool,
|
||||
start_time: f64,
|
||||
sequencer: Option<Sequencer>,
|
||||
streamed: bool,
|
||||
pending: Option<Pending>,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool {
|
||||
!error.is_instance_of::<PyException>(py)
|
||||
}
|
||||
|
||||
impl Then {
|
||||
impl Continuation {
|
||||
fn into_step(self) -> LifecycleStep {
|
||||
match self {
|
||||
Self::Done => LifecycleStep::Done,
|
||||
|
|
@ -69,12 +68,15 @@ impl Then {
|
|||
}
|
||||
|
||||
impl V1PythonLifecycle {
|
||||
pub fn new(surface: V1PythonSurface, subscribers: Vec<Subscriber>, asynchronous: bool) -> Self {
|
||||
pub(crate) fn new(
|
||||
surface: V1PythonSurface,
|
||||
subscribers: Vec<Subscriber>,
|
||||
asynchronous: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
surface,
|
||||
subscribers,
|
||||
asynchronous,
|
||||
start_time: 0.0,
|
||||
sequencer: None,
|
||||
streamed: false,
|
||||
pending: None,
|
||||
|
|
@ -92,10 +94,11 @@ impl V1PythonLifecycle {
|
|||
fn report(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
name: &str,
|
||||
index: usize,
|
||||
envelope: &Envelope,
|
||||
error: PyErr,
|
||||
) -> PyResult<()> {
|
||||
let name = &self.subscribers[index].name;
|
||||
let event: &'static str = envelope.event.kind().into();
|
||||
match V1Python::Report.call(py, (name, event, error.value(py))) {
|
||||
Err(report_error) if is_cancellation(py, &report_error) => Err(report_error),
|
||||
|
|
@ -103,59 +106,33 @@ impl V1PythonLifecycle {
|
|||
}
|
||||
}
|
||||
|
||||
fn observer_at(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
index: usize,
|
||||
envelope: &Envelope,
|
||||
) -> Option<(String, bool, Py<PyAny>)> {
|
||||
let subscriber = &self.subscribers[index];
|
||||
if subscriber.schema != envelope.schema
|
||||
|| !subscriber.events.contains(&envelope.event.kind())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
subscriber.observe.as_ref().map(|handler| {
|
||||
let asynchronous = matches!(handler, Handler::Async(_));
|
||||
(
|
||||
subscriber.name.clone(),
|
||||
asynchronous,
|
||||
handler.object().clone_ref(py),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn observe(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
envelope: Envelope,
|
||||
from: usize,
|
||||
then: Then,
|
||||
then: Continuation,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
for index in from..self.subscribers.len() {
|
||||
let Some((name, asynchronous, handler)) = self.observer_at(py, index, &envelope) else {
|
||||
let Some(handler) = self.subscribers[index]
|
||||
.observer(envelope.event.kind())
|
||||
.map(|handler| handler.clone_ref(py))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let argument = match to_py(py, &envelope) {
|
||||
Ok(argument) => argument,
|
||||
Err(error) => {
|
||||
self.report(py, &name, &envelope, error)?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match handler.bind(py).call1((argument,)) {
|
||||
Ok(awaitable) if asynchronous => {
|
||||
let outcome = to_py(py, &envelope).and_then(|argument| handler.call(py, argument));
|
||||
match outcome {
|
||||
Ok(awaitable) if handler.is_async() => {
|
||||
self.pending = Some(Pending::Observe {
|
||||
envelope,
|
||||
next: index + 1,
|
||||
index,
|
||||
then,
|
||||
awaiting: name,
|
||||
});
|
||||
return Ok(LifecycleStep::Await(awaitable.unbind()));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
Err(error) => self.report(py, &name, &envelope, error)?,
|
||||
Err(error) => self.report(py, index, &envelope, error)?,
|
||||
}
|
||||
}
|
||||
Ok(then.into_step())
|
||||
|
|
@ -164,7 +141,7 @@ impl V1PythonLifecycle {
|
|||
fn apply_patch(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
name: &str,
|
||||
index: usize,
|
||||
wire: WireRequest,
|
||||
result: Py<PyAny>,
|
||||
) -> PyResult<WireRequest> {
|
||||
|
|
@ -173,25 +150,11 @@ impl V1PythonLifecycle {
|
|||
} else {
|
||||
from_py::<WirePatch>(result.bind(py))?
|
||||
};
|
||||
let name = &self.subscribers[index].name;
|
||||
apply(wire, patch)
|
||||
.map_err(|error| PyValueError::new_err(format!("callback {name}: {error}")))
|
||||
}
|
||||
|
||||
fn interceptor_at(&self, py: Python<'_>, index: usize) -> Option<(String, bool, Py<PyAny>)> {
|
||||
let subscriber = &self.subscribers[index];
|
||||
if subscriber.schema != SCHEMA_V1 {
|
||||
return None;
|
||||
}
|
||||
subscriber.intercept.as_ref().map(|handler| {
|
||||
let asynchronous = matches!(handler, Handler::Async(_));
|
||||
(
|
||||
subscriber.name.clone(),
|
||||
asynchronous,
|
||||
handler.object().clone_ref(py),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn intercept(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
|
|
@ -201,29 +164,28 @@ impl V1PythonLifecycle {
|
|||
) -> PyResult<LifecycleStep> {
|
||||
let mut current = wire;
|
||||
for index in from..self.subscribers.len() {
|
||||
let Some((name, asynchronous, handler)) = self.interceptor_at(py, index) else {
|
||||
let Some(handler) = self.subscribers[index]
|
||||
.intercept
|
||||
.as_ref()
|
||||
.map(|handler| handler.clone_ref(py))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let request = to_py(py, &RequestFacts::new(¤t, &context))?;
|
||||
match handler.bind(py).call1((request,)) {
|
||||
Ok(awaitable) if asynchronous => {
|
||||
self.pending = Some(Pending::Intercept {
|
||||
wire: current,
|
||||
context,
|
||||
next: index + 1,
|
||||
awaiting: name,
|
||||
});
|
||||
return Ok(LifecycleStep::Await(awaitable.unbind()));
|
||||
}
|
||||
Ok(result) => {
|
||||
current = Box::new(self.apply_patch(py, &name, *current, result.unbind())?);
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
let result = handler.call(py, request)?;
|
||||
if handler.is_async() {
|
||||
self.pending = Some(Pending::Intercept {
|
||||
wire: current,
|
||||
context,
|
||||
index,
|
||||
});
|
||||
return Ok(LifecycleStep::Await(result.unbind()));
|
||||
}
|
||||
current = Box::new(self.apply_patch(py, index, *current, result.unbind())?);
|
||||
}
|
||||
let envelope =
|
||||
self.envelope(Event::RequestSending(RequestFacts::new(¤t, &context)))?;
|
||||
self.observe(py, envelope, 0, Then::Wire(current))
|
||||
self.observe(py, envelope, 0, Continuation::Wire(current))
|
||||
}
|
||||
|
||||
fn metadata(arguments: &Bound<'_, PyDict>) -> PyResult<(Map<String, Value>, Vec<String>)> {
|
||||
|
|
@ -280,18 +242,18 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
_started_at: f64,
|
||||
start_time: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let call_id = Self::call_id(py, arguments.bind(py))?;
|
||||
let (metadata, metadata_dropped) = Self::metadata(arguments.bind(py))?;
|
||||
self.sequencer = Some(Sequencer::new(call_id, self.surface.call_type));
|
||||
let envelope = self.envelope(Event::CallStarted(CallFacts {
|
||||
start_time: self.start_time,
|
||||
start_time,
|
||||
asynchronous: self.asynchronous,
|
||||
metadata,
|
||||
metadata_dropped,
|
||||
}))?;
|
||||
self.observe(py, envelope, 0, Then::Arguments(arguments))
|
||||
self.observe(py, envelope, 0, Continuation::Arguments(arguments))
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
|
|
@ -303,35 +265,20 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
self.intercept(py, wire, context.clone(), 0)
|
||||
}
|
||||
|
||||
fn after_success(
|
||||
&mut self,
|
||||
_py: Python<'_>,
|
||||
response: Py<PyAny>,
|
||||
_timing: Timing,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
Ok(LifecycleStep::Response(response))
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
match event {
|
||||
LifecycleEvent::Started { start_time } => {
|
||||
self.start_time = start_time;
|
||||
Ok(LifecycleStep::Done)
|
||||
}
|
||||
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
|
||||
let envelope = self.envelope(Event::ResponseReceived {
|
||||
body: raw.body.clone(),
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Then::Done)
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
}
|
||||
LifecycleEvent::Succeeded { timing, response } => {
|
||||
let projection = V1Python::ProjectResponse.call(py, (response,));
|
||||
let (response, response_error) = match projection {
|
||||
Ok(value) => match from_py::<Value>(&value) {
|
||||
Ok(response) => (response, None),
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
Err(error) => (Value::Null, Some(error.to_string())),
|
||||
},
|
||||
let projected = V1Python::ProjectResponse
|
||||
.call(py, (response,))
|
||||
.and_then(|value| from_py::<Value>(&value));
|
||||
let (response, response_error) = match projected {
|
||||
Ok(response) => (response, None),
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
Err(error) => (Value::Null, Some(error.to_string())),
|
||||
};
|
||||
|
|
@ -341,7 +288,7 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
response,
|
||||
response_error,
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Then::Done)
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
}
|
||||
LifecycleEvent::Failed {
|
||||
timing,
|
||||
|
|
@ -355,7 +302,7 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
origin: origin.into(),
|
||||
error: facts,
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Then::Done)
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -365,35 +312,28 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn delivered(&mut self, _py: Python<'_>, _chunk: &Py<PyAny>) -> PyResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
|
||||
match self.pending.take().ok_or_else(missing_state)? {
|
||||
Pending::Observe {
|
||||
envelope,
|
||||
next,
|
||||
index,
|
||||
then,
|
||||
awaiting,
|
||||
} => {
|
||||
if let Err(error) = result {
|
||||
if is_cancellation(py, &error) {
|
||||
return Err(error);
|
||||
}
|
||||
self.report(py, &awaiting, &envelope, error)?;
|
||||
self.report(py, index, &envelope, error)?;
|
||||
}
|
||||
self.observe(py, envelope, next, then)
|
||||
self.observe(py, envelope, index + 1, then)
|
||||
}
|
||||
Pending::Intercept {
|
||||
wire,
|
||||
context,
|
||||
next,
|
||||
awaiting,
|
||||
index,
|
||||
} => {
|
||||
let value = result?;
|
||||
let patched = self.apply_patch(py, &awaiting, *wire, value)?;
|
||||
self.intercept(py, Box::new(patched), context, next)
|
||||
let patched = self.apply_patch(py, index, *wire, result?)?;
|
||||
self.intercept(py, Box::new(patched), context, index + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -409,12 +349,7 @@ impl PythonLifecycle for V1PythonLifecycle {
|
|||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
for subscriber in &self.subscribers {
|
||||
if let Some(handler) = &subscriber.observe {
|
||||
handler.traverse(visit)?;
|
||||
}
|
||||
if let Some(handler) = &subscriber.intercept {
|
||||
handler.traverse(visit)?;
|
||||
}
|
||||
subscriber.traverse(visit)?;
|
||||
}
|
||||
match &self.pending {
|
||||
Some(Pending::Observe { then, .. }) => then.traverse(visit),
|
||||
|
|
|
|||
|
|
@ -1,36 +1,25 @@
|
|||
use litellm_host::{machine::Machine, route::Route};
|
||||
use litellm_host_python::{RouteHost, run_call};
|
||||
//! What a route tells the v1 contract about itself, and the one constructor callers get.
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
use crate::adapter::V1PythonLifecycle;
|
||||
use crate::registry::snapshot;
|
||||
use crate::subscribers::snapshot;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct V1PythonSurface {
|
||||
pub call_type: &'static str,
|
||||
}
|
||||
|
||||
pub fn run_v1_python_call<H, M>(
|
||||
py: Python<'_>,
|
||||
surface: V1PythonSurface,
|
||||
kwargs: &Bound<'_, PyDict>,
|
||||
machine: M,
|
||||
route: H,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Py<PyAny>>
|
||||
where
|
||||
H: RouteHost + 'static,
|
||||
M: Machine<Route = H::Route, Complete = <H::Route as Route>::Response> + 'static,
|
||||
{
|
||||
let subscribers = snapshot(py, asynchronous)?;
|
||||
let arguments = kwargs.clone().unbind();
|
||||
run_call(
|
||||
py,
|
||||
machine,
|
||||
route,
|
||||
Box::new(V1PythonLifecycle::new(surface, subscribers, asynchronous)),
|
||||
arguments,
|
||||
asynchronous,
|
||||
)
|
||||
impl V1PythonLifecycle {
|
||||
/// The lifecycle for one call, over the subscribers registered right now; later
|
||||
/// registrations do not reach that call. `None` when nothing is subscribed, so an
|
||||
/// unobserved call builds no envelope at all.
|
||||
pub fn subscribed(
|
||||
py: Python<'_>,
|
||||
surface: V1PythonSurface,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Option<Self>> {
|
||||
let subscribers = snapshot(py, asynchronous)?;
|
||||
Ok((!subscribers.is_empty()).then(|| Self::new(surface, subscribers, asynchronous)))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
//! The Python adapter of the v1 callback contract: the per-call subscriber snapshot and
|
||||
//! one [`PythonLifecycle`](litellm_host_python::PythonLifecycle) that marshals
|
||||
//! `litellm-callbacks-v1` envelopes and patches to Python handlers. The contract itself
|
||||
//! lives in `litellm-callbacks-v1`.
|
||||
|
||||
mod adapter;
|
||||
mod call;
|
||||
mod python;
|
||||
mod registry;
|
||||
mod subscribers;
|
||||
|
||||
pub use call::{V1PythonSurface, run_v1_python_call};
|
||||
pub use adapter::V1PythonLifecycle;
|
||||
pub use call::V1PythonSurface;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../tests/lifecycle.rs"]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! The Python contract of this crate: every function of
|
||||
//! `litellm/rust_bridge/callbacks_v1_python.py` that Rust calls, by name.
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use strum::{IntoStaticStr, VariantArray};
|
||||
|
||||
|
|
@ -35,17 +38,19 @@ mod tests {
|
|||
use strum::VariantArray;
|
||||
|
||||
use super::V1Python;
|
||||
use crate::test_support::PYTHON_CONTRACT;
|
||||
|
||||
#[test]
|
||||
fn every_function_is_in_the_python_contract() {
|
||||
let contract: serde_json::Map<String, serde_json::Value> =
|
||||
serde_json::from_str(include_str!("../python_contract.json")).unwrap();
|
||||
serde_json::from_str(PYTHON_CONTRACT).unwrap();
|
||||
let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect();
|
||||
let called: BTreeSet<&str> = V1Python::VARIANTS
|
||||
let called: Vec<&str> = V1Python::VARIANTS
|
||||
.iter()
|
||||
.copied()
|
||||
.map(V1Python::name)
|
||||
.collect();
|
||||
assert_eq!(called, declared);
|
||||
assert_eq!(called.len(), declared.len(), "a function is listed twice");
|
||||
assert_eq!(called.into_iter().collect::<BTreeSet<_>>(), declared);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, SCHEMA_V1, select_handler};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::python::V1Python;
|
||||
|
||||
pub enum Handler {
|
||||
Sync(Py<PyAny>),
|
||||
Async(Py<PyAny>),
|
||||
}
|
||||
|
||||
pub struct Subscriber {
|
||||
pub name: String,
|
||||
pub schema: u32,
|
||||
pub events: BTreeSet<EventKind>,
|
||||
pub observe: Option<Handler>,
|
||||
pub intercept: Option<Handler>,
|
||||
}
|
||||
|
||||
fn selected_handler(
|
||||
subscriber: &Bound<'_, PyAny>,
|
||||
asynchronous: bool,
|
||||
sync_name: &str,
|
||||
async_name: &str,
|
||||
) -> PyResult<Option<Handler>> {
|
||||
let sync = subscriber.getattr(sync_name)?;
|
||||
let asynchronous_handler = subscriber.getattr(async_name)?;
|
||||
let selection = select_handler(
|
||||
if asynchronous {
|
||||
ExecutionMode::Async
|
||||
} else {
|
||||
ExecutionMode::Sync
|
||||
},
|
||||
!sync.is_none(),
|
||||
!asynchronous_handler.is_none(),
|
||||
);
|
||||
Ok(match selection {
|
||||
HandlerSelection::Sync => Some(Handler::Sync(sync.unbind())),
|
||||
HandlerSelection::Async => Some(Handler::Async(asynchronous_handler.unbind())),
|
||||
HandlerSelection::Skip => None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<Subscriber>> {
|
||||
V1Python::Snapshot
|
||||
.call(py, ())?
|
||||
.try_iter()?
|
||||
.map(|item| {
|
||||
let subscriber = item?;
|
||||
let schema = subscriber.getattr("schema")?.extract::<u32>()?;
|
||||
if schema != SCHEMA_V1 {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"unsupported callback schema: {schema}"
|
||||
)));
|
||||
}
|
||||
let event_names = subscriber
|
||||
.getattr("events")?
|
||||
.extract::<BTreeSet<String>>()?;
|
||||
let events = event_names
|
||||
.into_iter()
|
||||
.map(|name| {
|
||||
name.parse::<EventKind>().map_err(|_| {
|
||||
PyValueError::new_err(format!("unknown callback event: {name}"))
|
||||
})
|
||||
})
|
||||
.collect::<PyResult<BTreeSet<_>>>()?;
|
||||
Ok(Subscriber {
|
||||
name: subscriber.getattr("name")?.extract()?,
|
||||
schema,
|
||||
events,
|
||||
observe: selected_handler(&subscriber, asynchronous, "on_event", "async_on_event")?,
|
||||
intercept: selected_handler(
|
||||
&subscriber,
|
||||
asynchronous,
|
||||
"before_send",
|
||||
"async_before_send",
|
||||
)?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Handler {
|
||||
pub fn object(&self) -> &Py<PyAny> {
|
||||
match self {
|
||||
Self::Sync(value) | Self::Async(value) => value,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
visit.call(self.object())
|
||||
}
|
||||
}
|
||||
113
litellm-rust/crates/callbacks-v1-python/src/subscribers.rs
Normal file
113
litellm-rust/crates/callbacks-v1-python/src/subscribers.rs
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
//! The per-call subscriber snapshot: what Python's `register` recorded, read once, with
|
||||
//! one handler chosen per role for the call's execution mode. `register` has already
|
||||
//! rejected everything rejectable, so this only parses.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, select_handler};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::python::V1Python;
|
||||
|
||||
/// One callable of a subscriber, tagged with whether calling it returns an awaitable.
|
||||
pub enum Handler {
|
||||
Sync(Py<PyAny>),
|
||||
Async(Py<PyAny>),
|
||||
}
|
||||
|
||||
pub struct Subscriber {
|
||||
pub name: String,
|
||||
pub events: BTreeSet<EventKind>,
|
||||
/// `on_event` or `async_on_event`: cannot affect the call.
|
||||
pub observe: Option<Handler>,
|
||||
/// `before_send` or `async_before_send`: returns a patch; its failure fails the call.
|
||||
pub intercept: Option<Handler>,
|
||||
}
|
||||
|
||||
fn selected_handler(
|
||||
subscriber: &Bound<'_, PyAny>,
|
||||
mode: ExecutionMode,
|
||||
sync_name: &str,
|
||||
async_name: &str,
|
||||
) -> PyResult<Option<Handler>> {
|
||||
let sync = subscriber.getattr(sync_name)?;
|
||||
let asynchronous = subscriber.getattr(async_name)?;
|
||||
Ok(
|
||||
match select_handler(mode, !sync.is_none(), !asynchronous.is_none()) {
|
||||
HandlerSelection::Sync => Some(Handler::Sync(sync.unbind())),
|
||||
HandlerSelection::Async => Some(Handler::Async(asynchronous.unbind())),
|
||||
HandlerSelection::Skip => None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<Subscriber>> {
|
||||
let mode = if asynchronous {
|
||||
ExecutionMode::Async
|
||||
} else {
|
||||
ExecutionMode::Sync
|
||||
};
|
||||
V1Python::Snapshot
|
||||
.call(py, ())?
|
||||
.try_iter()?
|
||||
.map(|item| {
|
||||
let subscriber = item?;
|
||||
let events = subscriber
|
||||
.getattr("events")?
|
||||
.extract::<BTreeSet<String>>()?
|
||||
.into_iter()
|
||||
.map(|name| {
|
||||
name.parse::<EventKind>().map_err(|_| {
|
||||
PyValueError::new_err(format!("unknown callback event: {name}"))
|
||||
})
|
||||
})
|
||||
.collect::<PyResult<BTreeSet<_>>>()?;
|
||||
Ok(Subscriber {
|
||||
name: subscriber.getattr("name")?.extract()?,
|
||||
events,
|
||||
observe: selected_handler(&subscriber, mode, "on_event", "async_on_event")?,
|
||||
intercept: selected_handler(&subscriber, mode, "before_send", "async_before_send")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Subscriber {
|
||||
pub fn observer(&self, kind: EventKind) -> Option<&Handler> {
|
||||
self.observe
|
||||
.as_ref()
|
||||
.filter(|_| self.events.contains(&kind))
|
||||
}
|
||||
|
||||
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
for handler in self.observe.iter().chain(&self.intercept) {
|
||||
visit.call(handler.object())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler {
|
||||
fn object(&self) -> &Py<PyAny> {
|
||||
match self {
|
||||
Self::Sync(value) | Self::Async(value) => value,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_async(&self) -> bool {
|
||||
matches!(self, Self::Async(_))
|
||||
}
|
||||
|
||||
pub fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
match self {
|
||||
Self::Sync(value) => Self::Sync(value.clone_ref(py)),
|
||||
Self::Async(value) => Self::Async(value.clone_ref(py)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn call<'py>(&self, py: Python<'py>, argument: Py<PyAny>) -> PyResult<Bound<'py, PyAny>> {
|
||||
self.object().bind(py).call1((argument,))
|
||||
}
|
||||
}
|
||||
|
|
@ -10,9 +10,9 @@ use serde_json::{Value, json};
|
|||
|
||||
use crate::adapter::V1PythonLifecycle;
|
||||
use crate::call::V1PythonSurface;
|
||||
use crate::registry::{Handler, Subscriber};
|
||||
use crate::subscribers::{Handler, Subscriber};
|
||||
use crate::test_support::{local, namespace};
|
||||
use litellm_callbacks_v1::{EventKind, SCHEMA_V1};
|
||||
use litellm_callbacks_v1::EventKind;
|
||||
|
||||
const TIMING: Timing = Timing {
|
||||
start_time: 10.0,
|
||||
|
|
@ -38,7 +38,6 @@ fn subscriber(
|
|||
};
|
||||
Subscriber {
|
||||
name: name.to_string(),
|
||||
schema: SCHEMA_V1,
|
||||
events: events.iter().copied().collect::<BTreeSet<_>>(),
|
||||
observe: handler(observe),
|
||||
intercept: handler(intercept),
|
||||
|
|
@ -53,15 +52,6 @@ fn lifecycle(subscribers: Vec<Subscriber>, asynchronous: bool) -> V1PythonLifecy
|
|||
)
|
||||
}
|
||||
|
||||
fn started(adapter: &mut V1PythonLifecycle, py: Python<'_>) {
|
||||
assert!(matches!(
|
||||
adapter
|
||||
.emit(py, LifecycleEvent::Started { start_time: 10.0 })
|
||||
.unwrap(),
|
||||
LifecycleStep::Done
|
||||
));
|
||||
}
|
||||
|
||||
fn context(secret: &str) -> RequestContext {
|
||||
RequestContext {
|
||||
model: "model".to_string(),
|
||||
|
|
@ -111,7 +101,6 @@ response = {'id': 'response'}
|
|||
None,
|
||||
);
|
||||
let mut adapter = lifecycle(vec![observer], false);
|
||||
started(&mut adapter, py);
|
||||
let arguments = PyDict::new(py);
|
||||
arguments.set_item("litellm_call_id", "call-123").unwrap();
|
||||
let LifecycleStep::Arguments(returned) =
|
||||
|
|
@ -202,7 +191,6 @@ def record(event):
|
|||
subscriber(&locals, "recorder", &event, Some(("record", false)), None),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
started(&mut adapter, py);
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
|
||||
let LifecycleStep::Wire(sent) = adapter
|
||||
.before_send(py, Box::new(wire("secret")), &context("secret"))
|
||||
|
|
@ -248,7 +236,6 @@ def record(event):
|
|||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
started(&mut adapter, py);
|
||||
assert!(matches!(
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(),
|
||||
LifecycleStep::Arguments(_)
|
||||
|
|
@ -299,7 +286,6 @@ def record(event):
|
|||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
started(&mut adapter, py);
|
||||
let error = match adapter.begin(py, PyDict::new(py).unbind(), 10.0) {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("cancellation was swallowed"),
|
||||
|
|
@ -337,7 +323,6 @@ def second(event):
|
|||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, true);
|
||||
started(&mut adapter, py);
|
||||
let arguments = PyDict::new(py).unbind();
|
||||
assert!(matches!(
|
||||
adapter.begin(py, arguments.clone_ref(py), 10.0).unwrap(),
|
||||
|
|
@ -387,7 +372,6 @@ def observe(event):
|
|||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
started(&mut adapter, py);
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
|
||||
let LifecycleStep::Wire(sent) = adapter
|
||||
.before_send(py, Box::new(wire("secret")), &context("secret"))
|
||||
|
|
@ -440,7 +424,6 @@ def observe(event):
|
|||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
started(&mut adapter, py);
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
|
||||
let error = match adapter.before_send(py, Box::new(wire("secret")), &context("secret")) {
|
||||
Err(error) => error,
|
||||
|
|
@ -474,7 +457,6 @@ response = Broken()
|
|||
None,
|
||||
);
|
||||
let mut adapter = lifecycle(vec![observer], false);
|
||||
started(&mut adapter, py);
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
|
||||
adapter.opened(py).unwrap();
|
||||
let response = local(&locals, "response").unbind();
|
||||
|
|
|
|||
|
|
@ -1,8 +1,17 @@
|
|||
- Target invariants, not completion claims
|
||||
- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction and validated wire patches
|
||||
- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction, handler selection and validated wire patches
|
||||
- This crate may depend on `litellm-host`, but never on `litellm-host-python`, PyO3 or a runtime registry
|
||||
- `golden/v1` pins every serialized envelope shape
|
||||
- `golden/v1` pins every serialized envelope shape; every `EventKind` has a golden file by construction, and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are tested against those files
|
||||
- Terms (shared ones are in `host-python/AGENTS.md`)
|
||||
- Subscriber: one registered consumer of the contract: a name, a schema, the event kinds it wants and its handlers
|
||||
- Handler: one callable of a subscriber; an observer receives envelopes and cannot affect the call; an interceptor receives `RequestFacts` and returns a patch
|
||||
- Handler selection (`select_handler`): an async call prefers the async handler and falls back to the sync one; a sync call never selects an async handler
|
||||
- Event: one thing that happened in a call; `EventKind` is its dotted name (`call.started`, `request.sending`, `response.received`, `call.succeeded`, `call.failed`), the serde tag and the subscription name at once
|
||||
- Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; `Sequencer` numbers a call's envelopes from zero
|
||||
- Facts (`CallFacts`, `RequestFacts`, `TimingFacts`, `ErrorFacts`): a redacted, serializable projection of runtime state; never a live object
|
||||
- Patch (`WirePatch`): an interceptor's edit of the wire: header `remove`, then header `set`, optional `body` replacement; `apply` validates it
|
||||
- Redaction: credential headers by name (`CREDENTIAL_HEADERS`) and optional params by the route's secret fields become `[REDACTED]`; shape never changes
|
||||
- No runtime object of a call reaches a callback contract value
|
||||
- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction
|
||||
- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction; its fields are private so no other crate can build one
|
||||
- A patch cannot change the URL or set or remove a credential header
|
||||
- Schema v1 is additive-only; a rename, removal or retype requires a new contract version
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
//! The v1 envelope: what a subscriber receives. An [`Envelope`] is the schema version, the
|
||||
//! call's identity, a sequence number and one [`Event`]; every `*Facts` type is a
|
||||
//! redacted, serializable projection that never holds a runtime object of the call.
|
||||
|
||||
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
|
@ -75,14 +79,16 @@ pub struct CallFacts {
|
|||
pub metadata_dropped: Vec<String>,
|
||||
}
|
||||
|
||||
/// The request as a subscriber may see it. The fields are private so that
|
||||
/// [`RequestFacts::new`], which redacts, is the only way to build one.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct RequestFacts {
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub optional_params: Value,
|
||||
pub url: String,
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub body: Value,
|
||||
model: String,
|
||||
custom_llm_provider: String,
|
||||
optional_params: Value,
|
||||
url: String,
|
||||
headers: Vec<(String, String)>,
|
||||
body: Value,
|
||||
}
|
||||
|
||||
impl RequestFacts {
|
||||
|
|
@ -243,21 +249,26 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case(EventKind::CallStarted, 0, include_str!("../golden/v1/call.started.json"))]
|
||||
#[case(EventKind::RequestSending, 1, include_str!("../golden/v1/request.sending.json"))]
|
||||
#[case(EventKind::ResponseReceived, 2, include_str!("../golden/v1/response.received.json"))]
|
||||
#[case(EventKind::CallSucceeded, 3, include_str!("../golden/v1/call.succeeded.json"))]
|
||||
#[case(EventKind::CallFailed, 4, include_str!("../golden/v1/call.failed.json"))]
|
||||
fn serialized_v1_envelopes_match_golden_contract(
|
||||
#[case] kind: EventKind,
|
||||
#[case] seq: u32,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
serde_json::to_value(golden(kind, seq)).unwrap(),
|
||||
serde_json::from_str::<Value>(expected).unwrap()
|
||||
);
|
||||
/// Exhaustive on purpose: a new event kind does not compile without a golden file,
|
||||
/// and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are pinned to the golden files.
|
||||
fn golden_json(kind: EventKind) -> &'static str {
|
||||
match kind {
|
||||
EventKind::CallStarted => include_str!("../golden/v1/call.started.json"),
|
||||
EventKind::RequestSending => include_str!("../golden/v1/request.sending.json"),
|
||||
EventKind::ResponseReceived => include_str!("../golden/v1/response.received.json"),
|
||||
EventKind::CallSucceeded => include_str!("../golden/v1/call.succeeded.json"),
|
||||
EventKind::CallFailed => include_str!("../golden/v1/call.failed.json"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serialized_v1_envelopes_match_golden_contract() {
|
||||
for (seq, &kind) in EventKind::VARIANTS.iter().enumerate() {
|
||||
assert_eq!(
|
||||
serde_json::to_value(golden(kind, seq as u32)).unwrap(),
|
||||
serde_json::from_str::<Value>(golden_json(kind)).unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
//! The runtime-neutral v1 callback contract: envelopes and their sequencing, credential
|
||||
//! redaction, handler selection and validated wire patches. No PyO3 and no registry; a
|
||||
//! language host marshals these values to its own subscribers.
|
||||
|
||||
mod envelope;
|
||||
mod patch;
|
||||
mod policy;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
//! The v1 patch: the only way an interceptor changes a request. A [`WirePatch`] removes
|
||||
//! and sets headers and may replace the body; it cannot name the URL, and [`apply`]
|
||||
//! rejects any credential header.
|
||||
|
||||
use litellm_host::event::WireRequest;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! Which of a subscriber's handlers a call uses: an async call prefers the async handler
|
||||
//! and falls back to the sync one; a sync call never selects an async handler.
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ExecutionMode {
|
||||
Sync,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! The contract's credential redaction: credential headers by name, optional params by
|
||||
//! the route's secret fields. Redaction replaces values and never changes shape.
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub const REDACTED: &str = "[REDACTED]";
|
||||
|
|
|
|||
|
|
@ -5,6 +5,26 @@
|
|||
- `RouteHost::invoke` receives the keyword view the adapter's `begin` returned, not the caller's dict; a route host that projects from it inherits that adapter's rewrites (for the legacy adapter: setup, deployment hooks, credential inheritance)
|
||||
- A native failure, including one a host op returns as `InvokeError::Native`, is classified exactly once through the route's `classify`; a Python exception raised inside the call, and a failure in `begin` or `after_success`, is raised as is
|
||||
- A failing `classify` is raised with the native error's text as its `__context__`, never swallowed
|
||||
- Terms; the callback crates build on these and never redefine them
|
||||
- Machine: the native call; it never calls out, it yields a `HostOp` (`Route`, `BeforeSend`, `Emit`, `Open`, `Deliver`) and waits for the `HostResult`
|
||||
- Host: whatever answers those ops; the Python host is two capabilities the driver composes: `RouteHost` (route ops, argument and response projection) and `PythonLifecycle` (callbacks)
|
||||
- Host language = callback language: a lifecycle speaks `Py<...>` and `PyErr` directly; another host language gets its own crate of this shape and reuses only `litellm-host` and `litellm-callbacks-v1`
|
||||
- Driver: runs one call; owns the machine, one route host and exactly one lifecycle
|
||||
- Lifecycle: one implementation of `PythonLifecycle`, one callback contract's view of a call; chain: several lifecycles behind one, in a fixed order
|
||||
- Step: one driver-to-lifecycle method call, answered with a `LifecycleStep`; a threading step (`begin`, `before_send`, `after_success`) returns the value the call continues with and defaults to returning what it was given; an observing step (`emit`, `opened`, `delivered`) returns nothing to the call
|
||||
- Suspension: a step answered with `Await`; the driver awaits it in the caller's task and continues the same step through `resume`
|
||||
- Terminal event: `Succeeded` or `Failed`, exactly once, last
|
||||
- Cancellation: a raised `BaseException` that is not an `Exception`; `is_cancellation` is the only definition; never reported, never swallowed, never followed by dispatch
|
||||
- Keyword view: the kwargs dict as rewritten so far; `begin` threads it and the route projects from the result
|
||||
- Wire: the `WireRequest` (URL, headers, body) about to be sent; `before_send` threads it
|
||||
- Surface: the static facts a route declares about itself to a contract (`call_type`, ...)
|
||||
- Python contract: a crate's `python.rs` enum, its `python_contract.json` and the one `litellm/rust_bridge/*.py` module behind them, tested from both sides
|
||||
- `PythonLifecycle` is the whole callback boundary: the driver holds exactly one and never learns how many contracts are behind it
|
||||
- `LifecycleChain` composes several in a fixed order: `begin`, `before_send` and `after_success` thread their value forward, events reach each in turn, any error ends the step with no further dispatch
|
||||
- `begin` carries the call's start time and is the first thing a lifecycle hears of a call; there is no separate started event
|
||||
- A lifecycle whose `begin` was never reached receives nothing of that call; the terminal event goes to the ones `begin` reached
|
||||
- A suspension resumes the lifecycle that suspended, then continues the same step with the ones after it; a retained failure is the raised exception object itself
|
||||
- Ordering policy (which contract runs first) belongs to whoever builds the chain, never to this crate
|
||||
- Use standard PyO3 ownership and conversion APIs
|
||||
- Prefer `Bound<'py, T>` for attached operations/results, `Py<T>` for retention; binding/unbinding does not copy payloads
|
||||
- Use `pythonize` for selected Serde data, never a JSON-text round trip; share conversion with `Pythonized<T>`
|
||||
|
|
|
|||
|
|
@ -1,15 +1,22 @@
|
|||
use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest};
|
||||
use litellm_host::route::Route;
|
||||
use pyo3::exceptions::PyRuntimeError;
|
||||
use pyo3::exceptions::{PyException, PyRuntimeError};
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
/// Whether an error ends the call outright: anything that is not a `PyException`, such
|
||||
/// as `CancelledError` or `KeyboardInterrupt`. A cancellation is never reported, never
|
||||
/// swallowed and never followed by further dispatch.
|
||||
pub fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool {
|
||||
!error.is_instance_of::<PyException>(py)
|
||||
}
|
||||
|
||||
pub fn missing_state() -> PyErr {
|
||||
PyRuntimeError::new_err("missing native call state")
|
||||
}
|
||||
|
||||
/// What an adapter step produced: either the value the driver asked for, or a Python
|
||||
/// What a lifecycle step produced: either the value the driver asked for, or a Python
|
||||
/// awaitable the driver hands back to the caller's task before asking again.
|
||||
pub enum LifecycleStep {
|
||||
Await(Py<PyAny>),
|
||||
|
|
@ -19,12 +26,10 @@ pub enum LifecycleStep {
|
|||
Done,
|
||||
}
|
||||
|
||||
/// What a lifecycle observes: the driver's start, the machine's own events, and one
|
||||
/// terminal event carrying the public value the caller receives.
|
||||
/// What a lifecycle observes: the machine's own events, and one terminal event
|
||||
/// carrying the public value the caller receives.
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum LifecycleEvent<'a> {
|
||||
Started {
|
||||
start_time: f64,
|
||||
},
|
||||
Machine(&'a MachineEvent),
|
||||
Succeeded {
|
||||
timing: Timing,
|
||||
|
|
@ -37,46 +42,73 @@ pub enum LifecycleEvent<'a> {
|
|||
},
|
||||
}
|
||||
|
||||
/// One consumer of a call's lifecycle on the Python side. The driver calls the steps in
|
||||
/// order: `begin` before the machine starts, `before_send` and `emit` while it runs,
|
||||
/// `after_success` and one terminal `emit` after it completes. Whenever a step returns
|
||||
/// [`LifecycleStep::Await`], the driver awaits it in the caller's task and continues the
|
||||
/// same step through `resume`.
|
||||
/// One consumer of a call's lifecycle on the Python side: the callback capability of the
|
||||
/// Python host, next to [`RouteHost`], the route capability.
|
||||
///
|
||||
/// The steps come in two kinds. A threading step receives a value and returns the value
|
||||
/// the call continues with, so a lifecycle may rewrite it; each defaults to returning
|
||||
/// what it was given. An observing step receives an event or a notice and returns
|
||||
/// nothing to the call. Two steps answer an operation the machine raised
|
||||
/// ([`HostOp`](litellm_host::host::HostOp)); the rest the driver originates around it:
|
||||
///
|
||||
/// | step | kind | raised by |
|
||||
/// |---|---|---|
|
||||
/// | `begin` | threads the keyword view | driver, before the machine starts |
|
||||
/// | `before_send` | threads the wire request | machine, `HostOp::BeforeSend` |
|
||||
/// | `emit(Machine)` | observes | machine, `HostOp::Emit` |
|
||||
/// | `opened`, `delivered` | observe | driver, answering `HostOp::Open` and `Deliver` |
|
||||
/// | `after_success` | threads the public response | driver, after the machine completes |
|
||||
/// | `emit(Succeeded \| Failed)` | observes | driver, exactly once, last |
|
||||
///
|
||||
/// Whenever a step returns [`LifecycleStep::Await`], the driver awaits it in the caller's
|
||||
/// task and continues the same step through `resume`.
|
||||
///
|
||||
/// A step that fails with an ordinary exception fails the call with that exception,
|
||||
/// except on a terminal event, where the adapter is expected to report and swallow its
|
||||
/// own errors. An exception that is not a `PyException`, such as a cancellation, ends
|
||||
/// the call without further dispatch.
|
||||
/// except on a terminal event, where the lifecycle is expected to report and swallow its
|
||||
/// own errors. A cancellation ([`is_cancellation`]) ends the call without further
|
||||
/// dispatch.
|
||||
pub trait PythonLifecycle: Send + Sync {
|
||||
/// `start_time` is the call's start in epoch seconds, the same value the terminal
|
||||
/// event's `Timing` carries.
|
||||
fn begin(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
_py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
started_at: f64,
|
||||
) -> PyResult<LifecycleStep>;
|
||||
_start_time: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
Ok(LifecycleStep::Arguments(arguments))
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
_py: Python<'_>,
|
||||
wire: Box<WireRequest>,
|
||||
context: &RequestContext,
|
||||
) -> PyResult<LifecycleStep>;
|
||||
_context: &RequestContext,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
Ok(LifecycleStep::Wire(wire))
|
||||
}
|
||||
|
||||
fn after_success(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
_py: Python<'_>,
|
||||
response: Py<PyAny>,
|
||||
timing: Timing,
|
||||
) -> PyResult<LifecycleStep>;
|
||||
_timing: Timing,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
Ok(LifecycleStep::Response(response))
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep>;
|
||||
|
||||
/// The call streams and its stream was handed to the caller. The caller is not
|
||||
/// inside an await here, so this step and `delivered` cannot suspend.
|
||||
fn opened(&mut self, py: Python<'_>) -> PyResult<()>;
|
||||
fn opened(&mut self, _py: Python<'_>) -> PyResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One chunk of an open stream is about to reach the caller.
|
||||
fn delivered(&mut self, py: Python<'_>, chunk: &Py<PyAny>) -> PyResult<()>;
|
||||
fn delivered(&mut self, _py: Python<'_>, _chunk: &Py<PyAny>) -> PyResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep>;
|
||||
|
||||
|
|
|
|||
560
litellm-rust/crates/host-python/src/chain.rs
Normal file
560
litellm-rust/crates/host-python/src/chain.rs
Normal file
|
|
@ -0,0 +1,560 @@
|
|||
//! Several lifecycles consuming one call as one [`PythonLifecycle`]. The driver keeps a
|
||||
//! single consumer; which contracts a call runs under is decided where the chain is
|
||||
//! built, never by the driver and never by a route.
|
||||
|
||||
use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest};
|
||||
use pyo3::exceptions::PyBaseException;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
use crate::adapter::{LifecycleEvent, LifecycleStep, PythonLifecycle, missing_state};
|
||||
|
||||
/// An event kept across a suspension, so the lifecycles after the suspended one still
|
||||
/// receive it. A failure keeps the raised exception object itself.
|
||||
enum RetainedEvent {
|
||||
Machine(MachineEvent),
|
||||
Succeeded {
|
||||
timing: Timing,
|
||||
response: Py<PyAny>,
|
||||
},
|
||||
Failed {
|
||||
timing: Timing,
|
||||
origin: FailureOrigin,
|
||||
error: Py<PyBaseException>,
|
||||
},
|
||||
}
|
||||
|
||||
/// The step a suspended lifecycle was answering, with what the later lifecycles need to
|
||||
/// answer it too. The value being threaded stays with the suspended lifecycle.
|
||||
enum Step {
|
||||
Begin { start_time: f64 },
|
||||
BeforeSend { context: RequestContext },
|
||||
AfterSuccess { timing: Timing },
|
||||
Emit(RetainedEvent),
|
||||
}
|
||||
|
||||
struct Suspended {
|
||||
index: usize,
|
||||
step: Step,
|
||||
}
|
||||
|
||||
/// Runs every step through its lifecycles in the order they were given.
|
||||
///
|
||||
/// `begin`, `before_send` and `after_success` thread their value: each lifecycle
|
||||
/// receives what the one before it returned, so a later lifecycle sees an earlier one's
|
||||
/// rewrites and never the reverse. Events reach each lifecycle in turn. An error from
|
||||
/// any lifecycle ends the step at once, with no dispatch to the ones after it.
|
||||
///
|
||||
/// A lifecycle observes a call only once its `begin` was reached: when an earlier
|
||||
/// lifecycle fails the call in `begin`, the later ones never learn of the call, and the
|
||||
/// terminal event goes to those that did.
|
||||
pub struct LifecycleChain {
|
||||
lifecycles: Vec<Box<dyn PythonLifecycle>>,
|
||||
/// How many lifecycles `begin` has reached.
|
||||
entered: usize,
|
||||
pending: Option<Suspended>,
|
||||
}
|
||||
|
||||
impl RetainedEvent {
|
||||
fn retain(py: Python<'_>, event: LifecycleEvent<'_>) -> Self {
|
||||
match event {
|
||||
LifecycleEvent::Machine(event) => Self::Machine(event.clone()),
|
||||
LifecycleEvent::Succeeded { timing, response } => Self::Succeeded {
|
||||
timing,
|
||||
response: response.clone_ref(py),
|
||||
},
|
||||
LifecycleEvent::Failed {
|
||||
timing,
|
||||
origin,
|
||||
error,
|
||||
} => Self::Failed {
|
||||
timing,
|
||||
origin,
|
||||
error: error.clone_ref(py).into_value(py),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn emit(&self, py: Python<'_>, lifecycle: &mut dyn PythonLifecycle) -> PyResult<LifecycleStep> {
|
||||
match self {
|
||||
Self::Machine(event) => lifecycle.emit(py, LifecycleEvent::Machine(event)),
|
||||
Self::Succeeded { timing, response } => lifecycle.emit(
|
||||
py,
|
||||
LifecycleEvent::Succeeded {
|
||||
timing: *timing,
|
||||
response,
|
||||
},
|
||||
),
|
||||
Self::Failed {
|
||||
timing,
|
||||
origin,
|
||||
error,
|
||||
} => {
|
||||
let error = PyErr::from_value(error.bind(py).clone().into_any());
|
||||
lifecycle.emit(
|
||||
py,
|
||||
LifecycleEvent::Failed {
|
||||
timing: *timing,
|
||||
origin: *origin,
|
||||
error: &error,
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
match self {
|
||||
Self::Succeeded { response, .. } => visit.call(response),
|
||||
Self::Failed { error, .. } => visit.call(error),
|
||||
Self::Machine(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LifecycleChain {
|
||||
pub fn new(lifecycles: Vec<Box<dyn PythonLifecycle>>) -> Self {
|
||||
Self {
|
||||
lifecycles,
|
||||
entered: 0,
|
||||
pending: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// How many lifecycles a step reaches: `begin` tries all of them, every later step
|
||||
/// reaches the ones `begin` reached.
|
||||
fn audience(&self, step: &Step) -> usize {
|
||||
match step {
|
||||
Step::Begin { .. } => self.lifecycles.len(),
|
||||
Step::BeforeSend { .. } | Step::AfterSuccess { .. } | Step::Emit(_) => self.entered,
|
||||
}
|
||||
}
|
||||
|
||||
fn call(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
index: usize,
|
||||
step: &Step,
|
||||
carried: LifecycleStep,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let lifecycle = self.lifecycles[index].as_mut();
|
||||
match (step, carried) {
|
||||
(Step::Begin { start_time }, LifecycleStep::Arguments(arguments)) => {
|
||||
self.entered = index + 1;
|
||||
lifecycle.begin(py, arguments, *start_time)
|
||||
}
|
||||
(Step::BeforeSend { context }, LifecycleStep::Wire(wire)) => {
|
||||
lifecycle.before_send(py, wire, context)
|
||||
}
|
||||
(Step::AfterSuccess { timing }, LifecycleStep::Response(response)) => {
|
||||
lifecycle.after_success(py, response, *timing)
|
||||
}
|
||||
(Step::Emit(event), LifecycleStep::Done) => event.emit(py, lifecycle),
|
||||
_ => Err(missing_state()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Continues `step` from the lifecycle at `next`, handing each one what the previous
|
||||
/// produced, until one suspends or the last has answered.
|
||||
fn proceed(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
mut next: usize,
|
||||
mut carried: LifecycleStep,
|
||||
step: Step,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
while next < self.audience(&step) {
|
||||
carried = match self.call(py, next, &step, carried)? {
|
||||
LifecycleStep::Await(awaitable) => {
|
||||
self.pending = Some(Suspended { index: next, step });
|
||||
return Ok(LifecycleStep::Await(awaitable));
|
||||
}
|
||||
produced => produced,
|
||||
};
|
||||
next += 1;
|
||||
}
|
||||
Ok(carried)
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonLifecycle for LifecycleChain {
|
||||
fn begin(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
start_time: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.proceed(
|
||||
py,
|
||||
0,
|
||||
LifecycleStep::Arguments(arguments),
|
||||
Step::Begin { start_time },
|
||||
)
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
wire: Box<WireRequest>,
|
||||
context: &RequestContext,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.proceed(
|
||||
py,
|
||||
0,
|
||||
LifecycleStep::Wire(wire),
|
||||
Step::BeforeSend {
|
||||
context: context.clone(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn after_success(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
response: Py<PyAny>,
|
||||
timing: Timing,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.proceed(
|
||||
py,
|
||||
0,
|
||||
LifecycleStep::Response(response),
|
||||
Step::AfterSuccess { timing },
|
||||
)
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
for index in 0..self.entered {
|
||||
match self.lifecycles[index].emit(py, event)? {
|
||||
LifecycleStep::Await(awaitable) => {
|
||||
self.pending = Some(Suspended {
|
||||
index,
|
||||
step: Step::Emit(RetainedEvent::retain(py, event)),
|
||||
});
|
||||
return Ok(LifecycleStep::Await(awaitable));
|
||||
}
|
||||
LifecycleStep::Done => {}
|
||||
_ => return Err(missing_state()),
|
||||
}
|
||||
}
|
||||
Ok(LifecycleStep::Done)
|
||||
}
|
||||
|
||||
fn opened(&mut self, py: Python<'_>) -> PyResult<()> {
|
||||
self.lifecycles[..self.entered]
|
||||
.iter_mut()
|
||||
.try_for_each(|lifecycle| lifecycle.opened(py))
|
||||
}
|
||||
|
||||
fn delivered(&mut self, py: Python<'_>, chunk: &Py<PyAny>) -> PyResult<()> {
|
||||
self.lifecycles[..self.entered]
|
||||
.iter_mut()
|
||||
.try_for_each(|lifecycle| lifecycle.delivered(py, chunk))
|
||||
}
|
||||
|
||||
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
|
||||
let Suspended { index, step } = self.pending.take().ok_or_else(missing_state)?;
|
||||
match self.lifecycles[index].resume(py, result)? {
|
||||
LifecycleStep::Await(awaitable) => {
|
||||
self.pending = Some(Suspended { index, step });
|
||||
Ok(LifecycleStep::Await(awaitable))
|
||||
}
|
||||
produced => self.proceed(py, index + 1, produced, step),
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&mut self, py: Python<'_>) {
|
||||
self.pending = None;
|
||||
for lifecycle in &mut self.lifecycles {
|
||||
lifecycle.close(py);
|
||||
}
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
for lifecycle in &self.lifecycles {
|
||||
lifecycle.traverse(visit)?;
|
||||
}
|
||||
match &self.pending {
|
||||
Some(Suspended {
|
||||
step: Step::Emit(event),
|
||||
..
|
||||
}) => event.traverse(visit),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use pyo3::exceptions::PyValueError;
|
||||
|
||||
use super::*;
|
||||
|
||||
type Log = Arc<Mutex<Vec<String>>>;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Script {
|
||||
Plain,
|
||||
FailBegin,
|
||||
SuspendBegin,
|
||||
SuspendTerminal,
|
||||
}
|
||||
|
||||
struct Recorder {
|
||||
name: &'static str,
|
||||
script: Script,
|
||||
log: Log,
|
||||
suspended: Option<LifecycleStep>,
|
||||
}
|
||||
|
||||
impl Recorder {
|
||||
fn record(&self, entry: &str) {
|
||||
self.log
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(format!("{}.{entry}", self.name));
|
||||
}
|
||||
|
||||
fn suspend(&mut self, py: Python<'_>, then: LifecycleStep) -> PyResult<LifecycleStep> {
|
||||
self.suspended = Some(then);
|
||||
Ok(LifecycleStep::Await(py.None()))
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonLifecycle for Recorder {
|
||||
fn begin(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
_: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let seen: Vec<String> = arguments
|
||||
.bind(py)
|
||||
.keys()
|
||||
.iter()
|
||||
.map(|key| key.extract().unwrap())
|
||||
.collect();
|
||||
self.record(&format!("begin{seen:?}"));
|
||||
arguments.bind(py).set_item(self.name, true)?;
|
||||
match self.script {
|
||||
Script::FailBegin => Err(PyValueError::new_err("begin failed")),
|
||||
Script::SuspendBegin => self.suspend(py, LifecycleStep::Arguments(arguments)),
|
||||
_ => Ok(LifecycleStep::Arguments(arguments)),
|
||||
}
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
&mut self,
|
||||
_: Python<'_>,
|
||||
mut wire: Box<WireRequest>,
|
||||
_: &RequestContext,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.record(&format!("before_send:{}", wire.url));
|
||||
wire.url.push_str(self.name);
|
||||
Ok(LifecycleStep::Wire(wire))
|
||||
}
|
||||
|
||||
fn after_success(
|
||||
&mut self,
|
||||
_: Python<'_>,
|
||||
response: Py<PyAny>,
|
||||
_: Timing,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.record("after_success");
|
||||
Ok(LifecycleStep::Response(response))
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
let terminal = match event {
|
||||
LifecycleEvent::Machine(_) => {
|
||||
self.record("machine");
|
||||
false
|
||||
}
|
||||
LifecycleEvent::Succeeded { response, .. } => {
|
||||
self.record(&format!("succeeded:{}", response.bind(py)));
|
||||
true
|
||||
}
|
||||
LifecycleEvent::Failed { error, .. } => {
|
||||
self.record(&format!("failed:{}", error.value(py)));
|
||||
true
|
||||
}
|
||||
};
|
||||
if terminal && self.script == Script::SuspendTerminal {
|
||||
return self.suspend(py, LifecycleStep::Done);
|
||||
}
|
||||
Ok(LifecycleStep::Done)
|
||||
}
|
||||
|
||||
fn opened(&mut self, _: Python<'_>) -> PyResult<()> {
|
||||
self.record("opened");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delivered(&mut self, _: Python<'_>, _: &Py<PyAny>) -> PyResult<()> {
|
||||
self.record("delivered");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resume(&mut self, _: Python<'_>, _: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
|
||||
self.record("resume");
|
||||
self.suspended.take().ok_or_else(missing_state)
|
||||
}
|
||||
|
||||
fn close(&mut self, _: Python<'_>) {
|
||||
self.record("close");
|
||||
}
|
||||
|
||||
fn traverse(&self, _: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn chain(scripts: [(&'static str, Script); 2]) -> (LifecycleChain, Log) {
|
||||
let log = Log::default();
|
||||
let lifecycles = scripts
|
||||
.into_iter()
|
||||
.map(|(name, script)| {
|
||||
Box::new(Recorder {
|
||||
name,
|
||||
script,
|
||||
log: Arc::clone(&log),
|
||||
suspended: None,
|
||||
}) as Box<dyn PythonLifecycle>
|
||||
})
|
||||
.collect();
|
||||
(LifecycleChain::new(lifecycles), log)
|
||||
}
|
||||
|
||||
fn entries(log: &Log) -> Vec<String> {
|
||||
std::mem::take(&mut *log.lock().unwrap())
|
||||
}
|
||||
|
||||
fn timing() -> Timing {
|
||||
Timing {
|
||||
start_time: 1.0,
|
||||
end_time: 2.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn context() -> RequestContext {
|
||||
RequestContext {
|
||||
model: "model".to_string(),
|
||||
custom_llm_provider: "provider".to_string(),
|
||||
optional_params: serde_json::Value::Null,
|
||||
secret_fields: Vec::new(),
|
||||
api_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_later_lifecycle_receives_what_an_earlier_one_returned() {
|
||||
crate::initialize_python();
|
||||
Python::attach(|py| {
|
||||
let (mut chain, log) = chain([("first", Script::Plain), ("second", Script::Plain)]);
|
||||
let arguments = PyDict::new(py).unbind();
|
||||
let step = chain.begin(py, arguments.clone_ref(py), 1.0).unwrap();
|
||||
assert!(matches!(step, LifecycleStep::Arguments(returned) if returned.is(&arguments)));
|
||||
let wire = Box::new(WireRequest {
|
||||
url: "url:".to_string(),
|
||||
headers: Vec::new(),
|
||||
body: serde_json::Value::Null,
|
||||
});
|
||||
let step = chain.before_send(py, wire, &context()).unwrap();
|
||||
assert!(matches!(step, LifecycleStep::Wire(wire) if wire.url == "url:firstsecond"));
|
||||
assert_eq!(
|
||||
entries(&log),
|
||||
[
|
||||
"first.begin[]",
|
||||
"second.begin[\"first\"]",
|
||||
"first.before_send:url:",
|
||||
"second.before_send:url:first",
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lifecycle_begin_never_reached_observes_nothing_of_the_call() {
|
||||
crate::initialize_python();
|
||||
Python::attach(|py| {
|
||||
let (mut chain, log) = chain([("first", Script::FailBegin), ("second", Script::Plain)]);
|
||||
let error = chain
|
||||
.begin(py, PyDict::new(py).unbind(), 1.0)
|
||||
.err()
|
||||
.expect("begin fails");
|
||||
let failed = LifecycleEvent::Failed {
|
||||
timing: timing(),
|
||||
origin: FailureOrigin::Host,
|
||||
error: &error,
|
||||
};
|
||||
assert!(matches!(
|
||||
chain.emit(py, failed).unwrap(),
|
||||
LifecycleStep::Done
|
||||
));
|
||||
chain.close(py);
|
||||
assert_eq!(
|
||||
entries(&log),
|
||||
[
|
||||
"first.begin[]",
|
||||
"first.failed:begin failed",
|
||||
"first.close",
|
||||
"second.close",
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_suspended_step_resumes_its_lifecycle_and_then_reaches_the_later_ones() {
|
||||
crate::initialize_python();
|
||||
Python::attach(|py| {
|
||||
let (mut chain, log) = chain([
|
||||
("first", Script::SuspendBegin),
|
||||
("second", Script::SuspendTerminal),
|
||||
]);
|
||||
let step = chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap();
|
||||
assert!(matches!(step, LifecycleStep::Await(_)));
|
||||
assert_eq!(entries(&log), ["first.begin[]"]);
|
||||
let step = chain.resume(py, Ok(py.None())).unwrap();
|
||||
assert!(matches!(step, LifecycleStep::Arguments(_)));
|
||||
assert_eq!(entries(&log), ["first.resume", "second.begin[\"first\"]"]);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_event_kept_across_a_suspension_reaches_the_later_lifecycles_unchanged() {
|
||||
crate::initialize_python();
|
||||
Python::attach(|py| {
|
||||
let (mut chain, log) = chain([
|
||||
("first", Script::SuspendTerminal),
|
||||
("second", Script::Plain),
|
||||
]);
|
||||
chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap();
|
||||
entries(&log);
|
||||
let raised = PyValueError::new_err("provider failed");
|
||||
let failed = LifecycleEvent::Failed {
|
||||
timing: timing(),
|
||||
origin: FailureOrigin::Call,
|
||||
error: &raised,
|
||||
};
|
||||
assert!(matches!(
|
||||
chain.emit(py, failed).unwrap(),
|
||||
LifecycleStep::Await(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
chain.resume(py, Ok(py.None())).unwrap(),
|
||||
LifecycleStep::Done
|
||||
));
|
||||
assert_eq!(
|
||||
entries(&log),
|
||||
[
|
||||
"first.failed:provider failed",
|
||||
"first.resume",
|
||||
"second.failed:provider failed",
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -46,7 +46,6 @@ enum Stage {
|
|||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Expect {
|
||||
Started,
|
||||
Arguments,
|
||||
Wire,
|
||||
Emitted,
|
||||
|
|
@ -159,13 +158,7 @@ where
|
|||
match (self.pending.take(), result) {
|
||||
(None, None) => {
|
||||
self.started_at = epoch_seconds();
|
||||
let started = LifecycleEvent::Started {
|
||||
start_time: self.started_at,
|
||||
};
|
||||
match self.adapter.emit(py, started) {
|
||||
Ok(step) => self.on_adapter(py, step, Expect::Started),
|
||||
Err(error) => self.adapter_failed(py, error),
|
||||
}
|
||||
self.begin(py)
|
||||
}
|
||||
(Some(Pending::Native), Some(Ok(_))) => {
|
||||
let result = self.take_native_result()?;
|
||||
|
|
@ -201,7 +194,6 @@ where
|
|||
self.pending = Some(Pending::Adapter(expect));
|
||||
Ok(ExecutionStep::Await(awaitable))
|
||||
}
|
||||
(Expect::Started, LifecycleStep::Done) => self.begin(py),
|
||||
(Expect::Arguments, LifecycleStep::Arguments(arguments)) => {
|
||||
self.arguments = Some(arguments);
|
||||
self.stage = Stage::Call;
|
||||
|
|
@ -795,7 +787,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
self.log.push(match event {
|
||||
LifecycleEvent::Started { .. } => "started".into(),
|
||||
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
|
||||
format!("response:{}", raw.body)
|
||||
}
|
||||
|
|
@ -927,7 +918,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"before_send",
|
||||
|
|
@ -973,7 +963,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"classify:provider exploded",
|
||||
|
|
@ -1007,7 +996,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"classify:op rejected",
|
||||
|
|
@ -1039,7 +1027,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"failed:Call:op failed",
|
||||
|
|
@ -1077,7 +1064,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"classify:provider exploded",
|
||||
|
|
@ -1108,7 +1094,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"failed:Host:begin failed",
|
||||
"adapter.close",
|
||||
|
|
@ -1239,10 +1224,7 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
)
|
||||
.unwrap_err();
|
||||
assert!(!error.is_instance_of::<pyo3::exceptions::PyException>(py));
|
||||
assert_eq!(
|
||||
log.entries(),
|
||||
["started", "begin", "route", "adapter.close"]
|
||||
);
|
||||
assert_eq!(log.entries(), ["begin", "route", "adapter.close"]);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
mod adapter;
|
||||
mod argument;
|
||||
mod callable;
|
||||
mod chain;
|
||||
mod driver;
|
||||
mod execution;
|
||||
mod gil;
|
||||
|
|
@ -13,10 +14,12 @@ mod handle;
|
|||
mod marshal;
|
||||
|
||||
pub use adapter::{
|
||||
InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, missing_state,
|
||||
InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, is_cancellation,
|
||||
missing_state,
|
||||
};
|
||||
pub use argument::lookup;
|
||||
pub use callable::wrap_failure;
|
||||
pub use chain::LifecycleChain;
|
||||
pub use driver::run_call;
|
||||
pub use execution::{poll_async_value, run_async, run_async_value, run_sync, run_sync_value};
|
||||
pub use gil::{release_count, release_gil};
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
- Target invariants, not completion claims; these supersede the crate guidance below where they conflict
|
||||
- Keep this crate the product-specific PyO3 consumer of `litellm-host-python`
|
||||
- Own registration, input projection, the route host and the caller callables it answers operations with (file readers, token providers), public response/error construction and the per-call composition of machine, route host and callback contract
|
||||
- Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `run_legacy_python_call`; the bridge hands the public call over and keeps no copy
|
||||
- Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `LegacyPythonLifecycle`; the bridge hands the public call over and keeps no copy
|
||||
- Routes never select a callback contract: `callbacks::run_python_call` is the only way a route starts a call, and `src/callbacks.rs` is the only place that names a contract crate
|
||||
- Every call runs the legacy `Logging` lifecycle, then the v1 lifecycle when anything is subscribed; with no subscriber the call is exactly the legacy-only call
|
||||
- A route describes itself once as a `CallSurface`; fields only the legacy contract reads are marked so and go when it goes
|
||||
- Retiring legacy for a route, or for all of them, is a change to that one composition, never to a route, the driver or an adapter
|
||||
- Value-oriented execution, sync waiting, nested-runtime checks, signal polling and panic containment live in `litellm-host-python`; native async work uses `pyo3-async-runtimes`, Serde output uses `Pythonized<T>`
|
||||
- Core owns typed native state, the route machine, provider preparation/I/O and normalization; the host driver owns terminal events; the legacy adapter in `litellm-callbacks-legacy-python` owns `Logging` dispatch policy
|
||||
- Python, Rust SDK and gateway use one lifecycle-bearing core route entrypoint; provider helpers stay private, never bridge-accessible transport drivers
|
||||
|
|
|
|||
|
|
@ -19,10 +19,12 @@ panic-test = []
|
|||
bytes.workspace = true
|
||||
litellm-auth.workspace = true
|
||||
litellm-callbacks-legacy-python.workspace = true
|
||||
litellm-callbacks-v1-python.workspace = true
|
||||
litellm-core.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-auth-gcp.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-host.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
litellm-types.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
|
|
|
|||
79
litellm-rust/crates/python-bridge/src/callbacks.rs
Normal file
79
litellm-rust/crates/python-bridge/src/callbacks.rs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
//! The callback contracts a Python call runs under, composed here once so that no route
|
||||
//! selects one. Every call runs the legacy `Logging` contract, for as long as Python's
|
||||
//! `Logging` exists, and then the v1 contract whenever anything is subscribed to it.
|
||||
//! Both are a [`PythonLifecycle`]; that trait is the whole boundary the driver knows.
|
||||
|
||||
use litellm_callbacks_legacy_python::{LegacyPythonLifecycle, LegacyPythonSurface};
|
||||
pub(crate) use litellm_callbacks_legacy_python::{PassThroughStream, PublicCall};
|
||||
use litellm_callbacks_v1_python::{V1PythonLifecycle, V1PythonSurface};
|
||||
use litellm_host::{machine::Machine, route::Route};
|
||||
use litellm_host_python::{LifecycleChain, PythonLifecycle, RouteHost, run_call};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
/// What a route tells the callback contracts about itself.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct CallSurface {
|
||||
pub call_type: &'static str,
|
||||
/// Legacy only: what `Logging.pre_call` is told the input was.
|
||||
pub input_description: &'static str,
|
||||
/// Legacy only: how a streamed response is billed; `None` for a route that never
|
||||
/// streams.
|
||||
pub stream: Option<PassThroughStream>,
|
||||
}
|
||||
|
||||
impl CallSurface {
|
||||
fn legacy(self) -> LegacyPythonSurface {
|
||||
LegacyPythonSurface {
|
||||
call_type: self.call_type,
|
||||
input_description: self.input_description,
|
||||
stream: self.stream,
|
||||
}
|
||||
}
|
||||
|
||||
fn v1(self) -> V1PythonSurface {
|
||||
V1PythonSurface {
|
||||
call_type: self.call_type,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Legacy first: v1 subscribers observe the keyword view and the wire request as the
|
||||
/// legacy contract left them, and share the `litellm_call_id` its setup assigned.
|
||||
fn lifecycle(
|
||||
py: Python<'_>,
|
||||
surface: CallSurface,
|
||||
call: PublicCall,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Box<dyn PythonLifecycle>> {
|
||||
let legacy = Box::new(LegacyPythonLifecycle::new(
|
||||
py,
|
||||
surface.legacy(),
|
||||
call,
|
||||
asynchronous,
|
||||
));
|
||||
Ok(
|
||||
match V1PythonLifecycle::subscribed(py, surface.v1(), asynchronous)? {
|
||||
Some(v1) => Box::new(LifecycleChain::new(vec![legacy, Box::new(v1)])),
|
||||
None => legacy,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Runs one native call for a Python caller. The route host projects from the keyword
|
||||
/// view the lifecycle's `begin` prepares.
|
||||
pub(crate) fn run_python_call<H, M>(
|
||||
py: Python<'_>,
|
||||
surface: CallSurface,
|
||||
call: PublicCall,
|
||||
machine: M,
|
||||
route: H,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Py<PyAny>>
|
||||
where
|
||||
H: RouteHost + 'static,
|
||||
M: Machine<Route = H::Route, Complete = <H::Route as Route>::Response> + 'static,
|
||||
{
|
||||
let arguments = call.arguments(py);
|
||||
let lifecycle = lifecycle(py, surface, call, asynchronous)?;
|
||||
run_call(py, machine, route, lifecycle, arguments, asynchronous)
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
mod callbacks;
|
||||
mod credentials;
|
||||
mod diagnostics;
|
||||
mod errors;
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
mod host;
|
||||
|
||||
use host::MessagesRouteHost;
|
||||
use litellm_callbacks_legacy_python::{
|
||||
LegacyPythonSurface, PassThroughStream, PublicCall, run_legacy_python_call,
|
||||
};
|
||||
use litellm_core::messages::route::{messages_machine, supports};
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
use crate::{
|
||||
callbacks::{CallSurface, PassThroughStream, PublicCall, run_python_call},
|
||||
errors::RustBridgeDeclined,
|
||||
};
|
||||
|
||||
const SURFACE: LegacyPythonSurface = LegacyPythonSurface {
|
||||
const SURFACE: CallSurface = CallSurface {
|
||||
call_type: "anthropic_messages",
|
||||
input_description: "Messages",
|
||||
stream: Some(PassThroughStream {
|
||||
|
|
@ -39,7 +39,7 @@ fn run_messages(
|
|||
"the Rust Messages route does not serve this provider",
|
||||
));
|
||||
}
|
||||
run_legacy_python_call(
|
||||
run_python_call(
|
||||
py,
|
||||
SURFACE,
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ use std::sync::{Arc, LazyLock};
|
|||
|
||||
use host::OcrRouteHost;
|
||||
use litellm_auth_gcp::VertexAuth;
|
||||
use litellm_callbacks_legacy_python::{LegacyPythonSurface, PublicCall, run_legacy_python_call};
|
||||
use litellm_core::ocr::route::ocr_machine;
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
|
|
@ -19,15 +18,20 @@ use pyo3::{
|
|||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::{errors::RustBridgeDeclined, http, python_settings::PythonSettings};
|
||||
use crate::{
|
||||
callbacks::{CallSurface, PublicCall, run_python_call},
|
||||
errors::RustBridgeDeclined,
|
||||
http,
|
||||
python_settings::PythonSettings,
|
||||
};
|
||||
|
||||
const SURFACE: LegacyPythonSurface = LegacyPythonSurface {
|
||||
const SURFACE: CallSurface = CallSurface {
|
||||
call_type: "ocr",
|
||||
input_description: "OCR document processing",
|
||||
stream: None,
|
||||
};
|
||||
|
||||
const ASYNC_SURFACE: LegacyPythonSurface = LegacyPythonSurface {
|
||||
const ASYNC_SURFACE: CallSurface = CallSurface {
|
||||
call_type: "aocr",
|
||||
..SURFACE
|
||||
};
|
||||
|
|
@ -52,7 +56,7 @@ fn run_ocr(
|
|||
secrets,
|
||||
)
|
||||
.map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?;
|
||||
run_legacy_python_call(
|
||||
run_python_call(
|
||||
py,
|
||||
if asynchronous { ASYNC_SURFACE } else { SURFACE },
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
|
|
|
|||
|
|
@ -35,10 +35,9 @@ def test_v1_golden_envelopes_match_the_typed_contract() -> None:
|
|||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning)
|
||||
adapter: Final = TypeAdapter(callbacks.EnvelopeV1)
|
||||
events: Final = {
|
||||
adapter.validate_json(path.read_text(), strict=True)["event"]["type"] for path in GOLDEN_PATH.glob("*.json")
|
||||
}
|
||||
assert events == callbacks.EVENTS
|
||||
envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")]
|
||||
assert {envelope["event"]["type"] for envelope in envelopes} == callbacks.EVENTS
|
||||
assert {envelope["schema"] for envelope in envelopes} == callbacks.SUPPORTED_SCHEMAS
|
||||
|
||||
|
||||
class ValidCallback:
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.rust_bridge import loader
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_loader() -> None:
|
||||
loader.reset_native_bridge_cache()
|
||||
yield
|
||||
loader.reset_native_bridge_cache()
|
||||
|
||||
|
||||
def test_child_forked_after_the_extension_loaded_reports_it_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(loader, "_cached_bridge", ModuleType("_native"))
|
||||
|
||||
loader._disable_in_forked_child()
|
||||
|
||||
assert loader.get_native_bridge() is None
|
||||
assert loader.native_bridge_available() is False
|
||||
|
||||
|
||||
def test_child_forked_before_the_extension_loaded_keeps_it() -> None:
|
||||
loader._disable_in_forked_child()
|
||||
|
||||
assert loader._forked_after_load is False
|
||||
141
tests/test_litellm_rust/ocr/test_callbacks_v1.py
Normal file
141
tests/test_litellm_rust/ocr/test_callbacks_v1.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
from collections.abc import Generator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.rust_bridge import callbacks_v1_python
|
||||
from litellm.rust_bridge.callbacks_v1_python import EVENTS, EnvelopeV1, RequestFactsV1, WirePatchV1
|
||||
from tests.test_litellm_rust.support.callback_recorder import drain_logging
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
|
||||
from tests.test_litellm_rust.support.requests import (
|
||||
OCR_RESPONSE,
|
||||
call_native_aocr,
|
||||
call_native_ocr,
|
||||
request_body,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ocr_server(recording_server: RecordingServer) -> RecordingServer:
|
||||
recording_server.default_response = ResponseSpec(body=OCR_RESPONSE)
|
||||
return recording_server
|
||||
|
||||
|
||||
class Observer:
|
||||
name: Final = "observer"
|
||||
schema: Final = 1
|
||||
events: Final = EVENTS
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.envelopes: Final[list[EnvelopeV1]] = []
|
||||
|
||||
def on_event(self, event: EnvelopeV1) -> None:
|
||||
self.envelopes.append(event)
|
||||
|
||||
|
||||
class Interceptor:
|
||||
name: Final = "interceptor"
|
||||
schema: Final = 1
|
||||
events: Final[frozenset[str]] = frozenset()
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.bodies: Final[list[object]] = []
|
||||
|
||||
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
self.bodies.append(request["body"])
|
||||
return {"headers": {"set": [("x-v1-interceptor", "seen")]}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def observer() -> Generator[Observer]:
|
||||
subscriber: Final = Observer()
|
||||
callbacks_v1_python.register(subscriber)
|
||||
try:
|
||||
yield subscriber
|
||||
finally:
|
||||
callbacks_v1_python.unregister(subscriber)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def interceptor() -> Generator[Interceptor]:
|
||||
subscriber: Final = Interceptor()
|
||||
callbacks_v1_python.register(subscriber)
|
||||
try:
|
||||
yield subscriber
|
||||
finally:
|
||||
callbacks_v1_python.unregister(subscriber)
|
||||
|
||||
|
||||
class LegacyCallId(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
self.call_ids: Final[list[str]] = []
|
||||
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
self.call_ids.append(kwargs["litellm_call_id"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_native_ocr_runs_legacy_and_v1_callbacks_on_the_same_call(
|
||||
ocr_server: RecordingServer, observer: Observer, asynchronous: bool
|
||||
) -> None:
|
||||
legacy: Final = LegacyCallId()
|
||||
|
||||
if asynchronous:
|
||||
await call_native_aocr(ocr_server, callbacks=[legacy])
|
||||
else:
|
||||
call_native_ocr(ocr_server, callbacks=[legacy])
|
||||
await drain_logging()
|
||||
|
||||
assert [envelope["event"]["type"] for envelope in observer.envelopes] == [
|
||||
"call.started",
|
||||
"request.sending",
|
||||
"response.received",
|
||||
"call.succeeded",
|
||||
]
|
||||
assert [envelope["seq"] for envelope in observer.envelopes] == sorted(
|
||||
envelope["seq"] for envelope in observer.envelopes
|
||||
)
|
||||
assert len(legacy.call_ids) == 1
|
||||
assert {envelope["call_id"] for envelope in observer.envelopes} == set(legacy.call_ids)
|
||||
|
||||
|
||||
def test_native_ocr_v1_interceptor_sees_the_wire_as_the_legacy_callback_left_it(
|
||||
ocr_server: RecordingServer, interceptor: Interceptor
|
||||
) -> None:
|
||||
class Edit(CustomLogger):
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
request_body(kwargs)["include_image_base64"] = True
|
||||
|
||||
call_native_ocr(ocr_server, callbacks=[Edit()])
|
||||
|
||||
assert len(interceptor.bodies) == 1
|
||||
body: Final = interceptor.bodies[0]
|
||||
assert isinstance(body, dict)
|
||||
assert body["include_image_base64"] is True
|
||||
request: Final = ocr_server.requests[-1]
|
||||
assert request.headers["x-v1-interceptor"] == "seen"
|
||||
|
||||
|
||||
def test_native_ocr_failure_reaches_both_contracts_once(
|
||||
recording_server: RecordingServer, observer: Observer
|
||||
) -> None:
|
||||
recording_server.default_response = ResponseSpec(status=400, body={"message": "bad document"})
|
||||
failures: Final[list[object]] = []
|
||||
|
||||
class Failure(CustomLogger):
|
||||
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
failures.append(kwargs["exception"])
|
||||
|
||||
with pytest.raises(Exception) as raised: # noqa: PT011 # the mapped provider error class is covered elsewhere
|
||||
call_native_ocr(recording_server, callbacks=[Failure()])
|
||||
|
||||
assert failures == [raised.value]
|
||||
terminal: Final = [
|
||||
envelope for envelope in observer.envelopes if envelope["event"]["type"] in {"call.succeeded", "call.failed"}
|
||||
]
|
||||
assert [envelope["event"]["type"] for envelope in terminal] == ["call.failed"]
|
||||
assert observer.envelopes[-1] is terminal[0]
|
||||
Loading…
Add table
Reference in a new issue