mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 57aa3f2895 into cbcccd531c
This commit is contained in:
commit
a8bef91c62
72 changed files with 3969 additions and 364 deletions
31
litellm-rust/Cargo.lock
generated
31
litellm-rust/Cargo.lock
generated
|
|
@ -2043,6 +2043,33 @@ dependencies = [
|
|||
"strum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-callbacks-v1"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-auth",
|
||||
"litellm-host",
|
||||
"proptest",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-callbacks-v1-python"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-auth",
|
||||
"litellm-callbacks-v1",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
"pyo3",
|
||||
"serde_json",
|
||||
"strum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-core"
|
||||
version = "0.1.0"
|
||||
|
|
@ -2085,6 +2112,7 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"fancy-regex",
|
||||
"litellm-types",
|
||||
"percent-encoding",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -2170,6 +2198,7 @@ dependencies = [
|
|||
"litellm-host",
|
||||
"litellm-http",
|
||||
"litellm-types",
|
||||
"percent-encoding",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
"serde",
|
||||
|
|
@ -2193,8 +2222,10 @@ dependencies = [
|
|||
"litellm-auth",
|
||||
"litellm-auth-gcp",
|
||||
"litellm-callbacks-legacy-python",
|
||||
"litellm-callbacks-v1-python",
|
||||
"litellm-core",
|
||||
"litellm-core-utils",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ repository = "https://github.com/BerriAI/litellm"
|
|||
litellm-core = { path = "crates/core" }
|
||||
litellm-host = { path = "crates/host" }
|
||||
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
||||
litellm-callbacks-v1 = { path = "crates/callbacks-v1" }
|
||||
litellm-callbacks-v1-python = { path = "crates/callbacks-v1-python" }
|
||||
litellm-framing = { path = "crates/framer" }
|
||||
litellm-auth = { path = "crates/auth" }
|
||||
litellm-auth-aws = { path = "crates/auth-aws" }
|
||||
|
|
@ -52,6 +54,7 @@ base64 = "0.22"
|
|||
moka = { version = "0.12.16", features = ["future"] }
|
||||
strum = { version = "0.28.0", features = ["derive"] }
|
||||
url = "2.5.8"
|
||||
percent-encoding = "2.3.2"
|
||||
webpki-roots = "1"
|
||||
time = { version = "0.3.53", features = ["parsing"] }
|
||||
criterion = "0.8.2"
|
||||
|
|
|
|||
|
|
@ -1,10 +1,20 @@
|
|||
- Target invariants, not completion claims
|
||||
- Keep this crate the legacy `@client` wrapper as the native call sees it, and nothing else: the `Logging` contract (`function_setup`, the deployment hooks, `pre_call`/`post_call`, the sync and async success and failure fan-out, the deferred proxy release, the argument sharing those callbacks rely on) plus the kwargs rewrites the wrapper makes on the way in (credential-name inheritance, the budget and retry-count limits)
|
||||
- The driver in `litellm-host-python`, the routes and core see one `PythonLifecycle`; they never learn which Python objects consume a call
|
||||
- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`)
|
||||
- Terms (lifecycle terms are in `host-python/AGENTS.md`)
|
||||
- `Logging`: litellm's Python class; logger: the one `Logging` instance of this call (`PythonLogger`), reused from `litellm_logging_obj` or built by `function_setup`
|
||||
- Borrowed function: a litellm Python internal Rust still calls, one variant of `LegacyPython`; a user's own callback is never one
|
||||
- Fan-out: `Logging` dispatching to every registered callback; family: the sync or the async set of success and failure handlers
|
||||
- Deployment hooks: the awaited pre-call, post-call and failure hooks on each callback; async calls only, as in `@client`
|
||||
- Re-aliasing: replacing body values with the caller's equal objects before `pre_call`, so a callback mutates what the caller holds
|
||||
- Deferred release: the proxy holds back async success logging until it accepts the response (`PendingLogging.release`)
|
||||
- Internal call: a proxy-internal call; when it is async, the async success dispatch and both failure families are skipped, and the sync success handlers still run
|
||||
- Pass-through billing (`PassThroughStream`): end-of-stream cost logging from the delivered chunks; a failure after the stream opened bills them as partial usage, and a sync call falls back to the plain failure handler
|
||||
- Redaction here is `****`, matching what the Python path logs; `[REDACTED]` belongs to the v1 contract
|
||||
- Rust drives the call; every litellm Python internal it still borrows is a variant of `LegacyPython`, grouped by subsystem (`Wrapper`, `Logging`, `DeploymentHooks`, `Streaming`)
|
||||
- The enum only shrinks: when Rust owns a subsystem, delete its group rather than adding a Rust path beside it
|
||||
- Calling a user's own callback directly is permanent Python surface and gets its own type outside `LegacyPython`
|
||||
- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes hand it over through `run_legacy_call` and keep no copy
|
||||
- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the legacy path rewrites it (setup, deployment hook, prepare) and the bound request object whose attributes back keywords the caller omitted; routes capture it and the bridge's `run_python_call` hands it to `LegacyPythonLifecycle::new`, keeping no copy
|
||||
- `setup` reuses a `Logging` the caller passed as `litellm_logging_obj` (the proxy and Router are the live cases) and otherwise builds one through `function_setup`, as `@client` does
|
||||
- Either way every phase calls the same `Logging` method the Python path calls; which callbacks run is `Logging`'s decision, never this crate's
|
||||
- Callbacks receive the caller's own objects and may mutate them; this crate alone carries that obligation
|
||||
|
|
|
|||
|
|
@ -9,13 +9,11 @@ autotests = false
|
|||
[dependencies]
|
||||
litellm-host.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
|
||||
pyo3.workspace = true
|
||||
strum.workspace = true
|
||||
serde_json.workspace = true
|
||||
strum.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-auth.workspace = true
|
||||
proptest.workspace = true
|
||||
rstest.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ use litellm_host::event::{
|
|||
FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest, epoch_seconds,
|
||||
};
|
||||
use litellm_host_python::{
|
||||
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, missing_state, to_py,
|
||||
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::{PyBaseException, PyException},
|
||||
exceptions::PyBaseException,
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
prelude::*,
|
||||
types::{PyDateTime, PyDict, PyList},
|
||||
|
|
@ -17,7 +17,8 @@ use pyo3::{
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger,
|
||||
PublicCall, PythonLogger, after_deployment_failure, after_deployment_success,
|
||||
before_deployment_call,
|
||||
deferred::{PendingLogging, PendingSuccess},
|
||||
finalize, is_internal_call, prepare,
|
||||
python::Streaming,
|
||||
|
|
@ -26,7 +27,7 @@ use crate::{
|
|||
|
||||
/// What the legacy contract needs to know about the route it is logging.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct LegacySurface {
|
||||
pub struct LegacyPythonSurface {
|
||||
pub call_type: &'static str,
|
||||
/// What `Logging.pre_call` is told the input was.
|
||||
pub input_description: &'static str,
|
||||
|
|
@ -55,8 +56,8 @@ enum Pending {
|
|||
AsyncFailure,
|
||||
}
|
||||
|
||||
pub struct LegacyLogging {
|
||||
surface: LegacySurface,
|
||||
pub struct LegacyPythonLifecycle {
|
||||
surface: LegacyPythonSurface,
|
||||
call: PublicCall,
|
||||
logger: Option<PythonLogger>,
|
||||
start: Py<PyAny>,
|
||||
|
|
@ -76,14 +77,10 @@ fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult<Py<PyAny>> {
|
|||
PyDateTime::from_timestamp(py, epoch_seconds, None).map(|value| value.into_any().unbind())
|
||||
}
|
||||
|
||||
fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool {
|
||||
!error.is_instance_of::<PyException>(py)
|
||||
}
|
||||
|
||||
impl LegacyLogging {
|
||||
impl LegacyPythonLifecycle {
|
||||
pub fn new(
|
||||
py: Python<'_>,
|
||||
surface: LegacySurface,
|
||||
surface: LegacyPythonSurface,
|
||||
call: PublicCall,
|
||||
asynchronous: bool,
|
||||
) -> Self {
|
||||
|
|
@ -118,6 +115,14 @@ impl LegacyLogging {
|
|||
}
|
||||
|
||||
fn prepare(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
|
||||
let model = self.call.lookup(py, "model")?;
|
||||
let provider = self.call.lookup(py, "custom_llm_provider")?;
|
||||
self.logger()?.initialize_failure_context(
|
||||
py,
|
||||
self.call.kwargs(),
|
||||
model.and_then(|model| model.extract::<String>().ok()),
|
||||
provider.and_then(|provider| provider.extract::<String>().ok()),
|
||||
)?;
|
||||
let prepared = prepare(py, self.call.kwargs().bind(py), self.logger()?)?.unbind();
|
||||
self.call.set_kwargs(prepared);
|
||||
Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py)))
|
||||
|
|
@ -140,7 +145,7 @@ impl LegacyLogging {
|
|||
|
||||
fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
|
||||
match self.try_dispatch_success(py) {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
Err(error) if !is_cancellation(py, &error) => {
|
||||
error.write_unraisable(py, self.logger.as_ref().map(|logger| logger.object(py)));
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -199,7 +204,7 @@ impl LegacyLogging {
|
|||
),
|
||||
);
|
||||
match billed {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
Err(error) if !is_cancellation(py, &error) => {
|
||||
error.write_unraisable(py, Some(logger.object(py)));
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -269,15 +274,15 @@ impl LegacyLogging {
|
|||
}
|
||||
}
|
||||
|
||||
impl PythonLifecycle for LegacyLogging {
|
||||
impl PythonLifecycle for LegacyPythonLifecycle {
|
||||
fn begin(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
started_at: f64,
|
||||
start_time: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.call.set_kwargs(arguments);
|
||||
self.start = datetime(py, started_at)?;
|
||||
self.start = datetime(py, start_time)?;
|
||||
self.internal = is_internal_call(py)?;
|
||||
let result = setup(
|
||||
py,
|
||||
|
|
@ -291,7 +296,7 @@ impl PythonLifecycle for LegacyLogging {
|
|||
self.call.set_kwargs(result.kwargs()?);
|
||||
if self.runs_deployment_hooks() {
|
||||
self.pending = Some(Pending::DeploymentPreCall);
|
||||
return Ok(LifecycleStep::Await(DeploymentHooks::before_call(
|
||||
return Ok(LifecycleStep::Await(before_deployment_call(
|
||||
py,
|
||||
self.call.kwargs(),
|
||||
self.surface.call_type,
|
||||
|
|
@ -354,7 +359,7 @@ impl PythonLifecycle for LegacyLogging {
|
|||
self.response = Some(response);
|
||||
if self.runs_deployment_hooks() {
|
||||
self.pending = Some(Pending::DeploymentPostCall);
|
||||
return Ok(LifecycleStep::Await(DeploymentHooks::after_success(
|
||||
return Ok(LifecycleStep::Await(after_deployment_success(
|
||||
py,
|
||||
self.call.kwargs(),
|
||||
&self.response,
|
||||
|
|
@ -366,7 +371,6 @@ impl PythonLifecycle for LegacyLogging {
|
|||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
match event {
|
||||
LifecycleEvent::Started { .. } => Ok(LifecycleStep::Done),
|
||||
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
|
||||
let api_key = self
|
||||
.context
|
||||
|
|
@ -407,7 +411,7 @@ impl PythonLifecycle for LegacyLogging {
|
|||
{
|
||||
let error = self.error.as_ref().ok_or_else(missing_state)?;
|
||||
self.pending = Some(Pending::DeploymentFailure);
|
||||
return Ok(LifecycleStep::Await(DeploymentHooks::after_failure(
|
||||
return Ok(LifecycleStep::Await(after_deployment_failure(
|
||||
py,
|
||||
self.call.kwargs(),
|
||||
error,
|
||||
|
|
@ -464,7 +468,11 @@ impl PythonLifecycle for LegacyLogging {
|
|||
{
|
||||
error.write_unraisable(py, None);
|
||||
}
|
||||
self.pending = None;
|
||||
self.response = None;
|
||||
self.error = None;
|
||||
self.body = None;
|
||||
self.headers = None;
|
||||
self.context = None;
|
||||
self.stream = None;
|
||||
}
|
||||
|
|
@ -482,7 +490,8 @@ impl PythonLifecycle for LegacyLogging {
|
|||
visit.call(&stream.chunks)?;
|
||||
visit.call(&stream.first_chunk)?;
|
||||
}
|
||||
visit.call(&self.body)
|
||||
visit.call(&self.body)?;
|
||||
visit.call(&self.headers)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,16 +3,13 @@
|
|||
//! lifetime. No other callback host has that obligation, which is why nothing outside
|
||||
//! this crate holds them.
|
||||
|
||||
use litellm_host::{machine::Machine, route::Route};
|
||||
use litellm_host_python::{RouteHost, lookup, run_call};
|
||||
use litellm_host_python::lookup;
|
||||
use pyo3::{
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
prelude::*,
|
||||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::{LegacyLogging, LegacySurface};
|
||||
|
||||
pub struct PublicCall {
|
||||
args: Py<PyTuple>,
|
||||
kwargs: Py<PyDict>,
|
||||
|
|
@ -34,6 +31,11 @@ impl PublicCall {
|
|||
})
|
||||
}
|
||||
|
||||
/// The keyword view the call starts from, which the driver hands to `begin`.
|
||||
pub fn arguments(&self, py: Python<'_>) -> Py<PyDict> {
|
||||
self.kwargs.clone_ref(py)
|
||||
}
|
||||
|
||||
pub(crate) fn args(&self) -> &Py<PyTuple> {
|
||||
&self.args
|
||||
}
|
||||
|
|
@ -63,31 +65,6 @@ impl PublicCall {
|
|||
}
|
||||
}
|
||||
|
||||
/// Runs one native call under the legacy `Logging` contract: the route host projects from
|
||||
/// the keyword view the contract prepares, and the contract observes the call.
|
||||
pub fn run_legacy_call<H, M>(
|
||||
py: Python<'_>,
|
||||
surface: LegacySurface,
|
||||
call: PublicCall,
|
||||
machine: M,
|
||||
route: H,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Py<PyAny>>
|
||||
where
|
||||
H: RouteHost + 'static,
|
||||
M: Machine<Route = H::Route, Complete = <H::Route as Route>::Response> + 'static,
|
||||
{
|
||||
let arguments = call.kwargs.clone_ref(py);
|
||||
run_call(
|
||||
py,
|
||||
machine,
|
||||
route,
|
||||
Box::new(LegacyLogging::new(py, surface, call, asynchronous)),
|
||||
arguments,
|
||||
asynchronous,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -9,78 +9,10 @@ use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
|
|||
use crate::logger::PythonLogger;
|
||||
use crate::python::{Logging, Wrapper};
|
||||
|
||||
pub trait LegacyCallbacks {
|
||||
impl PythonLogger {
|
||||
/// `Logging.update_from_kwargs`: what the logger is told about the request it is
|
||||
/// about to see, with consumed credentials redacted.
|
||||
fn update_from_kwargs(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
wire: &WireRequest,
|
||||
context: &RequestContext,
|
||||
) -> PyResult<()>;
|
||||
|
||||
/// `Logging.pre_call`.
|
||||
fn pre_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
input: &str,
|
||||
api_key: Option<&str>,
|
||||
body: &Bound<'_, PyDict>,
|
||||
headers: &Bound<'_, PyDict>,
|
||||
url: &str,
|
||||
) -> PyResult<()>;
|
||||
|
||||
/// `Logging.post_call`.
|
||||
fn post_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
original_response: &str,
|
||||
api_key: Option<&str>,
|
||||
body: Option<&Py<PyDict>>,
|
||||
headers: Option<&Py<PyDict>>,
|
||||
) -> PyResult<()>;
|
||||
|
||||
fn defers_async_logging(&self, py: Python<'_>) -> bool;
|
||||
|
||||
fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()>;
|
||||
|
||||
fn sync_success_for_async_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()>;
|
||||
|
||||
fn failure(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
error: &Py<PyBaseException>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Option<Py<PyAny>>>;
|
||||
|
||||
fn submit_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()>;
|
||||
|
||||
fn enqueue_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()>;
|
||||
}
|
||||
|
||||
impl LegacyCallbacks for PythonLogger {
|
||||
fn update_from_kwargs(
|
||||
pub(crate) fn update_from_kwargs(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
|
|
@ -128,7 +60,8 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn pre_call(
|
||||
/// `Logging.pre_call`.
|
||||
pub(crate) fn pre_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
input: &str,
|
||||
|
|
@ -145,7 +78,8 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn post_call(
|
||||
/// `Logging.post_call`.
|
||||
pub(crate) fn post_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
original_response: &str,
|
||||
|
|
@ -163,19 +97,19 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn defers_async_logging(&self, py: Python<'_>) -> bool {
|
||||
pub(crate) fn defers_async_logging(&self, py: Python<'_>) -> bool {
|
||||
Logging::DefersAsync
|
||||
.call(py, (self.object(py),))
|
||||
.and_then(|value| value.extract())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> {
|
||||
pub(crate) fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> {
|
||||
Logging::DeferSuccess.call(py, (self.object(py), pending))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sync_success_for_async_call(
|
||||
pub(crate) fn sync_success_for_async_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
|
|
@ -186,7 +120,7 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn failure(
|
||||
pub(crate) fn failure(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
error: &Py<PyBaseException>,
|
||||
|
|
@ -199,7 +133,7 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(asynchronous.then(|| value.unbind()))
|
||||
}
|
||||
|
||||
fn submit_success(
|
||||
pub(crate) fn submit_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
|
|
@ -210,7 +144,7 @@ impl LegacyCallbacks for PythonLogger {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn enqueue_success(
|
||||
pub(crate) fn enqueue_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
|
|
@ -227,6 +161,55 @@ impl LegacyCallbacks for PythonLogger {
|
|||
}
|
||||
enqueue.map(|_| ())
|
||||
}
|
||||
|
||||
pub(crate) fn initialize_failure_context(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
model: Option<String>,
|
||||
custom_llm_provider: Option<String>,
|
||||
) -> PyResult<()> {
|
||||
let Some(model) = model else {
|
||||
return Ok(());
|
||||
};
|
||||
let (provider, model) = match custom_llm_provider.filter(|provider| !provider.is_empty()) {
|
||||
Some(provider) => {
|
||||
let model = model
|
||||
.strip_prefix(provider.as_str())
|
||||
.and_then(|model| model.strip_prefix('/'))
|
||||
.unwrap_or(&model)
|
||||
.to_string();
|
||||
(provider, model)
|
||||
}
|
||||
None => match model.split_once('/') {
|
||||
Some((provider, model)) if !provider.is_empty() && !model.is_empty() => {
|
||||
(provider.to_string(), model.to_string())
|
||||
}
|
||||
_ => return Ok(()),
|
||||
},
|
||||
};
|
||||
let kwargs = kwargs.bind(py);
|
||||
let metadata = PyDict::new(py);
|
||||
for name in ["metadata", "litellm_metadata"] {
|
||||
if let Some(value) = kwargs.get_item(name)? {
|
||||
metadata.set_item(name, value)?;
|
||||
}
|
||||
}
|
||||
let params = PyDict::new(py);
|
||||
params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?;
|
||||
Logging::Update.call(
|
||||
py,
|
||||
(
|
||||
self.object(py),
|
||||
metadata,
|
||||
model,
|
||||
PyDict::new(py),
|
||||
params,
|
||||
provider,
|
||||
),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn custom_pricing_fields(py: Python<'_>) -> PyResult<Vec<String>> {
|
||||
|
|
@ -254,6 +237,6 @@ fn redact(
|
|||
}
|
||||
|
||||
/// Proxy-internal calls skip the legacy success fan-out.
|
||||
pub fn is_internal_call(py: Python<'_>) -> PyResult<bool> {
|
||||
pub(crate) fn is_internal_call(py: Python<'_>) -> PyResult<bool> {
|
||||
Wrapper::IsInternalCall.call(py, ())?.extract()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
//! The proxy's deferred success release: the async success handler is queued only once
|
||||
//! the proxy accepts the response, and at most once.
|
||||
|
||||
use pyo3::{exceptions::PyException, prelude::*};
|
||||
use litellm_host_python::is_cancellation;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::{LegacyCallbacks, PythonLogger};
|
||||
use crate::PythonLogger;
|
||||
|
||||
pub(crate) struct PendingSuccess {
|
||||
pub(crate) logger: PythonLogger,
|
||||
|
|
@ -37,7 +38,7 @@ impl PendingLogging {
|
|||
&& success
|
||||
{
|
||||
match pending.asynchronous(py) {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
Err(error) if !is_cancellation(py, &error) => {
|
||||
error.write_unraisable(py, Some(pending.logger.object(py)));
|
||||
}
|
||||
result => return result,
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
//! core never learn which Python object is on the other end.
|
||||
//!
|
||||
//! Legacy callbacks receive the caller's own objects and may mutate them. [`PublicCall`]
|
||||
//! is where those objects live, and [`run_legacy_call`] is how a route hands them over
|
||||
//! without keeping a copy.
|
||||
//! is where those objects live; the bridge hands it to [`LegacyPythonLifecycle::new`] and
|
||||
//! keeps no copy.
|
||||
|
||||
mod adapter;
|
||||
mod call;
|
||||
|
|
@ -20,9 +20,11 @@ mod python;
|
|||
#[path = "../tests/support.rs"]
|
||||
mod test_support;
|
||||
|
||||
pub(crate) use adapter::LegacyLogging;
|
||||
pub use adapter::{LegacySurface, PassThroughStream};
|
||||
pub use call::{PublicCall, run_legacy_call};
|
||||
pub(crate) use callbacks::{LegacyCallbacks, is_internal_call};
|
||||
pub(crate) use logger::{DeploymentHooks, PythonLogger, finalize, setup};
|
||||
pub use adapter::{LegacyPythonLifecycle, LegacyPythonSurface, PassThroughStream};
|
||||
pub use call::PublicCall;
|
||||
pub(crate) use callbacks::is_internal_call;
|
||||
pub(crate) use logger::{
|
||||
PythonLogger, after_deployment_failure, after_deployment_success, before_deployment_call,
|
||||
finalize, setup,
|
||||
};
|
||||
pub(crate) use preparation::prepare;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! One call's `Logging` instance (the logger) and the `@client` wrapper functions that
|
||||
//! create and finish it: `function_setup`, `finalize` and the awaited deployment hooks.
|
||||
|
||||
use pyo3::{
|
||||
exceptions::PyBaseException,
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
|
|
@ -5,10 +8,10 @@ use pyo3::{
|
|||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::python::{self, Wrapper};
|
||||
use crate::python::{DeploymentHooks, Wrapper};
|
||||
|
||||
/// The `Logging` instance one call fans out through.
|
||||
pub struct PythonLogger {
|
||||
pub(crate) struct PythonLogger {
|
||||
object: Py<PyAny>,
|
||||
}
|
||||
|
||||
|
|
@ -21,17 +24,17 @@ impl PythonLogger {
|
|||
self.object.bind(py)
|
||||
}
|
||||
|
||||
pub fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
Self {
|
||||
object: self.object.clone_ref(py),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
visit.call(&self.object)
|
||||
}
|
||||
|
||||
pub fn restore_context(&self, py: Python<'_>) -> PyResult<()> {
|
||||
pub(crate) fn restore_context(&self, py: Python<'_>) -> PyResult<()> {
|
||||
Wrapper::RestoreContext.call(py, (self.object(py),))?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -45,19 +48,19 @@ impl FromPyObject<'_, '_> for PythonLogger {
|
|||
}
|
||||
}
|
||||
|
||||
pub struct SetupResult<'py>(Bound<'py, PyAny>);
|
||||
pub(crate) struct SetupResult<'py>(Bound<'py, PyAny>);
|
||||
|
||||
impl SetupResult<'_> {
|
||||
pub fn logger(&self) -> PyResult<PythonLogger> {
|
||||
pub(crate) fn logger(&self) -> PyResult<PythonLogger> {
|
||||
Ok(PythonLogger::new(self.0.getattr("logger")?.unbind()))
|
||||
}
|
||||
|
||||
pub fn kwargs(&self) -> PyResult<Py<PyDict>> {
|
||||
pub(crate) fn kwargs(&self) -> PyResult<Py<PyDict>> {
|
||||
Ok(self.0.getattr("kwargs")?.extract()?)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn setup<'py>(
|
||||
pub(crate) fn setup<'py>(
|
||||
py: Python<'py>,
|
||||
call_type: &str,
|
||||
args: &Py<PyTuple>,
|
||||
|
|
@ -70,7 +73,7 @@ pub fn setup<'py>(
|
|||
.map(SetupResult)
|
||||
}
|
||||
|
||||
pub fn finalize(
|
||||
pub(crate) fn finalize(
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
logger: &PythonLogger,
|
||||
|
|
@ -82,40 +85,36 @@ pub fn finalize(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub struct DeploymentHooks;
|
||||
pub(crate) fn before_deployment_call(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
DeploymentHooks::BeforeDeploymentCall
|
||||
.call(py, (kwargs, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
impl DeploymentHooks {
|
||||
pub fn before_call(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
python::DeploymentHooks::BeforeDeploymentCall
|
||||
.call(py, (kwargs, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
pub(crate) fn after_deployment_success(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
DeploymentHooks::AfterDeploymentSuccess
|
||||
.call(py, (kwargs, response, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
pub fn after_success(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
python::DeploymentHooks::AfterDeploymentSuccess
|
||||
.call(py, (kwargs, response, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
pub fn after_failure(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
error: &Py<PyBaseException>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
python::DeploymentHooks::AfterDeploymentFailure
|
||||
.call(py, (kwargs, error, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
pub(crate) fn after_deployment_failure(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
error: &Py<PyBaseException>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
DeploymentHooks::AfterDeploymentFailure
|
||||
.call(py, (kwargs, error, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! The kwargs rewrites the `@client` wrapper makes before the call: the logger is attached,
|
||||
//! a named credential's values are inherited, and the budget and retry-count limits run.
|
||||
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyDict, PyList},
|
||||
|
|
@ -17,7 +20,7 @@ impl<'py> CredentialEntry<'py> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn prepare<'py>(
|
||||
pub(crate) fn prepare<'py>(
|
||||
py: Python<'py>,
|
||||
kwargs: &Bound<'py, PyDict>,
|
||||
logger: &crate::PythonLogger,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
//! The Python contract of this crate: every function of
|
||||
//! `litellm/rust_bridge/callbacks_legacy_python.py` that Rust calls, by name.
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use strum::{IntoStaticStr, VariantArray};
|
||||
|
||||
|
|
@ -153,7 +156,7 @@ mod tests {
|
|||
use crate::test_support::PYTHON_CONTRACT;
|
||||
|
||||
#[test]
|
||||
fn every_borrowed_function_is_in_the_python_contract() {
|
||||
fn every_function_is_in_the_python_contract() {
|
||||
let contract: serde_json::Map<String, serde_json::Value> =
|
||||
serde_json::from_str(PYTHON_CONTRACT).unwrap();
|
||||
let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect();
|
||||
|
|
@ -177,7 +180,7 @@ mod tests {
|
|||
)
|
||||
.map(LegacyPython::name)
|
||||
.collect();
|
||||
assert_eq!(called.len(), declared.len(), "a function is borrowed twice");
|
||||
assert_eq!(called.len(), declared.len(), "a function is listed twice");
|
||||
assert_eq!(called.into_iter().collect::<BTreeSet<_>>(), declared);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use pyo3::prelude::*;
|
|||
use pyo3::types::PyDict;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::LegacyLogging;
|
||||
use super::LegacyPythonLifecycle;
|
||||
use crate::test_support::{legacy_call, local, namespace, run};
|
||||
|
||||
const CALL: &CStr = c"
|
||||
|
|
@ -24,7 +24,7 @@ fn begin<'py>(
|
|||
py: Python<'py>,
|
||||
locals: &Bound<'py, PyDict>,
|
||||
asynchronous: bool,
|
||||
) -> (LegacyLogging, LifecycleStep) {
|
||||
) -> (LegacyPythonLifecycle, LifecycleStep) {
|
||||
let mut logging = legacy_call(py, locals, asynchronous);
|
||||
let kwargs = local(locals, "kwargs")
|
||||
.cast_into::<PyDict>()
|
||||
|
|
@ -63,6 +63,59 @@ fn deployment_pre_call_hook_runs_only_for_asynchronous_calls(#[case] asynchronou
|
|||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_context_is_initialized_before_argument_preparation() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
failure = RuntimeError('limit check failed')
|
||||
metadata = {'request': 'ocr-1'}
|
||||
|
||||
class FailingLogger(StubLogger):
|
||||
def update_from_kwargs(self, **update):
|
||||
self.record('update', update)
|
||||
self.update = update
|
||||
|
||||
def check_limits(self, arguments):
|
||||
self.record('check_limits', arguments)
|
||||
raise failure
|
||||
|
||||
logger = FailingLogger()
|
||||
kwargs = {
|
||||
'logger': logger,
|
||||
'model': 'provider/model',
|
||||
'metadata': metadata,
|
||||
'litellm_call_id': 'call-1',
|
||||
}
|
||||
",
|
||||
);
|
||||
let mut logging = legacy_call(py, &locals, false);
|
||||
let kwargs = local(&locals, "kwargs")
|
||||
.cast_into::<PyDict>()
|
||||
.unwrap()
|
||||
.unbind();
|
||||
let result = logging.begin(py, kwargs, 0.0);
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("argument preparation succeeded"),
|
||||
};
|
||||
assert!(error.value(py).is(local(&locals, "failure")));
|
||||
run(
|
||||
py,
|
||||
&locals,
|
||||
c"
|
||||
assert logger.names() == ['update', 'check_limits'], logger.calls
|
||||
assert logger.update['model'] == 'model', logger.update
|
||||
assert logger.update['custom_llm_provider'] == 'provider', logger.update
|
||||
assert logger.update['kwargs']['metadata'] is metadata, logger.update
|
||||
assert logger.update['litellm_params']['litellm_call_id'] == 'call-1', logger.update
|
||||
",
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kwargs_returned_by_the_pre_call_hook_are_what_the_call_prepares() {
|
||||
Python::initialize();
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use pyo3::prelude::*;
|
|||
use rstest::rstest;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use super::LegacyLogging;
|
||||
use super::LegacyPythonLifecycle;
|
||||
use crate::PythonLogger;
|
||||
use crate::test_support::{legacy_call, local, namespace, run};
|
||||
|
||||
|
|
@ -77,7 +77,7 @@ fn before_send_bound(
|
|||
locals.set_item(name, to_py(py, value).unwrap()).unwrap();
|
||||
}
|
||||
run(py, &locals, script);
|
||||
let mut logging = LegacyLogging {
|
||||
let mut logging = LegacyPythonLifecycle {
|
||||
logger: Some(PythonLogger::new(local(&locals, "logger").unbind())),
|
||||
..legacy_call(py, &locals, false)
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use std::ffi::CStr;
|
|||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyTuple};
|
||||
|
||||
use crate::{LegacyLogging, LegacySurface, PublicCall};
|
||||
use crate::{LegacyPythonLifecycle, LegacyPythonSurface, PublicCall};
|
||||
|
||||
/// The parameters of every `callbacks_legacy_python` function, as the real module declares them.
|
||||
/// `tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python
|
||||
|
|
@ -125,6 +125,9 @@ class StubLogger:
|
|||
self.hooks = {}
|
||||
self.on_enqueue = lambda coroutine: None
|
||||
|
||||
def update_from_kwargs(self, **update):
|
||||
self.update = update
|
||||
|
||||
def record(self, name, value):
|
||||
self.calls.append((name, value))
|
||||
|
||||
|
|
@ -181,7 +184,7 @@ pub(crate) fn legacy_call(
|
|||
py: Python<'_>,
|
||||
locals: &Bound<'_, PyDict>,
|
||||
asynchronous: bool,
|
||||
) -> LegacyLogging {
|
||||
) -> LegacyPythonLifecycle {
|
||||
let request = locals
|
||||
.get_item("request")
|
||||
.unwrap()
|
||||
|
|
@ -192,9 +195,9 @@ pub(crate) fn legacy_call(
|
|||
.map(|kwargs| kwargs.cast_into::<PyDict>().unwrap())
|
||||
.unwrap_or_else(|| PyDict::new(py));
|
||||
let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap();
|
||||
LegacyLogging::new(
|
||||
LegacyPythonLifecycle::new(
|
||||
py,
|
||||
LegacySurface {
|
||||
LegacyPythonSurface {
|
||||
call_type: "test",
|
||||
input_description: "test input",
|
||||
stream: None,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use pyo3::prelude::*;
|
|||
use pyo3::types::PyDict;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::LegacyLogging;
|
||||
use super::LegacyPythonLifecycle;
|
||||
use crate::PythonLogger;
|
||||
use crate::test_support::{legacy_call, local, namespace, run};
|
||||
|
||||
|
|
@ -17,8 +17,8 @@ const TIMING: Timing = Timing {
|
|||
end_time: 1.0,
|
||||
};
|
||||
|
||||
fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> LegacyLogging {
|
||||
LegacyLogging {
|
||||
fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> LegacyPythonLifecycle {
|
||||
LegacyPythonLifecycle {
|
||||
logger: Some(PythonLogger::new(local(locals, "logger").unbind())),
|
||||
..legacy_call(py, locals, asynchronous)
|
||||
}
|
||||
|
|
@ -27,7 +27,7 @@ fn logged(py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool) -> Leg
|
|||
fn succeed(
|
||||
py: Python<'_>,
|
||||
locals: &Bound<'_, PyDict>,
|
||||
logging: &mut LegacyLogging,
|
||||
logging: &mut LegacyPythonLifecycle,
|
||||
) -> LifecycleStep {
|
||||
let response = local(locals, "response").unbind();
|
||||
logging
|
||||
|
|
@ -41,7 +41,11 @@ fn succeed(
|
|||
.unwrap()
|
||||
}
|
||||
|
||||
fn fail(py: Python<'_>, locals: &Bound<'_, PyDict>, logging: &mut LegacyLogging) -> LifecycleStep {
|
||||
fn fail(
|
||||
py: Python<'_>,
|
||||
locals: &Bound<'_, PyDict>,
|
||||
logging: &mut LegacyPythonLifecycle,
|
||||
) -> LifecycleStep {
|
||||
let failure = PyErr::from_value(local(locals, "failure"));
|
||||
logging
|
||||
.emit(
|
||||
|
|
@ -105,7 +109,7 @@ fn internal_calls_skip_failure_callbacks_only_when_asynchronous(
|
|||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(py, c"failure = ValueError('provider')");
|
||||
let mut logging = LegacyLogging {
|
||||
let mut logging = LegacyPythonLifecycle {
|
||||
internal: true,
|
||||
..logged(py, &locals, asynchronous)
|
||||
};
|
||||
|
|
@ -127,7 +131,7 @@ fn internal_async_calls_skip_the_async_success_fan_out() {
|
|||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(py, c"response = object()");
|
||||
let mut logging = LegacyLogging {
|
||||
let mut logging = LegacyPythonLifecycle {
|
||||
internal: true,
|
||||
..logged(py, &locals, true)
|
||||
};
|
||||
|
|
|
|||
27
litellm-rust/crates/callbacks-v1-python/AGENTS.md
Normal file
27
litellm-rust/crates/callbacks-v1-python/AGENTS.md
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
- Target invariants, not completion claims
|
||||
- Keep this crate the Python runtime adapter for `litellm-callbacks-v1` and nothing else: the subscriber snapshot and one `PythonLifecycle` that delivers the core contract
|
||||
- Envelopes, sequencing, redaction, handler selection and wire patch validation belong to `litellm-callbacks-v1`
|
||||
- `litellm/rust_bridge/callbacks_v1_python.py` is the only Python module behind it; `python_contract.json` pins its functions on both sides
|
||||
- Terms (contract terms are in `callbacks-v1/AGENTS.md`, lifecycle terms in `host-python/AGENTS.md`)
|
||||
- Registry: the process-wide subscriber list; it lives in Python (`register`/`unregister`), never in Rust
|
||||
- Snapshot: that list read once per call, with one observer and one interceptor handler already selected per subscriber for the call's mode (`subscribers.rs`)
|
||||
- Projection: the public response turned into JSON by `project_response`; a failed projection becomes `response_error`, never a failed call
|
||||
- Report: logging an observer's swallowed error through `report`
|
||||
- Continuation: what a step returns to the driver once every observer of its envelope has run
|
||||
- "Callback" in the Python module (`Callback` protocol) is the user's object; once registered it is a subscriber
|
||||
- No Python object of the call reaches a callback; a callback influences a call only through a patch Rust validates
|
||||
- `begin` returns the dict it was given by identity; `after_success` and `delivered` stay the trait's pass-through defaults
|
||||
- Every callback gets a freshly built envelope per event; never cache or share a converted one
|
||||
- Contract values cross into Python only through a fresh conversion per callback
|
||||
- Schema v1 is owned by `litellm-callbacks-v1`; this crate only marshals it
|
||||
- Everything rejectable is rejected at `register`, never during a call; the snapshot only parses and the lifecycle never re-validates a subscriber
|
||||
- Observers cannot affect a call: ordinary exceptions are reported and swallowed at every event, awaited or not; an interceptor failure fails the call
|
||||
- A cancellation is returned at once, with no further dispatch and no report
|
||||
- Exactly one terminal envelope, last; none after a cancellation
|
||||
- In a sync call no step suspends: handler selection happens once, at snapshot, from what registration recorded
|
||||
- Every suspension is awaited inline in the caller's task; never `into_future`, never spawn
|
||||
- The subscriber snapshot is taken once per call before the driver starts; later registrations do not reach that call
|
||||
- `V1PythonLifecycle::subscribed` is the only constructor callers get; it returns `None` for an empty snapshot, so an unobserved call builds no envelope and projects no response
|
||||
- This lifecycle may run after another one in a `LifecycleChain`; it observes the arguments and the wire as that one left them and never assumes it is alone
|
||||
- `close` is idempotent; `traverse` visits every retained Python edge and calls no Python
|
||||
- This crate has none of `callbacks-legacy-python`'s obligations and must not grow them; shared behaviour belongs to `PythonLifecycle`, not to either adapter
|
||||
18
litellm-rust/crates/callbacks-v1-python/Cargo.toml
Normal file
18
litellm-rust/crates/callbacks-v1-python/Cargo.toml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
[package]
|
||||
name = "litellm-callbacks-v1-python"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
autotests = false
|
||||
|
||||
[dependencies]
|
||||
litellm-host.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
litellm-callbacks-v1.workspace = true
|
||||
pyo3.workspace = true
|
||||
serde_json.workspace = true
|
||||
strum.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-auth.workspace = true
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"snapshot": [],
|
||||
"project_response": ["response"],
|
||||
"new_call_id": [],
|
||||
"report": ["name", "event", "error"]
|
||||
}
|
||||
359
litellm-rust/crates/callbacks-v1-python/src/adapter.rs
Normal file
359
litellm-rust/crates/callbacks-v1-python/src/adapter.rs
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
//! The v1 lifecycle: each step builds one envelope, hands every subscribed observer a
|
||||
//! fresh conversion of it, and, before the request is sent, folds every interceptor's
|
||||
//! patch into the wire request.
|
||||
|
||||
use litellm_callbacks_v1::{
|
||||
CallFacts, Envelope, ErrorFacts, Event, RequestFacts, Sequencer, WirePatch, apply,
|
||||
};
|
||||
use litellm_host::event::{MachineEvent, RequestContext, WireRequest};
|
||||
use litellm_host_python::{
|
||||
LifecycleEvent, LifecycleStep, PythonLifecycle, from_py, is_cancellation, missing_state, to_py,
|
||||
};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyString};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::call::V1PythonSurface;
|
||||
use crate::python::V1Python;
|
||||
use crate::subscribers::Subscriber;
|
||||
|
||||
/// What a step hands back to the driver once every observer of its envelope has run.
|
||||
enum Continuation {
|
||||
Done,
|
||||
Arguments(Py<PyDict>),
|
||||
Wire(Box<WireRequest>),
|
||||
}
|
||||
|
||||
/// The suspended step; `index` is the subscriber whose awaitable the driver is awaiting.
|
||||
enum Pending {
|
||||
Observe {
|
||||
envelope: Envelope,
|
||||
index: usize,
|
||||
then: Continuation,
|
||||
},
|
||||
Intercept {
|
||||
wire: Box<WireRequest>,
|
||||
context: RequestContext,
|
||||
index: usize,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct V1PythonLifecycle {
|
||||
surface: V1PythonSurface,
|
||||
subscribers: Vec<Subscriber>,
|
||||
asynchronous: bool,
|
||||
sequencer: Option<Sequencer>,
|
||||
streamed: bool,
|
||||
pending: Option<Pending>,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
impl Continuation {
|
||||
fn into_step(self) -> LifecycleStep {
|
||||
match self {
|
||||
Self::Done => LifecycleStep::Done,
|
||||
Self::Arguments(arguments) => LifecycleStep::Arguments(arguments),
|
||||
Self::Wire(wire) => LifecycleStep::Wire(wire),
|
||||
}
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
match self {
|
||||
Self::Arguments(arguments) => visit.call(arguments),
|
||||
Self::Done | Self::Wire(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl V1PythonLifecycle {
|
||||
pub(crate) fn new(
|
||||
surface: V1PythonSurface,
|
||||
subscribers: Vec<Subscriber>,
|
||||
asynchronous: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
surface,
|
||||
subscribers,
|
||||
asynchronous,
|
||||
sequencer: None,
|
||||
streamed: false,
|
||||
pending: None,
|
||||
closed: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn envelope(&mut self, event: Event) -> PyResult<Envelope> {
|
||||
self.sequencer
|
||||
.as_mut()
|
||||
.map(|sequencer| sequencer.envelope(event))
|
||||
.ok_or_else(missing_state)
|
||||
}
|
||||
|
||||
fn report(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
index: usize,
|
||||
envelope: &Envelope,
|
||||
error: PyErr,
|
||||
) -> PyResult<()> {
|
||||
let name = &self.subscribers[index].name;
|
||||
let event: &'static str = envelope.event.kind().into();
|
||||
match V1Python::Report.call(py, (name, event, error.value(py))) {
|
||||
Err(report_error) if is_cancellation(py, &report_error) => Err(report_error),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
envelope: Envelope,
|
||||
from: usize,
|
||||
then: Continuation,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
for index in from..self.subscribers.len() {
|
||||
let Some(handler) = self.subscribers[index]
|
||||
.observer(envelope.event.kind())
|
||||
.map(|handler| handler.clone_ref(py))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let outcome = to_py(py, &envelope).and_then(|argument| handler.call(py, argument));
|
||||
match outcome {
|
||||
Ok(awaitable) if handler.is_async() => {
|
||||
self.pending = Some(Pending::Observe {
|
||||
envelope,
|
||||
index,
|
||||
then,
|
||||
});
|
||||
return Ok(LifecycleStep::Await(awaitable.unbind()));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
Err(error) => self.report(py, index, &envelope, error)?,
|
||||
}
|
||||
}
|
||||
Ok(then.into_step())
|
||||
}
|
||||
|
||||
fn apply_patch(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
index: usize,
|
||||
wire: WireRequest,
|
||||
result: Py<PyAny>,
|
||||
) -> PyResult<WireRequest> {
|
||||
let patch = if result.bind(py).is_none() {
|
||||
WirePatch::default()
|
||||
} else {
|
||||
from_py::<WirePatch>(result.bind(py))?
|
||||
};
|
||||
let name = &self.subscribers[index].name;
|
||||
apply(wire, patch)
|
||||
.map_err(|error| PyValueError::new_err(format!("callback {name}: {error}")))
|
||||
}
|
||||
|
||||
fn intercept(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
wire: Box<WireRequest>,
|
||||
context: RequestContext,
|
||||
from: usize,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let mut current = wire;
|
||||
for index in from..self.subscribers.len() {
|
||||
let Some(handler) = self.subscribers[index]
|
||||
.intercept
|
||||
.as_ref()
|
||||
.map(|handler| handler.clone_ref(py))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let request = to_py(py, &RequestFacts::new(¤t, &context))?;
|
||||
let result = handler.call(py, request)?;
|
||||
if handler.is_async() {
|
||||
self.pending = Some(Pending::Intercept {
|
||||
wire: current,
|
||||
context,
|
||||
index,
|
||||
});
|
||||
return Ok(LifecycleStep::Await(result.unbind()));
|
||||
}
|
||||
current = Box::new(self.apply_patch(py, index, *current, result.unbind())?);
|
||||
}
|
||||
let envelope =
|
||||
self.envelope(Event::RequestSending(RequestFacts::new(¤t, &context)))?;
|
||||
self.observe(py, envelope, 0, Continuation::Wire(current))
|
||||
}
|
||||
|
||||
fn metadata(arguments: &Bound<'_, PyDict>) -> PyResult<(Map<String, Value>, Vec<String>)> {
|
||||
let Some(value) = arguments.get_item("metadata")? else {
|
||||
return Ok((Map::new(), Vec::new()));
|
||||
};
|
||||
let Ok(metadata) = value.cast::<PyDict>() else {
|
||||
return Ok((Map::new(), vec!["metadata".to_string()]));
|
||||
};
|
||||
let mut included = Map::new();
|
||||
let mut dropped = Vec::new();
|
||||
for (key, value) in metadata.iter() {
|
||||
match key.extract::<String>() {
|
||||
Ok(name) => match from_py::<Value>(&value) {
|
||||
Ok(value) => {
|
||||
included.insert(name, value);
|
||||
}
|
||||
Err(_) => dropped.push(name),
|
||||
},
|
||||
Err(_) => dropped.push(key.str()?.to_string()),
|
||||
}
|
||||
}
|
||||
Ok((included, dropped))
|
||||
}
|
||||
|
||||
fn call_id(py: Python<'_>, arguments: &Bound<'_, PyDict>) -> PyResult<String> {
|
||||
if let Some(value) = arguments.get_item("litellm_call_id")?
|
||||
&& value.is_instance_of::<PyString>()
|
||||
{
|
||||
return value.extract();
|
||||
}
|
||||
V1Python::NewCallId.call(py, ())?.extract()
|
||||
}
|
||||
|
||||
fn error_facts(py: Python<'_>, error: &PyErr) -> PyResult<ErrorFacts> {
|
||||
let class = error.get_type(py);
|
||||
let module = class.getattr("__module__")?.extract::<String>()?;
|
||||
let qualified = class.getattr("__qualname__")?.extract::<String>()?;
|
||||
let status_code = error
|
||||
.value(py)
|
||||
.getattr("status_code")
|
||||
.ok()
|
||||
.and_then(|value| value.extract::<u16>().ok());
|
||||
Ok(ErrorFacts {
|
||||
class: format!("{module}.{qualified}"),
|
||||
message: error.value(py).str()?.to_string(),
|
||||
status_code,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonLifecycle for V1PythonLifecycle {
|
||||
fn begin(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
start_time: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let call_id = Self::call_id(py, arguments.bind(py))?;
|
||||
let (metadata, metadata_dropped) = Self::metadata(arguments.bind(py))?;
|
||||
self.sequencer = Some(Sequencer::new(call_id, self.surface.call_type));
|
||||
let envelope = self.envelope(Event::CallStarted(CallFacts {
|
||||
start_time,
|
||||
asynchronous: self.asynchronous,
|
||||
metadata,
|
||||
metadata_dropped,
|
||||
}))?;
|
||||
self.observe(py, envelope, 0, Continuation::Arguments(arguments))
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
wire: Box<WireRequest>,
|
||||
context: &RequestContext,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.intercept(py, wire, context.clone(), 0)
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
match event {
|
||||
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
|
||||
let envelope = self.envelope(Event::ResponseReceived {
|
||||
body: raw.body.clone(),
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
}
|
||||
LifecycleEvent::Succeeded { timing, response } => {
|
||||
let projected = V1Python::ProjectResponse
|
||||
.call(py, (response,))
|
||||
.and_then(|value| from_py::<Value>(&value));
|
||||
let (response, response_error) = match projected {
|
||||
Ok(response) => (response, None),
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
Err(error) => (Value::Null, Some(error.to_string())),
|
||||
};
|
||||
let envelope = self.envelope(Event::CallSucceeded {
|
||||
timing: timing.into(),
|
||||
streamed: self.streamed,
|
||||
response,
|
||||
response_error,
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
}
|
||||
LifecycleEvent::Failed {
|
||||
timing,
|
||||
origin,
|
||||
error,
|
||||
} => {
|
||||
let facts = Self::error_facts(py, error)?;
|
||||
let envelope = self.envelope(Event::CallFailed {
|
||||
timing: timing.into(),
|
||||
streamed: self.streamed,
|
||||
origin: origin.into(),
|
||||
error: facts,
|
||||
})?;
|
||||
self.observe(py, envelope, 0, Continuation::Done)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn opened(&mut self, _py: Python<'_>) -> PyResult<()> {
|
||||
self.streamed = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
|
||||
match self.pending.take().ok_or_else(missing_state)? {
|
||||
Pending::Observe {
|
||||
envelope,
|
||||
index,
|
||||
then,
|
||||
} => {
|
||||
if let Err(error) = result {
|
||||
if is_cancellation(py, &error) {
|
||||
return Err(error);
|
||||
}
|
||||
self.report(py, index, &envelope, error)?;
|
||||
}
|
||||
self.observe(py, envelope, index + 1, then)
|
||||
}
|
||||
Pending::Intercept {
|
||||
wire,
|
||||
context,
|
||||
index,
|
||||
} => {
|
||||
let patched = self.apply_patch(py, index, *wire, result?)?;
|
||||
self.intercept(py, Box::new(patched), context, index + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&mut self, _py: Python<'_>) {
|
||||
if self.closed {
|
||||
return;
|
||||
}
|
||||
self.closed = true;
|
||||
self.pending = None;
|
||||
self.subscribers.clear();
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
for subscriber in &self.subscribers {
|
||||
subscriber.traverse(visit)?;
|
||||
}
|
||||
match &self.pending {
|
||||
Some(Pending::Observe { then, .. }) => then.traverse(visit),
|
||||
Some(Pending::Intercept { .. }) | None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
25
litellm-rust/crates/callbacks-v1-python/src/call.rs
Normal file
25
litellm-rust/crates/callbacks-v1-python/src/call.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
//! What a route tells the v1 contract about itself, and the one constructor callers get.
|
||||
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::adapter::V1PythonLifecycle;
|
||||
use crate::subscribers::snapshot;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct V1PythonSurface {
|
||||
pub call_type: &'static str,
|
||||
}
|
||||
|
||||
impl V1PythonLifecycle {
|
||||
/// The lifecycle for one call, over the subscribers registered right now; later
|
||||
/// registrations do not reach that call. `None` when nothing is subscribed, so an
|
||||
/// unobserved call builds no envelope at all.
|
||||
pub fn subscribed(
|
||||
py: Python<'_>,
|
||||
surface: V1PythonSurface,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Option<Self>> {
|
||||
let subscribers = snapshot(py, asynchronous)?;
|
||||
Ok((!subscribers.is_empty()).then(|| Self::new(surface, subscribers, asynchronous)))
|
||||
}
|
||||
}
|
||||
20
litellm-rust/crates/callbacks-v1-python/src/lib.rs
Normal file
20
litellm-rust/crates/callbacks-v1-python/src/lib.rs
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
//! The Python adapter of the v1 callback contract: the per-call subscriber snapshot and
|
||||
//! one [`PythonLifecycle`](litellm_host_python::PythonLifecycle) that marshals
|
||||
//! `litellm-callbacks-v1` envelopes and patches to Python handlers. The contract itself
|
||||
//! lives in `litellm-callbacks-v1`.
|
||||
|
||||
mod adapter;
|
||||
mod call;
|
||||
mod python;
|
||||
mod subscribers;
|
||||
|
||||
pub use adapter::V1PythonLifecycle;
|
||||
pub use call::V1PythonSurface;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../tests/lifecycle.rs"]
|
||||
mod lifecycle_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../tests/support.rs"]
|
||||
mod test_support;
|
||||
56
litellm-rust/crates/callbacks-v1-python/src/python.rs
Normal file
56
litellm-rust/crates/callbacks-v1-python/src/python.rs
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
//! The Python contract of this crate: every function of
|
||||
//! `litellm/rust_bridge/callbacks_v1_python.py` that Rust calls, by name.
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use strum::{IntoStaticStr, VariantArray};
|
||||
|
||||
const MODULE: &str = "litellm.rust_bridge.callbacks_v1_python";
|
||||
|
||||
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)]
|
||||
pub(crate) enum V1Python {
|
||||
#[strum(serialize = "snapshot")]
|
||||
Snapshot,
|
||||
#[strum(serialize = "project_response")]
|
||||
ProjectResponse,
|
||||
#[strum(serialize = "new_call_id")]
|
||||
NewCallId,
|
||||
#[strum(serialize = "report")]
|
||||
Report,
|
||||
}
|
||||
|
||||
impl V1Python {
|
||||
fn name(self) -> &'static str {
|
||||
self.into()
|
||||
}
|
||||
|
||||
pub(crate) fn call<'py, A>(self, py: Python<'py>, args: A) -> PyResult<Bound<'py, PyAny>>
|
||||
where
|
||||
A: pyo3::call::PyCallArgs<'py>,
|
||||
{
|
||||
py.import(MODULE)?.getattr(self.name())?.call1(args)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use strum::VariantArray;
|
||||
|
||||
use super::V1Python;
|
||||
use crate::test_support::PYTHON_CONTRACT;
|
||||
|
||||
#[test]
|
||||
fn every_function_is_in_the_python_contract() {
|
||||
let contract: serde_json::Map<String, serde_json::Value> =
|
||||
serde_json::from_str(PYTHON_CONTRACT).unwrap();
|
||||
let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect();
|
||||
let called: Vec<&str> = V1Python::VARIANTS
|
||||
.iter()
|
||||
.copied()
|
||||
.map(V1Python::name)
|
||||
.collect();
|
||||
assert_eq!(called.len(), declared.len(), "a function is listed twice");
|
||||
assert_eq!(called.into_iter().collect::<BTreeSet<_>>(), declared);
|
||||
}
|
||||
}
|
||||
113
litellm-rust/crates/callbacks-v1-python/src/subscribers.rs
Normal file
113
litellm-rust/crates/callbacks-v1-python/src/subscribers.rs
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
//! The per-call subscriber snapshot: what Python's `register` recorded, read once, with
|
||||
//! one handler chosen per role for the call's execution mode. `register` has already
|
||||
//! rejected everything rejectable, so this only parses.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_callbacks_v1::{EventKind, ExecutionMode, HandlerSelection, select_handler};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::python::V1Python;
|
||||
|
||||
/// One callable of a subscriber, tagged with whether calling it returns an awaitable.
|
||||
pub enum Handler {
|
||||
Sync(Py<PyAny>),
|
||||
Async(Py<PyAny>),
|
||||
}
|
||||
|
||||
pub struct Subscriber {
|
||||
pub name: String,
|
||||
pub events: BTreeSet<EventKind>,
|
||||
/// `on_event` or `async_on_event`: cannot affect the call.
|
||||
pub observe: Option<Handler>,
|
||||
/// `before_send` or `async_before_send`: returns a patch; its failure fails the call.
|
||||
pub intercept: Option<Handler>,
|
||||
}
|
||||
|
||||
fn selected_handler(
|
||||
subscriber: &Bound<'_, PyAny>,
|
||||
mode: ExecutionMode,
|
||||
sync_name: &str,
|
||||
async_name: &str,
|
||||
) -> PyResult<Option<Handler>> {
|
||||
let sync = subscriber.getattr(sync_name)?;
|
||||
let asynchronous = subscriber.getattr(async_name)?;
|
||||
Ok(
|
||||
match select_handler(mode, !sync.is_none(), !asynchronous.is_none()) {
|
||||
HandlerSelection::Sync => Some(Handler::Sync(sync.unbind())),
|
||||
HandlerSelection::Async => Some(Handler::Async(asynchronous.unbind())),
|
||||
HandlerSelection::Skip => None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn snapshot(py: Python<'_>, asynchronous: bool) -> PyResult<Vec<Subscriber>> {
|
||||
let mode = if asynchronous {
|
||||
ExecutionMode::Async
|
||||
} else {
|
||||
ExecutionMode::Sync
|
||||
};
|
||||
V1Python::Snapshot
|
||||
.call(py, ())?
|
||||
.try_iter()?
|
||||
.map(|item| {
|
||||
let subscriber = item?;
|
||||
let events = subscriber
|
||||
.getattr("events")?
|
||||
.extract::<BTreeSet<String>>()?
|
||||
.into_iter()
|
||||
.map(|name| {
|
||||
name.parse::<EventKind>().map_err(|_| {
|
||||
PyValueError::new_err(format!("unknown callback event: {name}"))
|
||||
})
|
||||
})
|
||||
.collect::<PyResult<BTreeSet<_>>>()?;
|
||||
Ok(Subscriber {
|
||||
name: subscriber.getattr("name")?.extract()?,
|
||||
events,
|
||||
observe: selected_handler(&subscriber, mode, "on_event", "async_on_event")?,
|
||||
intercept: selected_handler(&subscriber, mode, "before_send", "async_before_send")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Subscriber {
|
||||
pub fn observer(&self, kind: EventKind) -> Option<&Handler> {
|
||||
self.observe
|
||||
.as_ref()
|
||||
.filter(|_| self.events.contains(&kind))
|
||||
}
|
||||
|
||||
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
for handler in self.observe.iter().chain(&self.intercept) {
|
||||
visit.call(handler.object())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler {
|
||||
fn object(&self) -> &Py<PyAny> {
|
||||
match self {
|
||||
Self::Sync(value) | Self::Async(value) => value,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_async(&self) -> bool {
|
||||
matches!(self, Self::Async(_))
|
||||
}
|
||||
|
||||
pub fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
match self {
|
||||
Self::Sync(value) => Self::Sync(value.clone_ref(py)),
|
||||
Self::Async(value) => Self::Async(value.clone_ref(py)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn call<'py>(&self, py: Python<'py>, argument: Py<PyAny>) -> PyResult<Bound<'py, PyAny>> {
|
||||
self.object().bind(py).call1((argument,))
|
||||
}
|
||||
}
|
||||
497
litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs
Normal file
497
litellm-rust/crates/callbacks-v1-python/tests/lifecycle.rs
Normal file
|
|
@ -0,0 +1,497 @@
|
|||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_auth::SecretValue;
|
||||
use litellm_host::event::{MachineEvent, RawResponse, RequestContext, Timing, WireRequest};
|
||||
use litellm_host_python::{LifecycleEvent, LifecycleStep, PythonLifecycle, from_py};
|
||||
use pyo3::exceptions::asyncio::CancelledError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::adapter::V1PythonLifecycle;
|
||||
use crate::call::V1PythonSurface;
|
||||
use crate::subscribers::{Handler, Subscriber};
|
||||
use crate::test_support::{local, namespace};
|
||||
use litellm_callbacks_v1::EventKind;
|
||||
|
||||
const TIMING: Timing = Timing {
|
||||
start_time: 10.0,
|
||||
end_time: 12.5,
|
||||
};
|
||||
|
||||
fn subscriber(
|
||||
locals: &Bound<'_, PyDict>,
|
||||
name: &str,
|
||||
events: &[EventKind],
|
||||
observe: Option<(&str, bool)>,
|
||||
intercept: Option<(&str, bool)>,
|
||||
) -> Subscriber {
|
||||
let handler = |spec: Option<(&str, bool)>| {
|
||||
spec.map(|(local_name, asynchronous)| {
|
||||
let callable = local(locals, local_name).unbind();
|
||||
if asynchronous {
|
||||
Handler::Async(callable)
|
||||
} else {
|
||||
Handler::Sync(callable)
|
||||
}
|
||||
})
|
||||
};
|
||||
Subscriber {
|
||||
name: name.to_string(),
|
||||
events: events.iter().copied().collect::<BTreeSet<_>>(),
|
||||
observe: handler(observe),
|
||||
intercept: handler(intercept),
|
||||
}
|
||||
}
|
||||
|
||||
fn lifecycle(subscribers: Vec<Subscriber>, asynchronous: bool) -> V1PythonLifecycle {
|
||||
V1PythonLifecycle::new(
|
||||
V1PythonSurface { call_type: "ocr" },
|
||||
subscribers,
|
||||
asynchronous,
|
||||
)
|
||||
}
|
||||
|
||||
fn context(secret: &str) -> RequestContext {
|
||||
RequestContext {
|
||||
model: "model".to_string(),
|
||||
custom_llm_provider: "provider".to_string(),
|
||||
optional_params: json!({"api_key": secret, "temperature": 0.2}),
|
||||
secret_fields: vec!["api_key".to_string()],
|
||||
api_key: Some(SecretValue::new(secret.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
fn wire(secret: &str) -> WireRequest {
|
||||
WireRequest {
|
||||
url: "https://provider.example/v1".to_string(),
|
||||
headers: vec![
|
||||
("Authorization".to_string(), secret.to_string()),
|
||||
("X-Trace".to_string(), "one".to_string()),
|
||||
],
|
||||
body: json!({"input": "hello"}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_sync_success_preserves_identity_sequence_and_secrets() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
import copy
|
||||
seen = []
|
||||
def observe(event):
|
||||
seen.append(copy.deepcopy(event))
|
||||
response = {'id': 'response'}
|
||||
",
|
||||
);
|
||||
let events = [
|
||||
EventKind::CallStarted,
|
||||
EventKind::RequestSending,
|
||||
EventKind::ResponseReceived,
|
||||
EventKind::CallSucceeded,
|
||||
];
|
||||
let observer = subscriber(
|
||||
&locals,
|
||||
"sync-success",
|
||||
&events,
|
||||
Some(("observe", false)),
|
||||
None,
|
||||
);
|
||||
let mut adapter = lifecycle(vec![observer], false);
|
||||
let arguments = PyDict::new(py);
|
||||
arguments.set_item("litellm_call_id", "call-123").unwrap();
|
||||
let LifecycleStep::Arguments(returned) =
|
||||
adapter.begin(py, arguments.clone().unbind(), 10.0).unwrap()
|
||||
else {
|
||||
panic!("begin did not return arguments");
|
||||
};
|
||||
assert!(returned.bind(py).is(&arguments));
|
||||
|
||||
let secret = "sentinel-secret-78e4";
|
||||
let LifecycleStep::Wire(sent) = adapter
|
||||
.before_send(py, Box::new(wire(secret)), &context(secret))
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("before_send did not return wire");
|
||||
};
|
||||
assert_eq!(sent.headers[0].1, secret);
|
||||
let raw = MachineEvent::ResponseReceived {
|
||||
raw: RawResponse {
|
||||
body: "raw response".to_string(),
|
||||
},
|
||||
};
|
||||
assert!(matches!(
|
||||
adapter.emit(py, LifecycleEvent::Machine(&raw)).unwrap(),
|
||||
LifecycleStep::Done
|
||||
));
|
||||
let response = local(&locals, "response").unbind();
|
||||
let LifecycleStep::Response(returned) = adapter
|
||||
.after_success(py, response.clone_ref(py), TIMING)
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("after_success did not return response");
|
||||
};
|
||||
assert!(returned.bind(py).is(response.bind(py)));
|
||||
assert!(matches!(
|
||||
adapter
|
||||
.emit(
|
||||
py,
|
||||
LifecycleEvent::Succeeded {
|
||||
timing: TIMING,
|
||||
response: &response,
|
||||
},
|
||||
)
|
||||
.unwrap(),
|
||||
LifecycleStep::Done
|
||||
));
|
||||
|
||||
let seen = from_py::<Vec<Value>>(&local(&locals, "seen")).unwrap();
|
||||
assert_eq!(
|
||||
seen.iter()
|
||||
.map(|envelope| envelope["event"]["type"].as_str().unwrap())
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
"call.started",
|
||||
"request.sending",
|
||||
"response.received",
|
||||
"call.succeeded",
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
seen.iter()
|
||||
.map(|envelope| envelope["seq"].as_u64().unwrap())
|
||||
.collect::<Vec<_>>(),
|
||||
[0, 1, 2, 3]
|
||||
);
|
||||
assert!(!serde_json::to_string(&seen).unwrap().contains(secret));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_observer_gets_a_fresh_envelope() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
import copy
|
||||
seen = []
|
||||
def mutate(event):
|
||||
event['event']['body']['input'] = 'changed'
|
||||
def record(event):
|
||||
seen.append(copy.deepcopy(event))
|
||||
",
|
||||
);
|
||||
let event = [EventKind::RequestSending];
|
||||
let subscribers = vec![
|
||||
subscriber(&locals, "mutator", &event, Some(("mutate", false)), None),
|
||||
subscriber(&locals, "recorder", &event, Some(("record", false)), None),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
|
||||
let LifecycleStep::Wire(sent) = adapter
|
||||
.before_send(py, Box::new(wire("secret")), &context("secret"))
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("before_send did not return wire");
|
||||
};
|
||||
assert_eq!(sent.body, json!({"input": "hello"}));
|
||||
let seen = from_py::<Vec<Value>>(&local(&locals, "seen")).unwrap();
|
||||
assert_eq!(seen[0]["event"]["body"], json!({"input": "hello"}));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_observer_errors_are_reported_and_swallowed() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
seen = []
|
||||
def fail(event):
|
||||
raise ValueError('observer failed')
|
||||
def record(event):
|
||||
seen.append(event['event']['type'])
|
||||
",
|
||||
);
|
||||
let event = [EventKind::CallStarted];
|
||||
let subscribers = vec![
|
||||
subscriber(
|
||||
&locals,
|
||||
"observer-error-401",
|
||||
&event,
|
||||
Some(("fail", false)),
|
||||
None,
|
||||
),
|
||||
subscriber(
|
||||
&locals,
|
||||
"after-error",
|
||||
&event,
|
||||
Some(("record", false)),
|
||||
None,
|
||||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
assert!(matches!(
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap(),
|
||||
LifecycleStep::Arguments(_)
|
||||
));
|
||||
assert_eq!(
|
||||
local(&locals, "seen").extract::<Vec<String>>().unwrap(),
|
||||
["call.started"]
|
||||
);
|
||||
let reports = py
|
||||
.import("litellm.rust_bridge.callbacks_v1_python")
|
||||
.unwrap()
|
||||
.getattr("reports")
|
||||
.unwrap();
|
||||
let names = reports
|
||||
.try_iter()
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|report| report.get_item(0).ok()?.extract::<String>().ok())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(names.contains(&"observer-error-401".to_string()));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_stops_observer_fanout() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
import asyncio
|
||||
seen = []
|
||||
def cancel(event):
|
||||
raise asyncio.CancelledError()
|
||||
def record(event):
|
||||
seen.append(event)
|
||||
",
|
||||
);
|
||||
let event = [EventKind::CallStarted];
|
||||
let subscribers = vec![
|
||||
subscriber(&locals, "cancel", &event, Some(("cancel", false)), None),
|
||||
subscriber(
|
||||
&locals,
|
||||
"after-cancel",
|
||||
&event,
|
||||
Some(("record", false)),
|
||||
None,
|
||||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
let error = match adapter.begin(py, PyDict::new(py).unbind(), 10.0) {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("cancellation was swallowed"),
|
||||
};
|
||||
assert!(error.is_instance_of::<CancelledError>(py));
|
||||
assert_eq!(local(&locals, "seen").len().unwrap(), 0);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn async_observers_resume_in_order_and_return_arguments() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
calls = []
|
||||
def first(event):
|
||||
calls.append('first')
|
||||
return object()
|
||||
def second(event):
|
||||
calls.append('second')
|
||||
return object()
|
||||
",
|
||||
);
|
||||
let event = [EventKind::CallStarted];
|
||||
let subscribers = vec![
|
||||
subscriber(&locals, "async-first", &event, Some(("first", true)), None),
|
||||
subscriber(
|
||||
&locals,
|
||||
"async-second",
|
||||
&event,
|
||||
Some(("second", true)),
|
||||
None,
|
||||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, true);
|
||||
let arguments = PyDict::new(py).unbind();
|
||||
assert!(matches!(
|
||||
adapter.begin(py, arguments.clone_ref(py), 10.0).unwrap(),
|
||||
LifecycleStep::Await(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
adapter.resume(py, Ok(py.None())).unwrap(),
|
||||
LifecycleStep::Await(_)
|
||||
));
|
||||
let LifecycleStep::Arguments(returned) = adapter.resume(py, Ok(py.None())).unwrap() else {
|
||||
panic!("resume did not return arguments");
|
||||
};
|
||||
assert!(returned.bind(py).is(arguments.bind(py)));
|
||||
assert_eq!(
|
||||
local(&locals, "calls").extract::<Vec<String>>().unwrap(),
|
||||
["first", "second"]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interceptors_chain_and_observers_see_the_final_wire() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
seen = []
|
||||
def first(request):
|
||||
return {'headers': {'set': [['X-Added', 'yes']]}, 'body': {'step': 1}}
|
||||
def second(request):
|
||||
seen.append(request)
|
||||
return None
|
||||
def observe(event):
|
||||
seen.append(event)
|
||||
",
|
||||
);
|
||||
let subscribers = vec![
|
||||
subscriber(&locals, "first", &[], None, Some(("first", false))),
|
||||
subscriber(&locals, "second", &[], None, Some(("second", false))),
|
||||
subscriber(
|
||||
&locals,
|
||||
"observer",
|
||||
&[EventKind::RequestSending],
|
||||
Some(("observe", false)),
|
||||
None,
|
||||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
|
||||
let LifecycleStep::Wire(sent) = adapter
|
||||
.before_send(py, Box::new(wire("secret")), &context("secret"))
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("before_send did not return wire");
|
||||
};
|
||||
assert_eq!(sent.body, json!({"step": 1}));
|
||||
assert_eq!(
|
||||
sent.headers.last().unwrap(),
|
||||
&("X-Added".to_string(), "yes".to_string())
|
||||
);
|
||||
let seen = from_py::<Vec<Value>>(&local(&locals, "seen")).unwrap();
|
||||
assert_eq!(seen[0]["body"], json!({"step": 1}));
|
||||
assert_eq!(seen[1]["event"]["body"], json!({"step": 1}));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_interceptor_patch_stops_later_handlers() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
seen = []
|
||||
def invalid(request):
|
||||
return {'headers': {'set': [['Authorization', 'replacement']]}}
|
||||
def later(request):
|
||||
seen.append('later')
|
||||
def observe(event):
|
||||
seen.append('observe')
|
||||
",
|
||||
);
|
||||
let subscribers = vec![
|
||||
subscriber(
|
||||
&locals,
|
||||
"invalid-patch",
|
||||
&[],
|
||||
None,
|
||||
Some(("invalid", false)),
|
||||
),
|
||||
subscriber(&locals, "later", &[], None, Some(("later", false))),
|
||||
subscriber(
|
||||
&locals,
|
||||
"observer",
|
||||
&[EventKind::RequestSending],
|
||||
Some(("observe", false)),
|
||||
None,
|
||||
),
|
||||
];
|
||||
let mut adapter = lifecycle(subscribers, false);
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
|
||||
let error = match adapter.before_send(py, Box::new(wire("secret")), &context("secret")) {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("invalid patch was accepted"),
|
||||
};
|
||||
assert!(error.to_string().contains("invalid-patch"));
|
||||
assert_eq!(local(&locals, "seen").len().unwrap(), 0);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_failure_is_observed_without_failing_terminal_dispatch() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
py,
|
||||
c"
|
||||
seen = []
|
||||
def observe(event):
|
||||
seen.append(event)
|
||||
class Broken:
|
||||
project_error = True
|
||||
response = Broken()
|
||||
",
|
||||
);
|
||||
let observer = subscriber(
|
||||
&locals,
|
||||
"terminal",
|
||||
&[EventKind::CallSucceeded],
|
||||
Some(("observe", false)),
|
||||
None,
|
||||
);
|
||||
let mut adapter = lifecycle(vec![observer], false);
|
||||
adapter.begin(py, PyDict::new(py).unbind(), 10.0).unwrap();
|
||||
adapter.opened(py).unwrap();
|
||||
let response = local(&locals, "response").unbind();
|
||||
assert!(matches!(
|
||||
adapter
|
||||
.emit(
|
||||
py,
|
||||
LifecycleEvent::Succeeded {
|
||||
timing: TIMING,
|
||||
response: &response,
|
||||
},
|
||||
)
|
||||
.unwrap(),
|
||||
LifecycleStep::Done
|
||||
));
|
||||
let seen = from_py::<Vec<Value>>(&local(&locals, "seen")).unwrap();
|
||||
assert_eq!(seen[0]["event"]["streamed"], true);
|
||||
assert!(seen[0]["event"]["response"].is_null());
|
||||
assert!(
|
||||
seen[0]["event"]["response_error"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("projection failed")
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_without_pending_fails_and_close_is_idempotent() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let _locals = namespace(py, c"");
|
||||
let mut adapter = lifecycle(Vec::new(), false);
|
||||
assert!(adapter.resume(py, Ok(py.None())).is_err());
|
||||
adapter.close(py);
|
||||
adapter.close(py);
|
||||
});
|
||||
}
|
||||
58
litellm-rust/crates/callbacks-v1-python/tests/support.rs
Normal file
58
litellm-rust/crates/callbacks-v1-python/tests/support.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
use std::ffi::CStr;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
|
||||
|
||||
const STUBS: &CStr = c"
|
||||
import inspect
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
|
||||
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_v1_python'):
|
||||
sys.modules.setdefault(name, types.ModuleType(name))
|
||||
|
||||
module = sys.modules['litellm.rust_bridge.callbacks_v1_python']
|
||||
contract = json.loads(python_contract)
|
||||
|
||||
def contracted(name, fake):
|
||||
signature = inspect.Signature([
|
||||
inspect.Parameter(parameter, inspect.Parameter.POSITIONAL_OR_KEYWORD)
|
||||
for parameter in contract[name]
|
||||
])
|
||||
def checked(*args, **kwargs):
|
||||
signature.bind(*args, **kwargs)
|
||||
return fake(*args, **kwargs)
|
||||
return checked
|
||||
|
||||
if not hasattr(module, 'reports'):
|
||||
module.reports = []
|
||||
|
||||
def project_response(response):
|
||||
if getattr(response, 'project_error', False):
|
||||
raise ValueError('projection failed')
|
||||
return response
|
||||
|
||||
functions = {
|
||||
'snapshot': lambda: (),
|
||||
'project_response': project_response,
|
||||
'new_call_id': lambda: 'generated-call-id',
|
||||
'report': lambda name, event, error: module.reports.append((name, event, error)),
|
||||
}
|
||||
for name, function in functions.items():
|
||||
setattr(module, name, contracted(name, function))
|
||||
";
|
||||
|
||||
pub(crate) fn namespace<'py>(py: Python<'py>, script: &CStr) -> Bound<'py, PyDict> {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("python_contract", PYTHON_CONTRACT).unwrap();
|
||||
py.run(STUBS, Some(&locals), Some(&locals)).unwrap();
|
||||
py.run(script, Some(&locals), Some(&locals)).unwrap();
|
||||
locals
|
||||
}
|
||||
|
||||
pub(crate) fn local<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py, PyAny> {
|
||||
locals.get_item(name).unwrap().unwrap()
|
||||
}
|
||||
17
litellm-rust/crates/callbacks-v1/AGENTS.md
Normal file
17
litellm-rust/crates/callbacks-v1/AGENTS.md
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
- Target invariants, not completion claims
|
||||
- Keep this crate the runtime-neutral v1 callback contract and nothing else: envelopes, sequencing, redaction, handler selection and validated wire patches
|
||||
- This crate may depend on `litellm-host`, but never on `litellm-host-python`, PyO3 or a runtime registry
|
||||
- `golden/v1` pins every serialized envelope shape; every `EventKind` has a golden file by construction, and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are tested against those files
|
||||
- Terms (shared ones are in `host-python/AGENTS.md`)
|
||||
- Subscriber: one registered consumer of the contract: a name, a schema, the event kinds it wants and its handlers
|
||||
- Handler: one callable of a subscriber; an observer receives envelopes and cannot affect the call; an interceptor receives `RequestFacts` and returns a patch
|
||||
- Handler selection (`select_handler`): an async call prefers the async handler and falls back to the sync one; a sync call never selects an async handler
|
||||
- Event: one thing that happened in a call; `EventKind` is its dotted name (`call.started`, `request.sending`, `response.received`, `call.succeeded`, `call.failed`), the serde tag and the subscription name at once
|
||||
- Envelope: schema + `call_id` + `call_type` + `seq` + event; the only value an observer receives; `Sequencer` numbers a call's envelopes from zero
|
||||
- Facts (`CallFacts`, `RequestFacts`, `TimingFacts`, `ErrorFacts`): a redacted, serializable projection of runtime state; never a live object
|
||||
- Patch (`WirePatch`): an interceptor's edit of the wire: header `remove`, then header `set`, optional `body` replacement; `apply` validates it
|
||||
- Redaction: credential headers by name (`CREDENTIAL_HEADERS`) and optional params by the route's secret fields become `[REDACTED]`; shape never changes
|
||||
- No runtime object of a call reaches a callback contract value
|
||||
- `RequestFacts::new` is the only request projection and always applies the contract's credential redaction; its fields are private so no other crate can build one
|
||||
- A patch cannot change the URL or set or remove a credential header
|
||||
- Schema v1 is additive-only; a rename, removal or retype requires a new contract version
|
||||
19
litellm-rust/crates/callbacks-v1/Cargo.toml
Normal file
19
litellm-rust/crates/callbacks-v1/Cargo.toml
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
[package]
|
||||
name = "litellm-callbacks-v1"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
autotests = false
|
||||
|
||||
[dependencies]
|
||||
litellm-host.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
strum.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-auth.workspace = true
|
||||
proptest.workspace = true
|
||||
rstest.workspace = true
|
||||
13
litellm-rust/crates/callbacks-v1/golden/v1/call.failed.json
Normal file
13
litellm-rust/crates/callbacks-v1/golden/v1/call.failed.json
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"schema": 1,
|
||||
"call_id": "call-123",
|
||||
"call_type": "ocr",
|
||||
"seq": 4,
|
||||
"event": {
|
||||
"type": "call.failed",
|
||||
"timing": {"start_time": 10.0, "end_time": 11.0, "duration": 1.0},
|
||||
"streamed": true,
|
||||
"origin": "call",
|
||||
"error": {"class": "builtins.ValueError", "message": "bad request", "status_code": 400}
|
||||
}
|
||||
}
|
||||
13
litellm-rust/crates/callbacks-v1/golden/v1/call.started.json
Normal file
13
litellm-rust/crates/callbacks-v1/golden/v1/call.started.json
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"schema": 1,
|
||||
"call_id": "call-123",
|
||||
"call_type": "ocr",
|
||||
"seq": 0,
|
||||
"event": {
|
||||
"type": "call.started",
|
||||
"start_time": 10.0,
|
||||
"asynchronous": true,
|
||||
"metadata": {"team": "core"},
|
||||
"metadata_dropped": ["broken"]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"schema": 1,
|
||||
"call_id": "call-123",
|
||||
"call_type": "ocr",
|
||||
"seq": 3,
|
||||
"event": {
|
||||
"type": "call.succeeded",
|
||||
"timing": {"start_time": 10.0, "end_time": 12.5, "duration": 2.5},
|
||||
"streamed": false,
|
||||
"response": {"id": "response"},
|
||||
"response_error": null
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"schema": 1,
|
||||
"call_id": "call-123",
|
||||
"call_type": "ocr",
|
||||
"seq": 1,
|
||||
"event": {
|
||||
"type": "request.sending",
|
||||
"model": "model",
|
||||
"custom_llm_provider": "provider",
|
||||
"optional_params": {"temperature": 0.2},
|
||||
"url": "https://provider.example/v1",
|
||||
"headers": [["Authorization", "[REDACTED]"]],
|
||||
"body": {"input": "hello"}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"schema": 1,
|
||||
"call_id": "call-123",
|
||||
"call_type": "ocr",
|
||||
"seq": 2,
|
||||
"event": {
|
||||
"type": "response.received",
|
||||
"body": "raw response"
|
||||
}
|
||||
}
|
||||
315
litellm-rust/crates/callbacks-v1/src/envelope.rs
Normal file
315
litellm-rust/crates/callbacks-v1/src/envelope.rs
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
//! The v1 envelope: what a subscriber receives. An [`Envelope`] is the schema version, the
|
||||
//! call's identity, a sequence number and one [`Event`]; every `*Facts` type is a
|
||||
//! redacted, serializable projection that never holds a runtime object of the call.
|
||||
|
||||
use litellm_host::event::{FailureOrigin, RequestContext, Timing, WireRequest};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
use strum::{EnumString, IntoStaticStr, VariantArray};
|
||||
|
||||
use crate::redact;
|
||||
|
||||
pub const SCHEMA_V1: u32 = 1;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct Envelope {
|
||||
pub schema: u32,
|
||||
pub call_id: String,
|
||||
pub call_type: &'static str,
|
||||
pub seq: u32,
|
||||
pub event: Event,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum Event {
|
||||
#[serde(rename = "call.started")]
|
||||
CallStarted(CallFacts),
|
||||
#[serde(rename = "request.sending")]
|
||||
RequestSending(RequestFacts),
|
||||
#[serde(rename = "response.received")]
|
||||
ResponseReceived { body: String },
|
||||
#[serde(rename = "call.succeeded")]
|
||||
CallSucceeded {
|
||||
timing: TimingFacts,
|
||||
streamed: bool,
|
||||
response: Value,
|
||||
response_error: Option<String>,
|
||||
},
|
||||
#[serde(rename = "call.failed")]
|
||||
CallFailed {
|
||||
timing: TimingFacts,
|
||||
streamed: bool,
|
||||
origin: Origin,
|
||||
error: ErrorFacts,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(
|
||||
EnumString,
|
||||
IntoStaticStr,
|
||||
VariantArray,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
Eq,
|
||||
Hash,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
Debug,
|
||||
)]
|
||||
pub enum EventKind {
|
||||
#[strum(serialize = "call.started")]
|
||||
CallStarted,
|
||||
#[strum(serialize = "request.sending")]
|
||||
RequestSending,
|
||||
#[strum(serialize = "response.received")]
|
||||
ResponseReceived,
|
||||
#[strum(serialize = "call.succeeded")]
|
||||
CallSucceeded,
|
||||
#[strum(serialize = "call.failed")]
|
||||
CallFailed,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct CallFacts {
|
||||
pub start_time: f64,
|
||||
pub asynchronous: bool,
|
||||
pub metadata: Map<String, Value>,
|
||||
pub metadata_dropped: Vec<String>,
|
||||
}
|
||||
|
||||
/// The request as a subscriber may see it. The fields are private so that
|
||||
/// [`RequestFacts::new`], which redacts, is the only way to build one.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct RequestFacts {
|
||||
model: String,
|
||||
custom_llm_provider: String,
|
||||
optional_params: Value,
|
||||
url: String,
|
||||
headers: Vec<(String, String)>,
|
||||
body: Value,
|
||||
}
|
||||
|
||||
impl RequestFacts {
|
||||
pub fn new(wire: &WireRequest, context: &RequestContext) -> Self {
|
||||
Self {
|
||||
model: context.model.clone(),
|
||||
custom_llm_provider: context.custom_llm_provider.clone(),
|
||||
optional_params: redact::params(&context.optional_params, &context.secret_fields),
|
||||
url: wire.url.clone(),
|
||||
headers: redact::headers(&wire.headers),
|
||||
body: wire.body.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
|
||||
pub struct TimingFacts {
|
||||
pub start_time: f64,
|
||||
pub end_time: f64,
|
||||
pub duration: f64,
|
||||
}
|
||||
|
||||
impl From<Timing> for TimingFacts {
|
||||
fn from(timing: Timing) -> Self {
|
||||
Self {
|
||||
start_time: timing.start_time,
|
||||
end_time: timing.end_time,
|
||||
duration: timing.end_time - timing.start_time,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Origin {
|
||||
Call,
|
||||
Host,
|
||||
}
|
||||
|
||||
impl From<FailureOrigin> for Origin {
|
||||
fn from(origin: FailureOrigin) -> Self {
|
||||
match origin {
|
||||
FailureOrigin::Call => Self::Call,
|
||||
FailureOrigin::Host => Self::Host,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct ErrorFacts {
|
||||
pub class: String,
|
||||
pub message: String,
|
||||
pub status_code: Option<u16>,
|
||||
}
|
||||
|
||||
impl Event {
|
||||
pub fn kind(&self) -> EventKind {
|
||||
match self {
|
||||
Self::CallStarted(_) => EventKind::CallStarted,
|
||||
Self::RequestSending(_) => EventKind::RequestSending,
|
||||
Self::ResponseReceived { .. } => EventKind::ResponseReceived,
|
||||
Self::CallSucceeded { .. } => EventKind::CallSucceeded,
|
||||
Self::CallFailed { .. } => EventKind::CallFailed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Sequencer {
|
||||
call_id: String,
|
||||
call_type: &'static str,
|
||||
next: u32,
|
||||
}
|
||||
|
||||
impl Sequencer {
|
||||
pub fn new(call_id: String, call_type: &'static str) -> Self {
|
||||
Self {
|
||||
call_id,
|
||||
call_type,
|
||||
next: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn envelope(&mut self, event: Event) -> Envelope {
|
||||
let envelope = Envelope {
|
||||
schema: SCHEMA_V1,
|
||||
call_id: self.call_id.clone(),
|
||||
call_type: self.call_type,
|
||||
seq: self.next,
|
||||
event,
|
||||
};
|
||||
self.next += 1;
|
||||
envelope
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_auth::SecretValue;
|
||||
use proptest::prelude::*;
|
||||
use serde_json::json;
|
||||
use strum::VariantArray;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn event(kind: EventKind) -> Event {
|
||||
match kind {
|
||||
EventKind::CallStarted => Event::CallStarted(CallFacts {
|
||||
start_time: 10.0,
|
||||
asynchronous: true,
|
||||
metadata: Map::from_iter([("team".to_string(), json!("core"))]),
|
||||
metadata_dropped: vec!["broken".to_string()],
|
||||
}),
|
||||
EventKind::RequestSending => Event::RequestSending(RequestFacts {
|
||||
model: "model".to_string(),
|
||||
custom_llm_provider: "provider".to_string(),
|
||||
optional_params: json!({"temperature": 0.2}),
|
||||
url: "https://provider.example/v1".to_string(),
|
||||
headers: vec![("Authorization".to_string(), redact::REDACTED.to_string())],
|
||||
body: json!({"input": "hello"}),
|
||||
}),
|
||||
EventKind::ResponseReceived => Event::ResponseReceived {
|
||||
body: "raw response".to_string(),
|
||||
},
|
||||
EventKind::CallSucceeded => Event::CallSucceeded {
|
||||
timing: TimingFacts {
|
||||
start_time: 10.0,
|
||||
end_time: 12.5,
|
||||
duration: 2.5,
|
||||
},
|
||||
streamed: false,
|
||||
response: json!({"id": "response"}),
|
||||
response_error: None,
|
||||
},
|
||||
EventKind::CallFailed => Event::CallFailed {
|
||||
timing: TimingFacts {
|
||||
start_time: 10.0,
|
||||
end_time: 11.0,
|
||||
duration: 1.0,
|
||||
},
|
||||
streamed: true,
|
||||
origin: Origin::Call,
|
||||
error: ErrorFacts {
|
||||
class: "builtins.ValueError".to_string(),
|
||||
message: "bad request".to_string(),
|
||||
status_code: Some(400),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn golden(kind: EventKind, seq: u32) -> Envelope {
|
||||
Envelope {
|
||||
schema: SCHEMA_V1,
|
||||
call_id: "call-123".to_string(),
|
||||
call_type: "ocr",
|
||||
seq,
|
||||
event: event(kind),
|
||||
}
|
||||
}
|
||||
|
||||
/// Exhaustive on purpose: a new event kind does not compile without a golden file,
|
||||
/// and Python's `EVENTS` and `SUPPORTED_SCHEMAS` are pinned to the golden files.
|
||||
fn golden_json(kind: EventKind) -> &'static str {
|
||||
match kind {
|
||||
EventKind::CallStarted => include_str!("../golden/v1/call.started.json"),
|
||||
EventKind::RequestSending => include_str!("../golden/v1/request.sending.json"),
|
||||
EventKind::ResponseReceived => include_str!("../golden/v1/response.received.json"),
|
||||
EventKind::CallSucceeded => include_str!("../golden/v1/call.succeeded.json"),
|
||||
EventKind::CallFailed => include_str!("../golden/v1/call.failed.json"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serialized_v1_envelopes_match_golden_contract() {
|
||||
for (seq, &kind) in EventKind::VARIANTS.iter().enumerate() {
|
||||
assert_eq!(
|
||||
serde_json::to_value(golden(kind, seq as u32)).unwrap(),
|
||||
serde_json::from_str::<Value>(golden_json(kind)).unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_tags_equal_subscription_names() {
|
||||
for &kind in EventKind::VARIANTS {
|
||||
let value = serde_json::to_value(event(kind)).unwrap();
|
||||
let expected: &'static str = kind.into();
|
||||
assert_eq!(value["type"], expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_facts_never_serialize_credentials() {
|
||||
let sentinel = "sentinel-secret-78e4";
|
||||
let wire = WireRequest {
|
||||
url: "https://provider.example".to_string(),
|
||||
headers: vec![("Authorization".to_string(), sentinel.to_string())],
|
||||
body: json!({"input": "safe"}),
|
||||
};
|
||||
let context = RequestContext {
|
||||
model: "model".to_string(),
|
||||
custom_llm_provider: "provider".to_string(),
|
||||
optional_params: json!({"api_key": sentinel}),
|
||||
secret_fields: vec!["api_key".to_string()],
|
||||
api_key: Some(SecretValue::new(sentinel.to_string())),
|
||||
};
|
||||
assert!(
|
||||
!serde_json::to_string(&RequestFacts::new(&wire, &context))
|
||||
.unwrap()
|
||||
.contains(sentinel)
|
||||
);
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn sequence_numbers_increase_from_zero(length in 0usize..100) {
|
||||
let mut sequencer = Sequencer::new("call".to_string(), "ocr");
|
||||
let sequence: Vec<u32> = (0..length)
|
||||
.map(|_| sequencer.envelope(event(EventKind::ResponseReceived)).seq)
|
||||
.collect();
|
||||
prop_assert_eq!(sequence, (0..length as u32).collect::<Vec<_>>());
|
||||
}
|
||||
}
|
||||
}
|
||||
15
litellm-rust/crates/callbacks-v1/src/lib.rs
Normal file
15
litellm-rust/crates/callbacks-v1/src/lib.rs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
//! The runtime-neutral v1 callback contract: envelopes and their sequencing, credential
|
||||
//! redaction, handler selection and validated wire patches. No PyO3 and no registry; a
|
||||
//! language host marshals these values to its own subscribers.
|
||||
|
||||
mod envelope;
|
||||
mod patch;
|
||||
mod policy;
|
||||
mod redact;
|
||||
|
||||
pub use envelope::{
|
||||
CallFacts, Envelope, ErrorFacts, Event, EventKind, Origin, RequestFacts, SCHEMA_V1, Sequencer,
|
||||
TimingFacts,
|
||||
};
|
||||
pub use patch::{HeaderPatch, PatchError, WirePatch, apply};
|
||||
pub use policy::{ExecutionMode, HandlerSelection, select_handler};
|
||||
209
litellm-rust/crates/callbacks-v1/src/patch.rs
Normal file
209
litellm-rust/crates/callbacks-v1/src/patch.rs
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
//! The v1 patch: the only way an interceptor changes a request. A [`WirePatch`] removes
|
||||
//! and sets headers and may replace the body; it cannot name the URL, and [`apply`]
|
||||
//! rejects any credential header.
|
||||
|
||||
use litellm_host::event::WireRequest;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::redact::is_credential_header;
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WirePatch {
|
||||
#[serde(default)]
|
||||
pub headers: HeaderPatch,
|
||||
#[serde(default)]
|
||||
pub body: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HeaderPatch {
|
||||
#[serde(default)]
|
||||
pub remove: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub set: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Error)]
|
||||
pub enum PatchError {
|
||||
#[error("protected credential header: {0}")]
|
||||
ProtectedHeader(String),
|
||||
#[error("invalid header name: {0}")]
|
||||
InvalidHeaderName(String),
|
||||
#[error("invalid header value for: {0}")]
|
||||
InvalidHeaderValue(String),
|
||||
}
|
||||
|
||||
fn valid_header_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
byte,
|
||||
b'!' | b'#'
|
||||
| b'$'
|
||||
| b'%'
|
||||
| b'&'
|
||||
| b'\''
|
||||
| b'*'
|
||||
| b'+'
|
||||
| b'-'
|
||||
| b'.'
|
||||
| b'^'
|
||||
| b'_'
|
||||
| b'`'
|
||||
| b'|'
|
||||
| b'~'
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_name(name: &str) -> Result<(), PatchError> {
|
||||
if is_credential_header(name) {
|
||||
return Err(PatchError::ProtectedHeader(name.to_string()));
|
||||
}
|
||||
if !valid_header_name(name) {
|
||||
return Err(PatchError::InvalidHeaderName(name.to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn apply(wire: WireRequest, patch: WirePatch) -> Result<WireRequest, PatchError> {
|
||||
for name in &patch.headers.remove {
|
||||
validate_name(name)?;
|
||||
}
|
||||
for (name, value) in &patch.headers.set {
|
||||
validate_name(name)?;
|
||||
if value
|
||||
.bytes()
|
||||
.any(|byte| matches!(byte, b'\r' | b'\n' | b'\0'))
|
||||
{
|
||||
return Err(PatchError::InvalidHeaderValue(name.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
let HeaderPatch { remove, set } = patch.headers;
|
||||
let retained: Vec<(String, String)> = wire
|
||||
.headers
|
||||
.into_iter()
|
||||
.filter(|(name, _)| {
|
||||
!remove
|
||||
.iter()
|
||||
.chain(set.iter().map(|(name, _)| name))
|
||||
.any(|changed| name.eq_ignore_ascii_case(changed))
|
||||
})
|
||||
.collect();
|
||||
let headers = retained.into_iter().chain(set).collect();
|
||||
Ok(WireRequest {
|
||||
url: wire.url,
|
||||
headers,
|
||||
body: patch.body.unwrap_or(wire.body),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use proptest::prelude::*;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn wire() -> WireRequest {
|
||||
WireRequest {
|
||||
url: "https://provider.example/v1".to_string(),
|
||||
headers: vec![
|
||||
("Authorization".to_string(), "secret".to_string()),
|
||||
("X-Trace".to_string(), "one".to_string()),
|
||||
("x-trace".to_string(), "two".to_string()),
|
||||
("Keep".to_string(), "yes".to_string()),
|
||||
],
|
||||
body: json!({"old": true}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_patch_is_identity() {
|
||||
let original = wire();
|
||||
assert_eq!(apply(original.clone(), WirePatch::default()), Ok(original));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_precedes_set_and_names_match_case_insensitively() {
|
||||
let patch = WirePatch {
|
||||
headers: HeaderPatch {
|
||||
remove: vec!["keep".to_string()],
|
||||
set: vec![("X-TRACE".to_string(), "three".to_string())],
|
||||
},
|
||||
body: Some(json!({"new": true})),
|
||||
};
|
||||
let result = apply(wire(), patch).unwrap();
|
||||
assert_eq!(
|
||||
result.headers,
|
||||
vec![
|
||||
("Authorization".to_string(), "secret".to_string()),
|
||||
("X-TRACE".to_string(), "three".to_string()),
|
||||
]
|
||||
);
|
||||
assert_eq!(result.body, json!({"new": true}));
|
||||
assert_eq!(result.url, "https://provider.example/v1");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(json!({"url": "https://attacker.example"}))]
|
||||
#[case(json!({"headers": {"append": []}}))]
|
||||
fn unknown_patch_fields_are_rejected(#[case] value: Value) {
|
||||
assert!(serde_json::from_value::<WirePatch>(value).is_err());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("Authorization", "value", PatchError::ProtectedHeader("Authorization".to_string()))]
|
||||
#[case("bad name", "value", PatchError::InvalidHeaderName("bad name".to_string()))]
|
||||
#[case("X-Test", "bad\r\nvalue", PatchError::InvalidHeaderValue("X-Test".to_string()))]
|
||||
fn invalid_set_is_rejected(
|
||||
#[case] name: &str,
|
||||
#[case] value: &str,
|
||||
#[case] expected: PatchError,
|
||||
) {
|
||||
let patch = WirePatch {
|
||||
headers: HeaderPatch {
|
||||
remove: vec![],
|
||||
set: vec![(name.to_string(), value.to_string())],
|
||||
},
|
||||
body: None,
|
||||
};
|
||||
assert_eq!(apply(wire(), patch), Err(expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_headers_cannot_be_removed() {
|
||||
let patch = WirePatch {
|
||||
headers: HeaderPatch {
|
||||
remove: vec!["AUTHORIZATION".to_string()],
|
||||
set: vec![],
|
||||
},
|
||||
body: None,
|
||||
};
|
||||
assert_eq!(
|
||||
apply(wire(), patch),
|
||||
Err(PatchError::ProtectedHeader("AUTHORIZATION".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn successful_patches_preserve_url(value in "[A-Za-z0-9]{0,20}") {
|
||||
let original = wire();
|
||||
let patch = WirePatch {
|
||||
headers: HeaderPatch { remove: vec![], set: vec![("X-Safe".to_string(), value)] },
|
||||
body: None,
|
||||
};
|
||||
let result = apply(original.clone(), patch).unwrap();
|
||||
prop_assert_eq!(result.url, original.url);
|
||||
prop_assert_eq!(result.headers[0].clone(), original.headers[0].clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
47
litellm-rust/crates/callbacks-v1/src/policy.rs
Normal file
47
litellm-rust/crates/callbacks-v1/src/policy.rs
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
//! Which of a subscriber's handlers a call uses: an async call prefers the async handler
|
||||
//! and falls back to the sync one; a sync call never selects an async handler.
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ExecutionMode {
|
||||
Sync,
|
||||
Async,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum HandlerSelection {
|
||||
Sync,
|
||||
Async,
|
||||
Skip,
|
||||
}
|
||||
|
||||
pub fn select_handler(mode: ExecutionMode, has_sync: bool, has_async: bool) -> HandlerSelection {
|
||||
match (mode, has_sync, has_async) {
|
||||
(ExecutionMode::Async, _, true) => HandlerSelection::Async,
|
||||
(_, true, _) => HandlerSelection::Sync,
|
||||
_ => HandlerSelection::Skip,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case(ExecutionMode::Async, true, true, HandlerSelection::Async)]
|
||||
#[case(ExecutionMode::Async, false, true, HandlerSelection::Async)]
|
||||
#[case(ExecutionMode::Async, true, false, HandlerSelection::Sync)]
|
||||
#[case(ExecutionMode::Sync, true, true, HandlerSelection::Sync)]
|
||||
#[case(ExecutionMode::Sync, true, false, HandlerSelection::Sync)]
|
||||
#[case(ExecutionMode::Sync, false, true, HandlerSelection::Skip)]
|
||||
#[case(ExecutionMode::Sync, false, false, HandlerSelection::Skip)]
|
||||
fn handler_selection_never_selects_async_for_sync_calls(
|
||||
#[case] mode: ExecutionMode,
|
||||
#[case] has_sync: bool,
|
||||
#[case] has_async: bool,
|
||||
#[case] expected: HandlerSelection,
|
||||
) {
|
||||
assert_eq!(select_handler(mode, has_sync, has_async), expected);
|
||||
}
|
||||
}
|
||||
104
litellm-rust/crates/callbacks-v1/src/redact.rs
Normal file
104
litellm-rust/crates/callbacks-v1/src/redact.rs
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
//! The contract's credential redaction: credential headers by name, optional params by
|
||||
//! the route's secret fields. Redaction replaces values and never changes shape.
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub const REDACTED: &str = "[REDACTED]";
|
||||
|
||||
pub const CREDENTIAL_HEADERS: &[&str] = &[
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"x-api-key",
|
||||
"api-key",
|
||||
"x-goog-api-key",
|
||||
"x-amz-security-token",
|
||||
"cookie",
|
||||
];
|
||||
|
||||
pub fn is_credential_header(name: &str) -> bool {
|
||||
CREDENTIAL_HEADERS
|
||||
.iter()
|
||||
.any(|credential| name.eq_ignore_ascii_case(credential))
|
||||
}
|
||||
|
||||
pub fn headers(values: &[(String, String)]) -> Vec<(String, String)> {
|
||||
values
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
let redacted = if is_credential_header(name) {
|
||||
REDACTED.to_string()
|
||||
} else {
|
||||
value.clone()
|
||||
};
|
||||
(name.clone(), redacted)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn params(value: &Value, secret_fields: &[String]) -> Value {
|
||||
let Value::Object(object) = value else {
|
||||
return value.clone();
|
||||
};
|
||||
Value::Object(
|
||||
object
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
let redacted = if secret_fields.iter().any(|secret| secret == name) {
|
||||
Value::String(REDACTED.to_string())
|
||||
} else {
|
||||
value.clone()
|
||||
};
|
||||
(name.clone(), redacted)
|
||||
})
|
||||
.collect::<Map<_, _>>(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use proptest::prelude::*;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn credential_header_matching_is_ascii_case_insensitive() {
|
||||
assert!(is_credential_header("AuThOrIzAtIoN"));
|
||||
assert!(!is_credential_header("x-request-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redaction_replaces_values_without_changing_shape() {
|
||||
let input = vec![
|
||||
("X-Request-ID".to_string(), "abc".to_string()),
|
||||
("Authorization".to_string(), "secret".to_string()),
|
||||
("authorization".to_string(), "other".to_string()),
|
||||
];
|
||||
assert_eq!(
|
||||
headers(&input),
|
||||
vec![
|
||||
("X-Request-ID".to_string(), "abc".to_string()),
|
||||
("Authorization".to_string(), REDACTED.to_string()),
|
||||
("authorization".to_string(), REDACTED.to_string()),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
params(
|
||||
&json!({"api_key": "secret", "model": "m"}),
|
||||
&["api_key".to_string()]
|
||||
),
|
||||
json!({"api_key": REDACTED, "model": "m"})
|
||||
);
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn redaction_is_idempotent(
|
||||
name in "[A-Za-z0-9-]{1,20}",
|
||||
value in "[A-Za-z0-9]{0,20}"
|
||||
) {
|
||||
let input = vec![(name, value)];
|
||||
prop_assert_eq!(headers(&headers(&input)), headers(&input));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ repository.workspace = true
|
|||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
litellm-types.workspace = true
|
||||
percent-encoding.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_path_to_error = "0.1"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use std::marker::PhantomData;
|
||||
|
||||
use percent_encoding::{AsciiSet, utf8_percent_encode};
|
||||
use url::Url;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
|
|
@ -64,6 +65,35 @@ impl ApiUrl<Complete> {
|
|||
self
|
||||
}
|
||||
|
||||
/// Like [`Self::append_query_pairs`], but percent-encodes keys and values with `escape`
|
||||
/// instead of form encoding, so callers choose which reserved characters stay literal.
|
||||
/// An existing query is kept; an empty result leaves no dangling `?`.
|
||||
pub fn append_query_pairs_escaped<'a>(
|
||||
mut self,
|
||||
pairs: impl IntoIterator<Item = (&'a str, &'a str)>,
|
||||
escape: &'static AsciiSet,
|
||||
) -> Self {
|
||||
let appended = pairs.into_iter().map(|(key, value)| {
|
||||
format!(
|
||||
"{}={}",
|
||||
utf8_percent_encode(key, escape),
|
||||
utf8_percent_encode(value, escape)
|
||||
)
|
||||
});
|
||||
let query = self
|
||||
.url
|
||||
.query()
|
||||
.filter(|existing| !existing.is_empty())
|
||||
.map(str::to_owned)
|
||||
.into_iter()
|
||||
.chain(appended)
|
||||
.collect::<Vec<_>>()
|
||||
.join("&");
|
||||
self.url
|
||||
.set_query(Some(query.as_str()).filter(|query| !query.is_empty()));
|
||||
self
|
||||
}
|
||||
|
||||
pub fn into_string(self) -> String {
|
||||
self.url.into()
|
||||
}
|
||||
|
|
@ -111,4 +141,41 @@ mod tests {
|
|||
"https://example.test/analyze?model=name+with+spaces"
|
||||
);
|
||||
}
|
||||
|
||||
const KEEP_COMMAS: &AsciiSet = &percent_encoding::NON_ALPHANUMERIC.remove(b',');
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::fresh_query("https://example.test", &[("pages", "1,2")], "?pages=1,2")]
|
||||
#[case::existing_query_kept(
|
||||
"https://example.test?tenant=a",
|
||||
&[("pages", "1,2")],
|
||||
"?tenant=a&pages=1,2"
|
||||
)]
|
||||
#[case::bare_question_mark("https://example.test?", &[("pages", "1")], "?pages=1")]
|
||||
#[case::no_pairs_no_query("https://example.test", &[], "")]
|
||||
#[case::no_pairs_keeps_query("https://example.test?tenant=a", &[], "?tenant=a")]
|
||||
#[case::outside_set_escaped(
|
||||
"https://example.test",
|
||||
&[("q", "a b&c=d+e%2C#")],
|
||||
"?q=a%20b%26c%3Dd%2Be%252C%23"
|
||||
)]
|
||||
#[case::keys_escaped_too("https://example.test", &[("a&b", "1")], "?a%26b=1")]
|
||||
#[case::non_ascii_as_utf8("https://example.test", &[("q", "é")], "?q=%C3%A9")]
|
||||
fn escaped_query_pairs(
|
||||
#[case] base: &str,
|
||||
#[case] pairs: &[(&str, &str)],
|
||||
#[case] expected_query: &str,
|
||||
) {
|
||||
let actual = ApiUrl::parse(base)
|
||||
.and_then(|url| url.complete_path(&["analyze"]))
|
||||
.map(|url| {
|
||||
url.append_query_pairs_escaped(pairs.iter().copied(), KEEP_COMMAS)
|
||||
.into_string()
|
||||
})
|
||||
.expect("url builds");
|
||||
assert_eq!(
|
||||
actual,
|
||||
format!("https://example.test/analyze{expected_query}")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
|
||||
use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
|
||||
use litellm_llms::{
|
||||
aws_textract::ocr::{
|
||||
analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation,
|
||||
|
|
@ -168,11 +168,8 @@ pub(crate) fn resolve_provider_config(
|
|||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Result<(String, OcrConfigKind), Error> {
|
||||
let provider =
|
||||
get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider {
|
||||
model,
|
||||
custom_llm_provider: OcrProvider::Mistral.into(),
|
||||
});
|
||||
let provider = get_custom_llm_provider(model, custom_llm_provider)
|
||||
.ok_or_else(|| Error::ProviderNotProvided(model.to_string()))?;
|
||||
let ocr_provider = provider
|
||||
.custom_llm_provider
|
||||
.parse::<OcrProvider>()
|
||||
|
|
@ -482,6 +479,15 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bare("mistral-ocr-latest")]
|
||||
#[case::other_route_model("gpt-4o")]
|
||||
fn ocr_contract_model_without_a_provider_is_rejected_like_python(#[case] model: &str) {
|
||||
let error = resolve_provider_config(model, None).unwrap_err();
|
||||
assert!(matches!(&error, Error::ProviderNotProvided(passed) if passed == model));
|
||||
assert_eq!(error.http_status_code(), Some(400));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::prefix("not_a_provider/model", None)]
|
||||
#[case::explicit("model", Some("not_a_provider"))]
|
||||
|
|
|
|||
|
|
@ -96,7 +96,17 @@ pub fn decode_request_input<D: Into<OcrDocumentInput>>(
|
|||
}
|
||||
|
||||
pub fn decode_document(value: Value) -> Result<OcrDocument, Error> {
|
||||
let kind = value.get("type").and_then(Value::as_str);
|
||||
let Some(fields) = value.as_object() else {
|
||||
return Err(Error::DocumentNotObject);
|
||||
};
|
||||
let kind = match fields.get("type") {
|
||||
Some(Value::String(kind)) if matches!(kind.as_str(), "document_url" | "image_url") => {
|
||||
Some(kind.as_str())
|
||||
}
|
||||
Some(Value::String(kind)) => return Err(Error::InvalidDocumentType(kind.clone())),
|
||||
Some(other) => return Err(Error::InvalidDocumentType(other.to_string())),
|
||||
None => return Err(Error::InvalidDocumentType("None".into())),
|
||||
};
|
||||
if matches!(kind, Some("document_url")) && value.get("document_url").is_none()
|
||||
|| matches!(kind, Some("image_url")) && value.get("image_url").is_none()
|
||||
{
|
||||
|
|
@ -122,9 +132,12 @@ mod tests {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::non_object(json!([]), "document")]
|
||||
#[case::missing_type(json!({"document_url":"https://example.com/a.pdf"}), "document")]
|
||||
#[case::unsupported_type(json!({"type":"text"}), "type")]
|
||||
#[case::non_object(json!([]), "document must be a dict")]
|
||||
#[case::missing_type(
|
||||
json!({"document_url":"https://example.com/a.pdf"}),
|
||||
"Invalid document type: None"
|
||||
)]
|
||||
#[case::unsupported_type(json!({"type":"text"}), "Invalid document type: text")]
|
||||
#[case::missing_document_url(json!({"type":"document_url"}), "Document URL")]
|
||||
#[case::missing_image_url(json!({"type":"image_url"}), "Document URL")]
|
||||
fn ocr_contract_malformed_document_is_bad_request(
|
||||
|
|
@ -134,7 +147,7 @@ mod tests {
|
|||
let error = decode_document(document).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::RequestField { .. } | Error::MissingDocumentUrl
|
||||
Error::DocumentNotObject | Error::InvalidDocumentType(_) | Error::MissingDocumentUrl
|
||||
));
|
||||
assert_eq!(error.http_status_code(), Some(400));
|
||||
assert!(error.to_string().contains(field), "{error}");
|
||||
|
|
|
|||
|
|
@ -5,6 +5,26 @@
|
|||
- `RouteHost::invoke` receives the keyword view the adapter's `begin` returned, not the caller's dict; a route host that projects from it inherits that adapter's rewrites (for the legacy adapter: setup, deployment hooks, credential inheritance)
|
||||
- A native failure, including one a host op returns as `InvokeError::Native`, is classified exactly once through the route's `classify`; a Python exception raised inside the call, and a failure in `begin` or `after_success`, is raised as is
|
||||
- A failing `classify` is raised with the native error's text as its `__context__`, never swallowed
|
||||
- Terms; the callback crates build on these and never redefine them
|
||||
- Machine: the native call; it never calls out, it yields a `HostOp` (`Route`, `BeforeSend`, `Emit`, `Open`, `Deliver`) and waits for the `HostResult`
|
||||
- Host: whatever answers those ops; the Python host is two capabilities the driver composes: `RouteHost` (route ops, argument and response projection) and `PythonLifecycle` (callbacks)
|
||||
- Host language = callback language: a lifecycle speaks `Py<...>` and `PyErr` directly; another host language gets its own crate of this shape and reuses only `litellm-host` and `litellm-callbacks-v1`
|
||||
- Driver: runs one call; owns the machine, one route host and exactly one lifecycle
|
||||
- Lifecycle: one implementation of `PythonLifecycle`, one callback contract's view of a call; chain: several lifecycles behind one, in a fixed order
|
||||
- Step: one driver-to-lifecycle method call, answered with a `LifecycleStep`; a threading step (`begin`, `before_send`, `after_success`) returns the value the call continues with and defaults to returning what it was given; an observing step (`emit`, `opened`, `delivered`) returns nothing to the call
|
||||
- Suspension: a step answered with `Await`; the driver awaits it in the caller's task and continues the same step through `resume`
|
||||
- Terminal event: `Succeeded` or `Failed`, exactly once, last
|
||||
- Cancellation: a raised `BaseException` that is not an `Exception`; `is_cancellation` is the only definition; never reported, never swallowed, never followed by dispatch
|
||||
- Keyword view: the kwargs dict as rewritten so far; `begin` threads it and the route projects from the result
|
||||
- Wire: the `WireRequest` (URL, headers, body) about to be sent; `before_send` threads it
|
||||
- Surface: the static facts a route declares about itself to a contract (`call_type`, ...)
|
||||
- Python contract: a crate's `python.rs` enum, its `python_contract.json` and the one `litellm/rust_bridge/*.py` module behind them, tested from both sides
|
||||
- `PythonLifecycle` is the whole callback boundary: the driver holds exactly one and never learns how many contracts are behind it
|
||||
- `LifecycleChain` composes several in a fixed order: `begin`, `before_send` and `after_success` thread their value forward, events reach each in turn, any error ends the step with no further dispatch
|
||||
- `begin` carries the call's start time and is the first thing a lifecycle hears of a call; there is no separate started event
|
||||
- A lifecycle whose `begin` was never reached receives nothing of that call; the terminal event goes to the ones `begin` reached
|
||||
- A suspension resumes the lifecycle that suspended, then continues the same step with the ones after it; a retained failure is the raised exception object itself
|
||||
- Ordering policy (which contract runs first) belongs to whoever builds the chain, never to this crate
|
||||
- Use standard PyO3 ownership and conversion APIs
|
||||
- Prefer `Bound<'py, T>` for attached operations/results, `Py<T>` for retention; binding/unbinding does not copy payloads
|
||||
- Use `pythonize` for selected Serde data, never a JSON-text round trip; share conversion with `Pythonized<T>`
|
||||
|
|
|
|||
|
|
@ -1,15 +1,22 @@
|
|||
use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest};
|
||||
use litellm_host::route::Route;
|
||||
use pyo3::exceptions::PyRuntimeError;
|
||||
use pyo3::exceptions::{PyException, PyRuntimeError};
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
/// Whether an error ends the call outright: anything that is not a `PyException`, such
|
||||
/// as `CancelledError` or `KeyboardInterrupt`. A cancellation is never reported, never
|
||||
/// swallowed and never followed by further dispatch.
|
||||
pub fn is_cancellation(py: Python<'_>, error: &PyErr) -> bool {
|
||||
!error.is_instance_of::<PyException>(py)
|
||||
}
|
||||
|
||||
pub fn missing_state() -> PyErr {
|
||||
PyRuntimeError::new_err("missing native call state")
|
||||
}
|
||||
|
||||
/// What an adapter step produced: either the value the driver asked for, or a Python
|
||||
/// What a lifecycle step produced: either the value the driver asked for, or a Python
|
||||
/// awaitable the driver hands back to the caller's task before asking again.
|
||||
pub enum LifecycleStep {
|
||||
Await(Py<PyAny>),
|
||||
|
|
@ -19,12 +26,10 @@ pub enum LifecycleStep {
|
|||
Done,
|
||||
}
|
||||
|
||||
/// What a lifecycle observes: the driver's start, the machine's own events, and one
|
||||
/// terminal event carrying the public value the caller receives.
|
||||
/// What a lifecycle observes: the machine's own events, and one terminal event
|
||||
/// carrying the public value the caller receives.
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum LifecycleEvent<'a> {
|
||||
Started {
|
||||
start_time: f64,
|
||||
},
|
||||
Machine(&'a MachineEvent),
|
||||
Succeeded {
|
||||
timing: Timing,
|
||||
|
|
@ -37,46 +42,73 @@ pub enum LifecycleEvent<'a> {
|
|||
},
|
||||
}
|
||||
|
||||
/// One consumer of a call's lifecycle on the Python side. The driver calls the steps in
|
||||
/// order: `begin` before the machine starts, `before_send` and `emit` while it runs,
|
||||
/// `after_success` and one terminal `emit` after it completes. Whenever a step returns
|
||||
/// [`LifecycleStep::Await`], the driver awaits it in the caller's task and continues the
|
||||
/// same step through `resume`.
|
||||
/// One consumer of a call's lifecycle on the Python side: the callback capability of the
|
||||
/// Python host, next to [`RouteHost`], the route capability.
|
||||
///
|
||||
/// The steps come in two kinds. A threading step receives a value and returns the value
|
||||
/// the call continues with, so a lifecycle may rewrite it; each defaults to returning
|
||||
/// what it was given. An observing step receives an event or a notice and returns
|
||||
/// nothing to the call. Two steps answer an operation the machine raised
|
||||
/// ([`HostOp`](litellm_host::host::HostOp)); the rest the driver originates around it:
|
||||
///
|
||||
/// | step | kind | raised by |
|
||||
/// |---|---|---|
|
||||
/// | `begin` | threads the keyword view | driver, before the machine starts |
|
||||
/// | `before_send` | threads the wire request | machine, `HostOp::BeforeSend` |
|
||||
/// | `emit(Machine)` | observes | machine, `HostOp::Emit` |
|
||||
/// | `opened`, `delivered` | observe | driver, answering `HostOp::Open` and `Deliver` |
|
||||
/// | `after_success` | threads the public response | driver, after the machine completes |
|
||||
/// | `emit(Succeeded \| Failed)` | observes | driver, exactly once, last |
|
||||
///
|
||||
/// Whenever a step returns [`LifecycleStep::Await`], the driver awaits it in the caller's
|
||||
/// task and continues the same step through `resume`.
|
||||
///
|
||||
/// A step that fails with an ordinary exception fails the call with that exception,
|
||||
/// except on a terminal event, where the adapter is expected to report and swallow its
|
||||
/// own errors. An exception that is not a `PyException`, such as a cancellation, ends
|
||||
/// the call without further dispatch.
|
||||
/// except on a terminal event, where the lifecycle is expected to report and swallow its
|
||||
/// own errors. A cancellation ([`is_cancellation`]) ends the call without further
|
||||
/// dispatch.
|
||||
pub trait PythonLifecycle: Send + Sync {
|
||||
/// `start_time` is the call's start in epoch seconds, the same value the terminal
|
||||
/// event's `Timing` carries.
|
||||
fn begin(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
_py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
started_at: f64,
|
||||
) -> PyResult<LifecycleStep>;
|
||||
_start_time: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
Ok(LifecycleStep::Arguments(arguments))
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
_py: Python<'_>,
|
||||
wire: Box<WireRequest>,
|
||||
context: &RequestContext,
|
||||
) -> PyResult<LifecycleStep>;
|
||||
_context: &RequestContext,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
Ok(LifecycleStep::Wire(wire))
|
||||
}
|
||||
|
||||
fn after_success(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
_py: Python<'_>,
|
||||
response: Py<PyAny>,
|
||||
timing: Timing,
|
||||
) -> PyResult<LifecycleStep>;
|
||||
_timing: Timing,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
Ok(LifecycleStep::Response(response))
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep>;
|
||||
|
||||
/// The call streams and its stream was handed to the caller. The caller is not
|
||||
/// inside an await here, so this step and `delivered` cannot suspend.
|
||||
fn opened(&mut self, py: Python<'_>) -> PyResult<()>;
|
||||
fn opened(&mut self, _py: Python<'_>) -> PyResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One chunk of an open stream is about to reach the caller.
|
||||
fn delivered(&mut self, py: Python<'_>, chunk: &Py<PyAny>) -> PyResult<()>;
|
||||
fn delivered(&mut self, _py: Python<'_>, _chunk: &Py<PyAny>) -> PyResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep>;
|
||||
|
||||
|
|
|
|||
560
litellm-rust/crates/host-python/src/chain.rs
Normal file
560
litellm-rust/crates/host-python/src/chain.rs
Normal file
|
|
@ -0,0 +1,560 @@
|
|||
//! Several lifecycles consuming one call as one [`PythonLifecycle`]. The driver keeps a
|
||||
//! single consumer; which contracts a call runs under is decided where the chain is
|
||||
//! built, never by the driver and never by a route.
|
||||
|
||||
use litellm_host::event::{FailureOrigin, MachineEvent, RequestContext, Timing, WireRequest};
|
||||
use pyo3::exceptions::PyBaseException;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
use crate::adapter::{LifecycleEvent, LifecycleStep, PythonLifecycle, missing_state};
|
||||
|
||||
/// An event kept across a suspension, so the lifecycles after the suspended one still
|
||||
/// receive it. A failure keeps the raised exception object itself.
|
||||
enum RetainedEvent {
|
||||
Machine(MachineEvent),
|
||||
Succeeded {
|
||||
timing: Timing,
|
||||
response: Py<PyAny>,
|
||||
},
|
||||
Failed {
|
||||
timing: Timing,
|
||||
origin: FailureOrigin,
|
||||
error: Py<PyBaseException>,
|
||||
},
|
||||
}
|
||||
|
||||
/// The step a suspended lifecycle was answering, with what the later lifecycles need to
|
||||
/// answer it too. The value being threaded stays with the suspended lifecycle.
|
||||
enum Step {
|
||||
Begin { start_time: f64 },
|
||||
BeforeSend { context: RequestContext },
|
||||
AfterSuccess { timing: Timing },
|
||||
Emit(RetainedEvent),
|
||||
}
|
||||
|
||||
struct Suspended {
|
||||
index: usize,
|
||||
step: Step,
|
||||
}
|
||||
|
||||
/// Runs every step through its lifecycles in the order they were given.
|
||||
///
|
||||
/// `begin`, `before_send` and `after_success` thread their value: each lifecycle
|
||||
/// receives what the one before it returned, so a later lifecycle sees an earlier one's
|
||||
/// rewrites and never the reverse. Events reach each lifecycle in turn. An error from
|
||||
/// any lifecycle ends the step at once, with no dispatch to the ones after it.
|
||||
///
|
||||
/// A lifecycle observes a call only once its `begin` was reached: when an earlier
|
||||
/// lifecycle fails the call in `begin`, the later ones never learn of the call, and the
|
||||
/// terminal event goes to those that did.
|
||||
pub struct LifecycleChain {
|
||||
lifecycles: Vec<Box<dyn PythonLifecycle>>,
|
||||
/// How many lifecycles `begin` has reached.
|
||||
entered: usize,
|
||||
pending: Option<Suspended>,
|
||||
}
|
||||
|
||||
impl RetainedEvent {
|
||||
fn retain(py: Python<'_>, event: LifecycleEvent<'_>) -> Self {
|
||||
match event {
|
||||
LifecycleEvent::Machine(event) => Self::Machine(event.clone()),
|
||||
LifecycleEvent::Succeeded { timing, response } => Self::Succeeded {
|
||||
timing,
|
||||
response: response.clone_ref(py),
|
||||
},
|
||||
LifecycleEvent::Failed {
|
||||
timing,
|
||||
origin,
|
||||
error,
|
||||
} => Self::Failed {
|
||||
timing,
|
||||
origin,
|
||||
error: error.clone_ref(py).into_value(py),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn emit(&self, py: Python<'_>, lifecycle: &mut dyn PythonLifecycle) -> PyResult<LifecycleStep> {
|
||||
match self {
|
||||
Self::Machine(event) => lifecycle.emit(py, LifecycleEvent::Machine(event)),
|
||||
Self::Succeeded { timing, response } => lifecycle.emit(
|
||||
py,
|
||||
LifecycleEvent::Succeeded {
|
||||
timing: *timing,
|
||||
response,
|
||||
},
|
||||
),
|
||||
Self::Failed {
|
||||
timing,
|
||||
origin,
|
||||
error,
|
||||
} => {
|
||||
let error = PyErr::from_value(error.bind(py).clone().into_any());
|
||||
lifecycle.emit(
|
||||
py,
|
||||
LifecycleEvent::Failed {
|
||||
timing: *timing,
|
||||
origin: *origin,
|
||||
error: &error,
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
match self {
|
||||
Self::Succeeded { response, .. } => visit.call(response),
|
||||
Self::Failed { error, .. } => visit.call(error),
|
||||
Self::Machine(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LifecycleChain {
|
||||
pub fn new(lifecycles: Vec<Box<dyn PythonLifecycle>>) -> Self {
|
||||
Self {
|
||||
lifecycles,
|
||||
entered: 0,
|
||||
pending: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// How many lifecycles a step reaches: `begin` tries all of them, every later step
|
||||
/// reaches the ones `begin` reached.
|
||||
fn audience(&self, step: &Step) -> usize {
|
||||
match step {
|
||||
Step::Begin { .. } => self.lifecycles.len(),
|
||||
Step::BeforeSend { .. } | Step::AfterSuccess { .. } | Step::Emit(_) => self.entered,
|
||||
}
|
||||
}
|
||||
|
||||
fn call(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
index: usize,
|
||||
step: &Step,
|
||||
carried: LifecycleStep,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let lifecycle = self.lifecycles[index].as_mut();
|
||||
match (step, carried) {
|
||||
(Step::Begin { start_time }, LifecycleStep::Arguments(arguments)) => {
|
||||
self.entered = index + 1;
|
||||
lifecycle.begin(py, arguments, *start_time)
|
||||
}
|
||||
(Step::BeforeSend { context }, LifecycleStep::Wire(wire)) => {
|
||||
lifecycle.before_send(py, wire, context)
|
||||
}
|
||||
(Step::AfterSuccess { timing }, LifecycleStep::Response(response)) => {
|
||||
lifecycle.after_success(py, response, *timing)
|
||||
}
|
||||
(Step::Emit(event), LifecycleStep::Done) => event.emit(py, lifecycle),
|
||||
_ => Err(missing_state()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Continues `step` from the lifecycle at `next`, handing each one what the previous
|
||||
/// produced, until one suspends or the last has answered.
|
||||
fn proceed(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
mut next: usize,
|
||||
mut carried: LifecycleStep,
|
||||
step: Step,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
while next < self.audience(&step) {
|
||||
carried = match self.call(py, next, &step, carried)? {
|
||||
LifecycleStep::Await(awaitable) => {
|
||||
self.pending = Some(Suspended { index: next, step });
|
||||
return Ok(LifecycleStep::Await(awaitable));
|
||||
}
|
||||
produced => produced,
|
||||
};
|
||||
next += 1;
|
||||
}
|
||||
Ok(carried)
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonLifecycle for LifecycleChain {
|
||||
fn begin(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
start_time: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.proceed(
|
||||
py,
|
||||
0,
|
||||
LifecycleStep::Arguments(arguments),
|
||||
Step::Begin { start_time },
|
||||
)
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
wire: Box<WireRequest>,
|
||||
context: &RequestContext,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.proceed(
|
||||
py,
|
||||
0,
|
||||
LifecycleStep::Wire(wire),
|
||||
Step::BeforeSend {
|
||||
context: context.clone(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn after_success(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
response: Py<PyAny>,
|
||||
timing: Timing,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.proceed(
|
||||
py,
|
||||
0,
|
||||
LifecycleStep::Response(response),
|
||||
Step::AfterSuccess { timing },
|
||||
)
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
for index in 0..self.entered {
|
||||
match self.lifecycles[index].emit(py, event)? {
|
||||
LifecycleStep::Await(awaitable) => {
|
||||
self.pending = Some(Suspended {
|
||||
index,
|
||||
step: Step::Emit(RetainedEvent::retain(py, event)),
|
||||
});
|
||||
return Ok(LifecycleStep::Await(awaitable));
|
||||
}
|
||||
LifecycleStep::Done => {}
|
||||
_ => return Err(missing_state()),
|
||||
}
|
||||
}
|
||||
Ok(LifecycleStep::Done)
|
||||
}
|
||||
|
||||
fn opened(&mut self, py: Python<'_>) -> PyResult<()> {
|
||||
self.lifecycles[..self.entered]
|
||||
.iter_mut()
|
||||
.try_for_each(|lifecycle| lifecycle.opened(py))
|
||||
}
|
||||
|
||||
fn delivered(&mut self, py: Python<'_>, chunk: &Py<PyAny>) -> PyResult<()> {
|
||||
self.lifecycles[..self.entered]
|
||||
.iter_mut()
|
||||
.try_for_each(|lifecycle| lifecycle.delivered(py, chunk))
|
||||
}
|
||||
|
||||
fn resume(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
|
||||
let Suspended { index, step } = self.pending.take().ok_or_else(missing_state)?;
|
||||
match self.lifecycles[index].resume(py, result)? {
|
||||
LifecycleStep::Await(awaitable) => {
|
||||
self.pending = Some(Suspended { index, step });
|
||||
Ok(LifecycleStep::Await(awaitable))
|
||||
}
|
||||
produced => self.proceed(py, index + 1, produced, step),
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&mut self, py: Python<'_>) {
|
||||
self.pending = None;
|
||||
for lifecycle in &mut self.lifecycles {
|
||||
lifecycle.close(py);
|
||||
}
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
for lifecycle in &self.lifecycles {
|
||||
lifecycle.traverse(visit)?;
|
||||
}
|
||||
match &self.pending {
|
||||
Some(Suspended {
|
||||
step: Step::Emit(event),
|
||||
..
|
||||
}) => event.traverse(visit),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use pyo3::exceptions::PyValueError;
|
||||
|
||||
use super::*;
|
||||
|
||||
type Log = Arc<Mutex<Vec<String>>>;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Script {
|
||||
Plain,
|
||||
FailBegin,
|
||||
SuspendBegin,
|
||||
SuspendTerminal,
|
||||
}
|
||||
|
||||
struct Recorder {
|
||||
name: &'static str,
|
||||
script: Script,
|
||||
log: Log,
|
||||
suspended: Option<LifecycleStep>,
|
||||
}
|
||||
|
||||
impl Recorder {
|
||||
fn record(&self, entry: &str) {
|
||||
self.log
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(format!("{}.{entry}", self.name));
|
||||
}
|
||||
|
||||
fn suspend(&mut self, py: Python<'_>, then: LifecycleStep) -> PyResult<LifecycleStep> {
|
||||
self.suspended = Some(then);
|
||||
Ok(LifecycleStep::Await(py.None()))
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonLifecycle for Recorder {
|
||||
fn begin(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
arguments: Py<PyDict>,
|
||||
_: f64,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let seen: Vec<String> = arguments
|
||||
.bind(py)
|
||||
.keys()
|
||||
.iter()
|
||||
.map(|key| key.extract().unwrap())
|
||||
.collect();
|
||||
self.record(&format!("begin{seen:?}"));
|
||||
arguments.bind(py).set_item(self.name, true)?;
|
||||
match self.script {
|
||||
Script::FailBegin => Err(PyValueError::new_err("begin failed")),
|
||||
Script::SuspendBegin => self.suspend(py, LifecycleStep::Arguments(arguments)),
|
||||
_ => Ok(LifecycleStep::Arguments(arguments)),
|
||||
}
|
||||
}
|
||||
|
||||
fn before_send(
|
||||
&mut self,
|
||||
_: Python<'_>,
|
||||
mut wire: Box<WireRequest>,
|
||||
_: &RequestContext,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.record(&format!("before_send:{}", wire.url));
|
||||
wire.url.push_str(self.name);
|
||||
Ok(LifecycleStep::Wire(wire))
|
||||
}
|
||||
|
||||
fn after_success(
|
||||
&mut self,
|
||||
_: Python<'_>,
|
||||
response: Py<PyAny>,
|
||||
_: Timing,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
self.record("after_success");
|
||||
Ok(LifecycleStep::Response(response))
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
let terminal = match event {
|
||||
LifecycleEvent::Machine(_) => {
|
||||
self.record("machine");
|
||||
false
|
||||
}
|
||||
LifecycleEvent::Succeeded { response, .. } => {
|
||||
self.record(&format!("succeeded:{}", response.bind(py)));
|
||||
true
|
||||
}
|
||||
LifecycleEvent::Failed { error, .. } => {
|
||||
self.record(&format!("failed:{}", error.value(py)));
|
||||
true
|
||||
}
|
||||
};
|
||||
if terminal && self.script == Script::SuspendTerminal {
|
||||
return self.suspend(py, LifecycleStep::Done);
|
||||
}
|
||||
Ok(LifecycleStep::Done)
|
||||
}
|
||||
|
||||
fn opened(&mut self, _: Python<'_>) -> PyResult<()> {
|
||||
self.record("opened");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delivered(&mut self, _: Python<'_>, _: &Py<PyAny>) -> PyResult<()> {
|
||||
self.record("delivered");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resume(&mut self, _: Python<'_>, _: PyResult<Py<PyAny>>) -> PyResult<LifecycleStep> {
|
||||
self.record("resume");
|
||||
self.suspended.take().ok_or_else(missing_state)
|
||||
}
|
||||
|
||||
fn close(&mut self, _: Python<'_>) {
|
||||
self.record("close");
|
||||
}
|
||||
|
||||
fn traverse(&self, _: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn chain(scripts: [(&'static str, Script); 2]) -> (LifecycleChain, Log) {
|
||||
let log = Log::default();
|
||||
let lifecycles = scripts
|
||||
.into_iter()
|
||||
.map(|(name, script)| {
|
||||
Box::new(Recorder {
|
||||
name,
|
||||
script,
|
||||
log: Arc::clone(&log),
|
||||
suspended: None,
|
||||
}) as Box<dyn PythonLifecycle>
|
||||
})
|
||||
.collect();
|
||||
(LifecycleChain::new(lifecycles), log)
|
||||
}
|
||||
|
||||
fn entries(log: &Log) -> Vec<String> {
|
||||
std::mem::take(&mut *log.lock().unwrap())
|
||||
}
|
||||
|
||||
fn timing() -> Timing {
|
||||
Timing {
|
||||
start_time: 1.0,
|
||||
end_time: 2.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn context() -> RequestContext {
|
||||
RequestContext {
|
||||
model: "model".to_string(),
|
||||
custom_llm_provider: "provider".to_string(),
|
||||
optional_params: serde_json::Value::Null,
|
||||
secret_fields: Vec::new(),
|
||||
api_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_later_lifecycle_receives_what_an_earlier_one_returned() {
|
||||
crate::initialize_python();
|
||||
Python::attach(|py| {
|
||||
let (mut chain, log) = chain([("first", Script::Plain), ("second", Script::Plain)]);
|
||||
let arguments = PyDict::new(py).unbind();
|
||||
let step = chain.begin(py, arguments.clone_ref(py), 1.0).unwrap();
|
||||
assert!(matches!(step, LifecycleStep::Arguments(returned) if returned.is(&arguments)));
|
||||
let wire = Box::new(WireRequest {
|
||||
url: "url:".to_string(),
|
||||
headers: Vec::new(),
|
||||
body: serde_json::Value::Null,
|
||||
});
|
||||
let step = chain.before_send(py, wire, &context()).unwrap();
|
||||
assert!(matches!(step, LifecycleStep::Wire(wire) if wire.url == "url:firstsecond"));
|
||||
assert_eq!(
|
||||
entries(&log),
|
||||
[
|
||||
"first.begin[]",
|
||||
"second.begin[\"first\"]",
|
||||
"first.before_send:url:",
|
||||
"second.before_send:url:first",
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lifecycle_begin_never_reached_observes_nothing_of_the_call() {
|
||||
crate::initialize_python();
|
||||
Python::attach(|py| {
|
||||
let (mut chain, log) = chain([("first", Script::FailBegin), ("second", Script::Plain)]);
|
||||
let error = chain
|
||||
.begin(py, PyDict::new(py).unbind(), 1.0)
|
||||
.err()
|
||||
.expect("begin fails");
|
||||
let failed = LifecycleEvent::Failed {
|
||||
timing: timing(),
|
||||
origin: FailureOrigin::Host,
|
||||
error: &error,
|
||||
};
|
||||
assert!(matches!(
|
||||
chain.emit(py, failed).unwrap(),
|
||||
LifecycleStep::Done
|
||||
));
|
||||
chain.close(py);
|
||||
assert_eq!(
|
||||
entries(&log),
|
||||
[
|
||||
"first.begin[]",
|
||||
"first.failed:begin failed",
|
||||
"first.close",
|
||||
"second.close",
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_suspended_step_resumes_its_lifecycle_and_then_reaches_the_later_ones() {
|
||||
crate::initialize_python();
|
||||
Python::attach(|py| {
|
||||
let (mut chain, log) = chain([
|
||||
("first", Script::SuspendBegin),
|
||||
("second", Script::SuspendTerminal),
|
||||
]);
|
||||
let step = chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap();
|
||||
assert!(matches!(step, LifecycleStep::Await(_)));
|
||||
assert_eq!(entries(&log), ["first.begin[]"]);
|
||||
let step = chain.resume(py, Ok(py.None())).unwrap();
|
||||
assert!(matches!(step, LifecycleStep::Arguments(_)));
|
||||
assert_eq!(entries(&log), ["first.resume", "second.begin[\"first\"]"]);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_event_kept_across_a_suspension_reaches_the_later_lifecycles_unchanged() {
|
||||
crate::initialize_python();
|
||||
Python::attach(|py| {
|
||||
let (mut chain, log) = chain([
|
||||
("first", Script::SuspendTerminal),
|
||||
("second", Script::Plain),
|
||||
]);
|
||||
chain.begin(py, PyDict::new(py).unbind(), 1.0).unwrap();
|
||||
entries(&log);
|
||||
let raised = PyValueError::new_err("provider failed");
|
||||
let failed = LifecycleEvent::Failed {
|
||||
timing: timing(),
|
||||
origin: FailureOrigin::Call,
|
||||
error: &raised,
|
||||
};
|
||||
assert!(matches!(
|
||||
chain.emit(py, failed).unwrap(),
|
||||
LifecycleStep::Await(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
chain.resume(py, Ok(py.None())).unwrap(),
|
||||
LifecycleStep::Done
|
||||
));
|
||||
assert_eq!(
|
||||
entries(&log),
|
||||
[
|
||||
"first.failed:provider failed",
|
||||
"first.resume",
|
||||
"second.failed:provider failed",
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -46,7 +46,6 @@ enum Stage {
|
|||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Expect {
|
||||
Started,
|
||||
Arguments,
|
||||
Wire,
|
||||
Emitted,
|
||||
|
|
@ -159,13 +158,7 @@ where
|
|||
match (self.pending.take(), result) {
|
||||
(None, None) => {
|
||||
self.started_at = epoch_seconds();
|
||||
let started = LifecycleEvent::Started {
|
||||
start_time: self.started_at,
|
||||
};
|
||||
match self.adapter.emit(py, started) {
|
||||
Ok(step) => self.on_adapter(py, step, Expect::Started),
|
||||
Err(error) => self.adapter_failed(py, error),
|
||||
}
|
||||
self.begin(py)
|
||||
}
|
||||
(Some(Pending::Native), Some(Ok(_))) => {
|
||||
let result = self.take_native_result()?;
|
||||
|
|
@ -201,7 +194,6 @@ where
|
|||
self.pending = Some(Pending::Adapter(expect));
|
||||
Ok(ExecutionStep::Await(awaitable))
|
||||
}
|
||||
(Expect::Started, LifecycleStep::Done) => self.begin(py),
|
||||
(Expect::Arguments, LifecycleStep::Arguments(arguments)) => {
|
||||
self.arguments = Some(arguments);
|
||||
self.stage = Stage::Call;
|
||||
|
|
@ -789,7 +781,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
self.log.push(match event {
|
||||
LifecycleEvent::Started { .. } => "started".into(),
|
||||
LifecycleEvent::Machine(MachineEvent::ResponseReceived { raw }) => {
|
||||
format!("response:{}", raw.body)
|
||||
}
|
||||
|
|
@ -921,7 +912,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"before_send",
|
||||
|
|
@ -967,7 +957,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"classify:provider exploded",
|
||||
|
|
@ -1001,7 +990,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"classify:op rejected",
|
||||
|
|
@ -1033,7 +1021,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"failed:Call:op failed",
|
||||
|
|
@ -1071,7 +1058,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"route:project",
|
||||
"classify:provider exploded",
|
||||
|
|
@ -1102,7 +1088,6 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
assert_eq!(
|
||||
log,
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"failed:Host:begin failed",
|
||||
"adapter.close",
|
||||
|
|
@ -1225,10 +1210,7 @@ sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bri
|
|||
)
|
||||
.unwrap_err();
|
||||
assert!(!error.is_instance_of::<pyo3::exceptions::PyException>(py));
|
||||
assert_eq!(
|
||||
log.entries(),
|
||||
["started", "begin", "route", "adapter.close"]
|
||||
);
|
||||
assert_eq!(log.entries(), ["begin", "route", "adapter.close"]);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
mod adapter;
|
||||
mod argument;
|
||||
mod callable;
|
||||
mod chain;
|
||||
mod driver;
|
||||
mod execution;
|
||||
mod fork_gate;
|
||||
|
|
@ -14,10 +15,12 @@ mod handle;
|
|||
mod marshal;
|
||||
|
||||
pub use adapter::{
|
||||
InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, missing_state,
|
||||
InvokeError, LifecycleEvent, LifecycleStep, PythonLifecycle, RouteHost, is_cancellation,
|
||||
missing_state,
|
||||
};
|
||||
pub use argument::lookup;
|
||||
pub use callable::wrap_failure;
|
||||
pub use chain::LifecycleChain;
|
||||
pub use driver::run_call;
|
||||
pub use execution::{
|
||||
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value,
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ base64.workspace = true
|
|||
bytes.workspace = true
|
||||
data-url = "0.3.2"
|
||||
futures-util.workspace = true
|
||||
percent-encoding.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use litellm_core_utils::{
|
|||
serde_compat::{FiniteF64, LaxI64},
|
||||
url_utils::ApiUrl,
|
||||
};
|
||||
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC};
|
||||
use reqwest::Url;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
|
@ -549,6 +550,17 @@ async fn poll_operation(
|
|||
}
|
||||
}
|
||||
|
||||
/// Python parity: the Python provider builds this query with
|
||||
/// `urllib.parse.quote(value, safe=",")`, which escapes everything except RFC 3986 unreserved
|
||||
/// characters and `,`. The literal commas are also what the service documents for the
|
||||
/// `pages` and `features` lists; form encoding would send `%2C`.
|
||||
const PYTHON_QUOTE_KEEP_COMMAS: &AsciiSet = &NON_ALPHANUMERIC
|
||||
.remove(b'-')
|
||||
.remove(b'.')
|
||||
.remove(b'_')
|
||||
.remove(b'~')
|
||||
.remove(b',');
|
||||
|
||||
impl AzureDocumentIntelligenceOcrConfig {
|
||||
fn build_ocr_url(
|
||||
&self,
|
||||
|
|
@ -561,16 +573,15 @@ impl AzureDocumentIntelligenceOcrConfig {
|
|||
ApiUrl::parse(endpoint)
|
||||
.and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model]))
|
||||
.map(|url| {
|
||||
url.append_query_pairs(
|
||||
[("api-version", api_version)]
|
||||
.into_iter()
|
||||
.chain(params.pages.iter().map(|pages| ("pages", pages.as_str())))
|
||||
.chain(
|
||||
params
|
||||
.features
|
||||
.iter()
|
||||
.map(|features| ("features", features.as_str())),
|
||||
),
|
||||
url.append_query_pairs_escaped(
|
||||
[
|
||||
("api-version", Some(api_version)),
|
||||
("pages", params.pages.as_deref()),
|
||||
("features", params.features.as_deref()),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(key, value)| Some((key, value?))),
|
||||
PYTHON_QUOTE_KEEP_COMMAS,
|
||||
)
|
||||
.into_string()
|
||||
})
|
||||
|
|
@ -649,6 +660,54 @@ mod tests {
|
|||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn query_escape_set_matches_python_quote() {
|
||||
// Expected output from Python:
|
||||
// quote("".join(map(chr, range(0x20, 0x7f))) + "é", safe=",")
|
||||
let input = (0x20u8..0x7f)
|
||||
.map(char::from)
|
||||
.chain(['é'])
|
||||
.collect::<String>();
|
||||
assert_eq!(
|
||||
percent_encoding::utf8_percent_encode(&input, PYTHON_QUOTE_KEEP_COMMAS).to_string(),
|
||||
"%20%21%22%23%24%25%26%27%28%29%2A%2B,-.%2F0123456789%3A%3B%3C%3D%3E%3F%40\
|
||||
ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%C3%A9"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_lists(None, None, "")]
|
||||
#[case::pages_only(Some("1,3-4"), None, "&pages=1,3-4")]
|
||||
#[case::features_only(
|
||||
None,
|
||||
Some("keyValuePairs,languages"),
|
||||
"&features=keyValuePairs,languages"
|
||||
)]
|
||||
#[case::both_lists(
|
||||
Some("1,3-4"),
|
||||
Some("keyValuePairs,languages"),
|
||||
"&pages=1,3-4&features=keyValuePairs,languages"
|
||||
)]
|
||||
fn ocr_url_query_matches_python(
|
||||
#[case] pages: Option<&str>,
|
||||
#[case] features: Option<&str>,
|
||||
#[case] expected_lists: &str,
|
||||
) {
|
||||
let params = DocumentIntelligenceParams {
|
||||
pages: pages.map(Into::into),
|
||||
features: features.map(Into::into),
|
||||
};
|
||||
assert_eq!(
|
||||
AzureDocumentIntelligenceOcrConfig
|
||||
.build_ocr_url("https://di.test", "prebuilt-layout", ¶ms, "2024-11-30")
|
||||
.unwrap(),
|
||||
format!(
|
||||
"https://di.test/documentintelligence/documentModels/prebuilt-layout:analyze\
|
||||
?api-version=2024-11-30{expected_lists}"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
fn map(value: Value) -> Result<DocumentIntelligenceParams, Error> {
|
||||
let arguments = serde_json::from_value(value).unwrap();
|
||||
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model")
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ mod tests {
|
|||
client.document_fetcher(),
|
||||
OcrDocument::ImageUrl {
|
||||
image_url: format!("http://{address}/image"),
|
||||
extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]),
|
||||
extra_fields: Map::from_iter([("detail".into(), "high".into())]),
|
||||
},
|
||||
&OcrConnection::default(),
|
||||
)
|
||||
|
|
@ -210,7 +210,7 @@ mod tests {
|
|||
converted,
|
||||
OcrDocument::ImageUrl {
|
||||
image_url: "data:image/png;base64,YWJj".into(),
|
||||
extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]),
|
||||
extra_fields: Map::from_iter([("detail".into(), "high".into())]),
|
||||
}
|
||||
);
|
||||
assert!(!request.to_ascii_lowercase().contains("authorization"));
|
||||
|
|
|
|||
|
|
@ -30,6 +30,10 @@ pub enum Error {
|
|||
MissingField(&'static str),
|
||||
#[error("Document URL is required")]
|
||||
MissingDocumentUrl,
|
||||
#[error("document must be a dict with 'type' and URL/file field")]
|
||||
DocumentNotObject,
|
||||
#[error("Invalid document type: {0}. Must be 'document_url', 'image_url', or 'file'")]
|
||||
InvalidDocumentType(String),
|
||||
#[error("invalid OCR document data URI")]
|
||||
InvalidDataUri,
|
||||
#[error(
|
||||
|
|
@ -76,6 +80,10 @@ pub enum Error {
|
|||
Unsupported(&'static str),
|
||||
#[error("invalid provider: {0}")]
|
||||
InvalidProvider(String),
|
||||
#[error(
|
||||
"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={0}"
|
||||
)]
|
||||
ProviderNotProvided(String),
|
||||
#[error("invalid model: {provider} has no model {model:?} - use one of: {}", supported.join(", "))]
|
||||
InvalidModel {
|
||||
provider: &'static str,
|
||||
|
|
@ -149,6 +157,8 @@ impl Error {
|
|||
| Self::RequestField { .. }
|
||||
| Self::MissingField(_)
|
||||
| Self::MissingDocumentUrl
|
||||
| Self::DocumentNotObject
|
||||
| Self::InvalidDocumentType(_)
|
||||
| Self::InvalidDataUri
|
||||
| Self::ReductoSource
|
||||
| Self::InlineDocumentTooLarge
|
||||
|
|
@ -161,6 +171,7 @@ impl Error {
|
|||
| Self::DotModel
|
||||
| Self::InvalidRequest(_)
|
||||
| Self::InvalidProvider(_)
|
||||
| Self::ProviderNotProvided(_)
|
||||
| Self::InvalidModel { .. }
|
||||
| Self::Params(_)
|
||||
| Self::Headers(_)
|
||||
|
|
|
|||
|
|
@ -32,16 +32,31 @@ pub enum OcrDocument {
|
|||
DocumentUrl {
|
||||
document_url: String,
|
||||
#[serde(flatten)]
|
||||
extra_fields: BTreeMap<String, Option<String>>,
|
||||
extra_fields: BTreeMap<String, Value>,
|
||||
},
|
||||
#[serde(rename = "image_url")]
|
||||
ImageUrl {
|
||||
#[serde(deserialize_with = "image_url_source")]
|
||||
image_url: String,
|
||||
#[serde(flatten)]
|
||||
extra_fields: BTreeMap<String, Option<String>>,
|
||||
extra_fields: BTreeMap<String, Value>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Mistral accepts `image_url` as a string or as `{"url": ...}`; providers only consume the
|
||||
/// source, so the object form is read down to its URL.
|
||||
fn image_url_source<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<String, D::Error> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum ImageUrl {
|
||||
Source(String),
|
||||
Object { url: String },
|
||||
}
|
||||
Ok(match ImageUrl::deserialize(deserializer)? {
|
||||
ImageUrl::Source(url) | ImageUrl::Object { url } => url,
|
||||
})
|
||||
}
|
||||
|
||||
impl OcrDocument {
|
||||
pub fn source(&self) -> &str {
|
||||
match self {
|
||||
|
|
@ -346,8 +361,13 @@ pub struct DecodedOcrResponse<T> {
|
|||
|
||||
pub fn decode_request_value<T: DeserializeOwned>(value: Value, prefix: &str) -> Result<T, Error> {
|
||||
serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| {
|
||||
let path = error.path().to_string();
|
||||
Error::RequestField {
|
||||
path: format!("{prefix}.{}", error.path()),
|
||||
path: if path == "." {
|
||||
prefix.to_string()
|
||||
} else {
|
||||
format!("{prefix}.{path}")
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -639,10 +659,8 @@ mod tests {
|
|||
assert!(serde_json::from_value::<LiteLLMOcrResponse>(Value::Object(payload)).is_err());
|
||||
}
|
||||
assert!(
|
||||
serde_json::from_value::<OcrDocument>(json!({
|
||||
"type":"image_url", "image_url":"https://example.com/image", "detail":42
|
||||
}))
|
||||
.is_err()
|
||||
serde_json::from_value::<OcrDocument>(json!({"type":"image_url", "image_url":42}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -694,6 +712,20 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn documents_accept_what_python_forwards_to_the_provider() {
|
||||
let document: OcrDocument = serde_json::from_value(json!({
|
||||
"type":"image_url", "image_url":{"url":"https://example.com/image", "detail":"high"}
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(document.source(), "https://example.com/image");
|
||||
let document = json!({
|
||||
"type":"document_url", "document_url":"https://example.com/a.pdf", "n":3
|
||||
});
|
||||
let decoded: OcrDocument = serde_json::from_value(document.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(decoded).unwrap(), document);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_serialization_flattens_extra_fields_and_omits_absent_native_response() {
|
||||
let response = LiteLLMOcrResponse {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
- Target invariants, not completion claims; these supersede the crate guidance below where they conflict
|
||||
- Keep this crate the product-specific PyO3 consumer of `litellm-host-python`
|
||||
- Own registration, input projection, the route host and the caller callables it answers operations with (file readers, token providers), public response/error construction and the per-call composition of machine, route host and callback contract
|
||||
- Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `run_legacy_call`; the bridge hands the public call over and keeps no copy
|
||||
- Legacy callback sharing (the caller's args, kwargs and request object, body/header roots, re-aliasing unchanged body keys) lives in `litellm-callbacks-legacy-python` behind `PublicCall` and `LegacyPythonLifecycle`; the bridge hands the public call over and keeps no copy
|
||||
- Routes never select a callback contract: `callbacks::run_python_call` is the only way a route starts a call, and `src/callbacks.rs` is the only place that names a contract crate
|
||||
- Every call runs the legacy `Logging` lifecycle, then the v1 lifecycle when anything is subscribed; with no subscriber the call is exactly the legacy-only call
|
||||
- A route describes itself once as a `CallSurface`; fields only the legacy contract reads are marked so and go when it goes
|
||||
- Retiring legacy for a route, or for all of them, is a change to that one composition, never to a route, the driver or an adapter
|
||||
- Value-oriented execution, sync waiting, nested-runtime checks, signal polling and panic containment live in `litellm-host-python`; native async work uses `pyo3-async-runtimes`, Serde output uses `Pythonized<T>`
|
||||
- Core owns typed native state, the route machine, provider preparation/I/O and normalization; the host driver owns terminal events; the legacy adapter in `litellm-callbacks-legacy-python` owns `Logging` dispatch policy
|
||||
- Python, Rust SDK and gateway use one lifecycle-bearing core route entrypoint; provider helpers stay private, never bridge-accessible transport drivers
|
||||
|
|
|
|||
|
|
@ -19,10 +19,12 @@ panic-test = []
|
|||
bytes.workspace = true
|
||||
litellm-auth.workspace = true
|
||||
litellm-callbacks-legacy-python.workspace = true
|
||||
litellm-callbacks-v1-python.workspace = true
|
||||
litellm-core.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-auth-gcp.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-host.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
litellm-types.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
|
|
|
|||
79
litellm-rust/crates/python-bridge/src/callbacks.rs
Normal file
79
litellm-rust/crates/python-bridge/src/callbacks.rs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
//! The callback contracts a Python call runs under, composed here once so that no route
|
||||
//! selects one. Every call runs the legacy `Logging` contract, for as long as Python's
|
||||
//! `Logging` exists, and then the v1 contract whenever anything is subscribed to it.
|
||||
//! Both are a [`PythonLifecycle`]; that trait is the whole boundary the driver knows.
|
||||
|
||||
use litellm_callbacks_legacy_python::{LegacyPythonLifecycle, LegacyPythonSurface};
|
||||
pub(crate) use litellm_callbacks_legacy_python::{PassThroughStream, PublicCall};
|
||||
use litellm_callbacks_v1_python::{V1PythonLifecycle, V1PythonSurface};
|
||||
use litellm_host::{machine::Machine, route::Route};
|
||||
use litellm_host_python::{LifecycleChain, PythonLifecycle, RouteHost, run_call};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
/// What a route tells the callback contracts about itself.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct CallSurface {
|
||||
pub call_type: &'static str,
|
||||
/// Legacy only: what `Logging.pre_call` is told the input was.
|
||||
pub input_description: &'static str,
|
||||
/// Legacy only: how a streamed response is billed; `None` for a route that never
|
||||
/// streams.
|
||||
pub stream: Option<PassThroughStream>,
|
||||
}
|
||||
|
||||
impl CallSurface {
|
||||
fn legacy(self) -> LegacyPythonSurface {
|
||||
LegacyPythonSurface {
|
||||
call_type: self.call_type,
|
||||
input_description: self.input_description,
|
||||
stream: self.stream,
|
||||
}
|
||||
}
|
||||
|
||||
fn v1(self) -> V1PythonSurface {
|
||||
V1PythonSurface {
|
||||
call_type: self.call_type,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Legacy first: v1 subscribers observe the keyword view and the wire request as the
|
||||
/// legacy contract left them, and share the `litellm_call_id` its setup assigned.
|
||||
fn lifecycle(
|
||||
py: Python<'_>,
|
||||
surface: CallSurface,
|
||||
call: PublicCall,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Box<dyn PythonLifecycle>> {
|
||||
let legacy = Box::new(LegacyPythonLifecycle::new(
|
||||
py,
|
||||
surface.legacy(),
|
||||
call,
|
||||
asynchronous,
|
||||
));
|
||||
Ok(
|
||||
match V1PythonLifecycle::subscribed(py, surface.v1(), asynchronous)? {
|
||||
Some(v1) => Box::new(LifecycleChain::new(vec![legacy, Box::new(v1)])),
|
||||
None => legacy,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Runs one native call for a Python caller. The route host projects from the keyword
|
||||
/// view the lifecycle's `begin` prepares.
|
||||
pub(crate) fn run_python_call<H, M>(
|
||||
py: Python<'_>,
|
||||
surface: CallSurface,
|
||||
call: PublicCall,
|
||||
machine: M,
|
||||
route: H,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Py<PyAny>>
|
||||
where
|
||||
H: RouteHost + 'static,
|
||||
M: Machine<Route = H::Route, Complete = <H::Route as Route>::Response> + 'static,
|
||||
{
|
||||
let arguments = call.arguments(py);
|
||||
let lifecycle = lifecycle(py, surface, call, asynchronous)?;
|
||||
run_call(py, machine, route, lifecycle, arguments, asynchronous)
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
mod callbacks;
|
||||
mod credentials;
|
||||
mod diagnostics;
|
||||
mod errors;
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
mod host;
|
||||
|
||||
use host::MessagesRouteHost;
|
||||
use litellm_callbacks_legacy_python::{
|
||||
LegacySurface, PassThroughStream, PublicCall, run_legacy_call,
|
||||
};
|
||||
use litellm_core::messages::route::{messages_machine, supports};
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
use crate::{
|
||||
callbacks::{CallSurface, PassThroughStream, PublicCall, run_python_call},
|
||||
errors::RustBridgeDeclined,
|
||||
};
|
||||
|
||||
const SURFACE: LegacySurface = LegacySurface {
|
||||
const SURFACE: CallSurface = CallSurface {
|
||||
call_type: "anthropic_messages",
|
||||
input_description: "Messages",
|
||||
stream: Some(PassThroughStream {
|
||||
|
|
@ -39,7 +39,7 @@ fn run_messages(
|
|||
"the Rust Messages route does not serve this provider",
|
||||
));
|
||||
}
|
||||
run_legacy_call(
|
||||
run_python_call(
|
||||
py,
|
||||
SURFACE,
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,14 @@ pub(super) fn to_pyerr(error: Error) -> PyErr {
|
|||
.ok();
|
||||
error
|
||||
}
|
||||
Error::InvalidProvider(provider) => {
|
||||
let error = core_error_to_pyerr(Error::InvalidProvider(provider.clone()).into());
|
||||
error
|
||||
.value(py)
|
||||
.setattr("ocr_invalid_provider", provider)
|
||||
.ok();
|
||||
error
|
||||
}
|
||||
Error::FileRead { path, source } if source.kind() == std::io::ErrorKind::NotFound => {
|
||||
PyFileNotFoundError::new_err(format!("File not found: {}", path.display()))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ use std::sync::{Arc, LazyLock};
|
|||
|
||||
use host::OcrRouteHost;
|
||||
use litellm_auth_gcp::VertexAuth;
|
||||
use litellm_callbacks_legacy_python::{LegacySurface, PublicCall, run_legacy_call};
|
||||
use litellm_core::ocr::route::ocr_machine;
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
|
|
@ -19,15 +18,20 @@ use pyo3::{
|
|||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::{errors::RustBridgeDeclined, http, python_settings::PythonSettings};
|
||||
use crate::{
|
||||
callbacks::{CallSurface, PublicCall, run_python_call},
|
||||
errors::RustBridgeDeclined,
|
||||
http,
|
||||
python_settings::PythonSettings,
|
||||
};
|
||||
|
||||
const SURFACE: LegacySurface = LegacySurface {
|
||||
const SURFACE: CallSurface = CallSurface {
|
||||
call_type: "ocr",
|
||||
input_description: "OCR document processing",
|
||||
stream: None,
|
||||
};
|
||||
|
||||
const ASYNC_SURFACE: LegacySurface = LegacySurface {
|
||||
const ASYNC_SURFACE: CallSurface = CallSurface {
|
||||
call_type: "aocr",
|
||||
..SURFACE
|
||||
};
|
||||
|
|
@ -52,7 +56,7 @@ fn run_ocr(
|
|||
secrets,
|
||||
)
|
||||
.map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?;
|
||||
run_legacy_call(
|
||||
run_python_call(
|
||||
py,
|
||||
if asynchronous { ASYNC_SURFACE } else { SURFACE },
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
|
|
|
|||
|
|
@ -83,21 +83,27 @@ enum ProjectedDocument {
|
|||
|
||||
impl ProjectedDocument {
|
||||
fn project(document: &Bound<'_, PyAny>) -> PyResult<Self> {
|
||||
let kind: String = document
|
||||
.get_item("type")
|
||||
.and_then(|value| value.extract())
|
||||
.map_err(|error| {
|
||||
let py = document.py();
|
||||
if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py)
|
||||
|| error.is_instance_of::<pyo3::exceptions::PyTypeError>(py)
|
||||
{
|
||||
ocr_error_to_pyerr(Error::RequestField {
|
||||
path: "document.type".into(),
|
||||
})
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let py = document.py();
|
||||
if !document.is_instance_of::<PyDict>() && !document.hasattr("__getitem__")? {
|
||||
return Err(ocr_error_to_pyerr(Error::DocumentNotObject));
|
||||
}
|
||||
let kind = match document.get_item("type") {
|
||||
Ok(value) => value,
|
||||
Err(error) if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py) => {
|
||||
return Err(ocr_error_to_pyerr(Error::InvalidDocumentType(
|
||||
"None".into(),
|
||||
)));
|
||||
}
|
||||
Err(error) if error.is_instance_of::<pyo3::exceptions::PyTypeError>(py) => {
|
||||
return Err(ocr_error_to_pyerr(Error::DocumentNotObject));
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let Ok(kind) = kind.extract::<String>() else {
|
||||
return Err(ocr_error_to_pyerr(Error::InvalidDocumentType(
|
||||
kind.str()?.to_string(),
|
||||
)));
|
||||
};
|
||||
if kind != "file" {
|
||||
return Ok(Self::Other(from_py(document)?));
|
||||
}
|
||||
|
|
@ -471,21 +477,25 @@ document = Document()
|
|||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::missing(c"{}")]
|
||||
#[case::non_string(c"{'type': 1}")]
|
||||
#[case::list(c"[]")]
|
||||
fn malformed_document_discriminators_are_bad_requests_naming_the_field(
|
||||
#[case::missing(
|
||||
c"{}",
|
||||
"Invalid document type: None. Must be 'document_url', 'image_url', or 'file'"
|
||||
)]
|
||||
#[case::non_string(
|
||||
c"{'type': 1}",
|
||||
"Invalid document type: 1. Must be 'document_url', 'image_url', or 'file'"
|
||||
)]
|
||||
#[case::list(c"[]", "document must be a dict with 'type' and URL/file field")]
|
||||
fn malformed_document_discriminators_are_bad_requests_with_the_python_message(
|
||||
#[case] document: &std::ffi::CStr,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let error = project_document(&py.eval(document, None, None).unwrap()).unwrap_err();
|
||||
let value = error.value(py);
|
||||
assert!(error.is_instance_of::<PyValueError>(py));
|
||||
assert_eq!(
|
||||
value.to_string(),
|
||||
"invalid OCR request field: document.type"
|
||||
);
|
||||
assert_eq!(value.to_string(), expected);
|
||||
assert_eq!(
|
||||
value
|
||||
.getattr("status_code")
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ def _prepare_ocr_request(
|
|||
raise litellm.BadRequestError(
|
||||
message="document must be a dict with 'type' and URL/file field",
|
||||
model=model,
|
||||
llm_provider=_error_provider(model, custom_llm_provider) or "",
|
||||
llm_provider=resolve_error_provider(model, custom_llm_provider) or "",
|
||||
)
|
||||
|
||||
normalized_document: Final = (
|
||||
|
|
@ -85,13 +85,13 @@ def _prepare_ocr_request(
|
|||
raise litellm.BadRequestError(
|
||||
message=f"Invalid document type: {doc_type}. Must be 'document_url', 'image_url', or 'file'",
|
||||
model=model,
|
||||
llm_provider=_error_provider(model, custom_llm_provider) or "",
|
||||
llm_provider=resolve_error_provider(model, custom_llm_provider) or "",
|
||||
)
|
||||
if not normalized_document.get(doc_type):
|
||||
raise litellm.BadRequestError(
|
||||
message="Document URL is required",
|
||||
model=model,
|
||||
llm_provider=_error_provider(model, custom_llm_provider) or "",
|
||||
llm_provider=resolve_error_provider(model, custom_llm_provider) or "",
|
||||
)
|
||||
|
||||
(
|
||||
|
|
@ -182,7 +182,7 @@ def _prepare_ocr_request(
|
|||
)
|
||||
|
||||
|
||||
def _error_provider(model: str, custom_llm_provider: str | None) -> str | None:
|
||||
def resolve_error_provider(model: str, custom_llm_provider: str | None) -> str | None:
|
||||
if custom_llm_provider is not None:
|
||||
return custom_llm_provider
|
||||
prefix: Final = model.partition("/")[0]
|
||||
|
|
@ -252,7 +252,7 @@ async def aocr(
|
|||
|
||||
return response
|
||||
except Exception as e:
|
||||
error_provider: Final = _error_provider(model, custom_llm_provider)
|
||||
error_provider: Final = resolve_error_provider(model, custom_llm_provider)
|
||||
error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model
|
||||
raise litellm.exception_type(
|
||||
model=error_model,
|
||||
|
|
@ -412,7 +412,7 @@ def ocr(
|
|||
|
||||
return response
|
||||
except Exception as e:
|
||||
error_provider: Final = _error_provider(model, custom_llm_provider)
|
||||
error_provider: Final = resolve_error_provider(model, custom_llm_provider)
|
||||
error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model
|
||||
raise litellm.exception_type(
|
||||
model=error_model,
|
||||
|
|
|
|||
272
litellm/rust_bridge/callbacks_v1_python.py
Normal file
272
litellm/rust_bridge/callbacks_v1_python.py
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
import inspect
|
||||
import json
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal, Protocol, TypeAlias, TypeGuard # noqa: TID251 # recursive JSON validation
|
||||
|
||||
from typing_extensions import ReadOnly, TypeAliasType, TypedDict
|
||||
|
||||
JSONScalar: TypeAlias = None | bool | int | float | str
|
||||
JSONValue = TypeAliasType( # rebind-ok: named recursive alias required for runtime validation
|
||||
"JSONValue", JSONScalar | Sequence["JSONValue"] | Mapping[str, "JSONValue"]
|
||||
)
|
||||
|
||||
SUPPORTED_SCHEMAS: Final = frozenset({1})
|
||||
EVENTS: Final = frozenset(
|
||||
{
|
||||
"call.started",
|
||||
"request.sending",
|
||||
"response.received",
|
||||
"call.succeeded",
|
||||
"call.failed",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TimingFactsV1(TypedDict):
|
||||
start_time: ReadOnly[float]
|
||||
end_time: ReadOnly[float]
|
||||
duration: ReadOnly[float]
|
||||
|
||||
|
||||
class CallFactsV1(TypedDict):
|
||||
start_time: ReadOnly[float]
|
||||
asynchronous: ReadOnly[bool]
|
||||
metadata: ReadOnly[Mapping[str, JSONValue]]
|
||||
metadata_dropped: ReadOnly[Sequence[str]]
|
||||
|
||||
|
||||
class RequestFactsV1(TypedDict):
|
||||
model: ReadOnly[str]
|
||||
custom_llm_provider: ReadOnly[str]
|
||||
optional_params: ReadOnly[JSONValue]
|
||||
url: ReadOnly[str]
|
||||
headers: ReadOnly[Sequence[tuple[str, str]]]
|
||||
body: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
ErrorFactsV1 = TypedDict( # rebind-ok: runtime TypedDict type for the reserved class field
|
||||
"ErrorFactsV1",
|
||||
{ # mutable-ok: functional TypedDict syntax requires a dict for the reserved class field
|
||||
"class": ReadOnly[str],
|
||||
"message": ReadOnly[str],
|
||||
"status_code": ReadOnly[int | None],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class CallStartedV1(CallFactsV1):
|
||||
type: ReadOnly[Literal["call.started"]]
|
||||
|
||||
|
||||
class RequestSendingV1(RequestFactsV1):
|
||||
type: ReadOnly[Literal["request.sending"]]
|
||||
|
||||
|
||||
class ResponseReceivedV1(TypedDict):
|
||||
type: ReadOnly[Literal["response.received"]]
|
||||
body: ReadOnly[str]
|
||||
|
||||
|
||||
class CallSucceededV1(TypedDict):
|
||||
type: ReadOnly[Literal["call.succeeded"]]
|
||||
timing: ReadOnly[TimingFactsV1]
|
||||
streamed: ReadOnly[bool]
|
||||
response: ReadOnly[JSONValue]
|
||||
response_error: ReadOnly[str | None]
|
||||
|
||||
|
||||
class CallFailedV1(TypedDict):
|
||||
type: ReadOnly[Literal["call.failed"]]
|
||||
timing: ReadOnly[TimingFactsV1]
|
||||
streamed: ReadOnly[bool]
|
||||
origin: ReadOnly[Literal["call", "host"]]
|
||||
error: ReadOnly[ErrorFactsV1]
|
||||
|
||||
|
||||
EventV1: TypeAlias = CallStartedV1 | RequestSendingV1 | ResponseReceivedV1 | CallSucceededV1 | CallFailedV1
|
||||
|
||||
|
||||
class EnvelopeV1(TypedDict):
|
||||
schema: ReadOnly[Literal[1]]
|
||||
call_id: ReadOnly[str]
|
||||
call_type: ReadOnly[str]
|
||||
seq: ReadOnly[int]
|
||||
event: ReadOnly[EventV1]
|
||||
|
||||
|
||||
class HeaderPatchV1(TypedDict, total=False):
|
||||
remove: ReadOnly[Sequence[str]]
|
||||
set: ReadOnly[Sequence[tuple[str, str]]]
|
||||
|
||||
|
||||
class WirePatchV1(TypedDict, total=False):
|
||||
headers: ReadOnly[HeaderPatchV1]
|
||||
body: ReadOnly[JSONValue]
|
||||
|
||||
|
||||
class Callback(Protocol):
|
||||
name: str
|
||||
schema: int
|
||||
events: frozenset[str]
|
||||
|
||||
def on_event(self, event: EnvelopeV1) -> None: ...
|
||||
|
||||
async def async_on_event(self, event: EnvelopeV1) -> None: ...
|
||||
|
||||
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
|
||||
|
||||
async def async_before_send(self, request: RequestFactsV1) -> WirePatchV1 | None: ...
|
||||
|
||||
|
||||
class RegistrationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedSchema(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownEvent(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class MissingHandler(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class EmptyObserverEvents(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class HandlerKindMismatch(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
class DuplicateName(RegistrationError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Subscriber:
|
||||
name: str
|
||||
schema: int
|
||||
events: frozenset[str]
|
||||
on_event: object | None
|
||||
async_on_event: object | None
|
||||
before_send: object | None
|
||||
async_before_send: object | None
|
||||
|
||||
|
||||
class _Registry:
|
||||
def __init__(self) -> None:
|
||||
self.subscribers: tuple[Subscriber, ...] = ()
|
||||
self.lock: Final = threading.Lock()
|
||||
|
||||
|
||||
_REGISTRY: Final = _Registry()
|
||||
|
||||
|
||||
def _handler(callback: Callback, name: str) -> object | None:
|
||||
return getattr(callback, name, None)
|
||||
|
||||
|
||||
def _validate_kind(name: str, handler: object | None, asynchronous: bool) -> None:
|
||||
if handler is None:
|
||||
return
|
||||
if not callable(handler):
|
||||
raise HandlerKindMismatch(f"{name} must be callable")
|
||||
is_async: Final = inspect.iscoroutinefunction(handler)
|
||||
if is_async != asynchronous:
|
||||
expected: Final = "async" if asynchronous else "sync"
|
||||
raise HandlerKindMismatch(f"{name} must be {expected}")
|
||||
|
||||
|
||||
def register(callback: Callback) -> None:
|
||||
schema: Final = callback.schema
|
||||
if schema not in SUPPORTED_SCHEMAS:
|
||||
raise UnsupportedSchema(f"unsupported callback schema: {schema}")
|
||||
events: Final = frozenset(callback.events)
|
||||
unknown: Final = events - EVENTS
|
||||
if unknown:
|
||||
raise UnknownEvent(f"unknown callback events: {sorted(unknown)}")
|
||||
|
||||
on_event: Final = _handler(callback, "on_event")
|
||||
async_on_event: Final = _handler(callback, "async_on_event")
|
||||
before_send: Final = _handler(callback, "before_send")
|
||||
async_before_send: Final = _handler(callback, "async_before_send")
|
||||
handlers: Final = (on_event, async_on_event, before_send, async_before_send)
|
||||
if all(handler is None for handler in handlers):
|
||||
raise MissingHandler("callback must define at least one handler")
|
||||
if (on_event is not None or async_on_event is not None) and not events:
|
||||
raise EmptyObserverEvents("observer callback events cannot be empty")
|
||||
|
||||
_validate_kind("on_event", on_event, False)
|
||||
_validate_kind("async_on_event", async_on_event, True)
|
||||
_validate_kind("before_send", before_send, False)
|
||||
_validate_kind("async_before_send", async_before_send, True)
|
||||
|
||||
subscriber: Final = Subscriber(
|
||||
name=callback.name,
|
||||
schema=schema,
|
||||
events=events,
|
||||
on_event=on_event,
|
||||
async_on_event=async_on_event,
|
||||
before_send=before_send,
|
||||
async_before_send=async_before_send,
|
||||
)
|
||||
with _REGISTRY.lock:
|
||||
if any(existing.name == subscriber.name for existing in _REGISTRY.subscribers):
|
||||
raise DuplicateName(f"duplicate callback name: {subscriber.name}")
|
||||
_REGISTRY.subscribers = (*_REGISTRY.subscribers, subscriber)
|
||||
|
||||
|
||||
def unregister(callback: Callback) -> None:
|
||||
with _REGISTRY.lock:
|
||||
_REGISTRY.subscribers = tuple(
|
||||
subscriber for subscriber in _REGISTRY.subscribers if subscriber.name != callback.name
|
||||
)
|
||||
|
||||
|
||||
def snapshot() -> tuple[Subscriber, ...]:
|
||||
return _REGISTRY.subscribers
|
||||
|
||||
|
||||
def _is_json_value(
|
||||
value: object,
|
||||
) -> TypeGuard[JSONValue]: # guard-ok: recursively validates the complete JSON value
|
||||
if value is None or isinstance(value, (bool, int, float, str)):
|
||||
return True
|
||||
if isinstance(value, list):
|
||||
return all(
|
||||
_is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # list items are validated recursively
|
||||
for item in value # pyright: ignore[reportUnknownVariableType] # list items are validated recursively
|
||||
)
|
||||
if isinstance(value, dict):
|
||||
return all(
|
||||
isinstance(key, str) and _is_json_value(item) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # mapping entries are validated recursively
|
||||
for key, item in value.items() # pyright: ignore[reportUnknownVariableType] # mapping entries are validated recursively
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def project_response(response: object) -> JSONValue:
|
||||
model_dump: Final = getattr(response, "model_dump", None)
|
||||
projected: Final = model_dump(mode="json") if callable(model_dump) else response
|
||||
json.dumps(projected, allow_nan=False)
|
||||
if _is_json_value(projected):
|
||||
return projected
|
||||
raise TypeError(f"unsupported response type: {type(response).__qualname__}")
|
||||
|
||||
|
||||
def new_call_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def report(name: str, event: str, error: BaseException) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
exception: Final = (type(error), error, error.__traceback__)
|
||||
verbose_logger.exception("callback %s failed while handling %s", name, event, exc_info=exception)
|
||||
|
|
@ -17,8 +17,8 @@ _UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]])
|
|||
|
||||
class UpstreamFailure(Exception):
|
||||
def __init__(self, response: httpx.Response, cause: Exception) -> None:
|
||||
super().__init__(str(cause))
|
||||
self.message: Final = str(cause)
|
||||
super().__init__(response.text)
|
||||
self.message: Final = response.text
|
||||
self.response: Final = response
|
||||
self.status_code: Final = response.status_code
|
||||
self.__cause__ = cause
|
||||
|
|
@ -49,13 +49,13 @@ class ExceptionMapper(Protocol):
|
|||
) -> Exception: ...
|
||||
|
||||
|
||||
def map_failure(error: Exception, model: str, request_provider: str, kwargs: Mapping[str, object]) -> Exception:
|
||||
def map_failure(error: Exception, model: str, request_provider: str | None, kwargs: Mapping[str, object]) -> Exception:
|
||||
mapper: Final = cast( # cast-ok: bounded adapter for the legacy public exception mapper
|
||||
ExceptionMapper, litellm.exception_type
|
||||
)
|
||||
try:
|
||||
return mapper(
|
||||
model=model.removeprefix(f"{request_provider}/"),
|
||||
model=model.removeprefix(f"{request_provider}/") if request_provider else model,
|
||||
custom_llm_provider=request_provider,
|
||||
original_exception=error,
|
||||
completion_kwargs=dict(kwargs), # mutable-ok: exception mapper requires owned kwargs
|
||||
|
|
@ -67,7 +67,11 @@ def map_failure(error: Exception, model: str, request_provider: str, kwargs: Map
|
|||
|
||||
|
||||
def map_native_failure(
|
||||
error: Exception, model: str, request_provider: str, kwargs: Mapping[str, object], api_base: str | None = None
|
||||
error: Exception,
|
||||
model: str,
|
||||
request_provider: str | None,
|
||||
kwargs: Mapping[str, object],
|
||||
api_base: str | None = None,
|
||||
) -> Exception:
|
||||
"""`map_failure`, reading a native `(status, body)` provider failure as the HTTP response it was."""
|
||||
original: Final = _upstream_failure(error, api_base)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse
|
||||
from litellm.ocr.main import resolve_error_provider
|
||||
from litellm.rust_bridge import failures
|
||||
from litellm.rust_bridge.failures import UpstreamFailure
|
||||
from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest
|
||||
|
|
@ -31,11 +32,31 @@ def arguments(request: LiteLLMOcrRequest) -> Mapping[str, object]:
|
|||
return request.kwargs
|
||||
|
||||
|
||||
def map_failure(error: Exception, request: LiteLLMOcrRequest, request_provider: str) -> Exception:
|
||||
def map_failure(error: Exception, request: LiteLLMOcrRequest, projected_provider: str) -> Exception:
|
||||
request_provider: Final = projected_provider or resolve_error_provider(request.model, request.custom_llm_provider)
|
||||
if getattr(error, "ocr_request_format_error", False):
|
||||
return litellm.UnsupportedParamsError(
|
||||
message=f"Invalid `req_format`: {request.kwargs.get('req_format')!r}. Expected 'native' or 'litellm'.",
|
||||
model=request.model.removeprefix(f"{request_provider}/"),
|
||||
llm_provider=request_provider,
|
||||
message=f"Invalid `req_format`: {request.kwargs.get('req_format')!r}. Expected one of litellm, native.",
|
||||
model=request.model.removeprefix(f"{request_provider}/") if request_provider else request.model,
|
||||
llm_provider=request_provider or "",
|
||||
)
|
||||
invalid_provider: Final = getattr(error, "ocr_invalid_provider", None)
|
||||
if isinstance(invalid_provider, str):
|
||||
return _invalid_provider_failure(invalid_provider, request)
|
||||
return failures.map_native_failure(error, request.model, request_provider, arguments(request), request.api_base)
|
||||
|
||||
|
||||
def _invalid_provider_failure(provider: str, request: LiteLLMOcrRequest) -> Exception:
|
||||
"""Python rejects a LiteLLM provider without OCR support differently from an unknown provider prefix."""
|
||||
if provider in litellm.provider_list:
|
||||
return failures.map_failure(
|
||||
ValueError(f"OCR is not supported for provider: {provider}"), request.model, None, arguments(request)
|
||||
)
|
||||
return litellm.BadRequestError(
|
||||
message=(
|
||||
"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. "
|
||||
f"You passed model={request.model}"
|
||||
),
|
||||
model=request.model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ IGNORE_FUNCTIONS = [
|
|||
"_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side.
|
||||
"_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
|
||||
"completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None.
|
||||
"_is_json_value", # walks a tree that already passed json.dumps(allow_nan=False): finite, acyclic, and bounded by JSON nesting depth.
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -52,13 +52,21 @@ class ExecutionVariant:
|
|||
|
||||
|
||||
class SubprocessWorker:
|
||||
def __init__(self, runner: SubprocessRunner, provider: ReplayServer, variant: ExecutionVariant) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
runner: SubprocessRunner,
|
||||
provider: ReplayServer,
|
||||
variant: ExecutionVariant,
|
||||
*,
|
||||
baseline: bool,
|
||||
) -> None:
|
||||
project_root: Final = str(PROJECT_ROOT)
|
||||
existing_pythonpath: Final = os.environ.get("PYTHONPATH")
|
||||
env: Final = {
|
||||
**os.environ,
|
||||
**dict(variant.environment),
|
||||
"LITELLM_USER_AGENT": runner.baseline_user_agent,
|
||||
# Both implementations honor LITELLM_USER_AGENT, so only the baseline carries the sentinel.
|
||||
**({"LITELLM_USER_AGENT": runner.baseline_user_agent} if baseline else {}),
|
||||
"PYTHONPATH": os.pathsep.join(path for path in (project_root, existing_pythonpath) if path),
|
||||
}
|
||||
self.mode: Final = variant.name
|
||||
|
|
@ -159,9 +167,11 @@ class SubprocessWorker:
|
|||
def execution_worker(
|
||||
runner: SubprocessRunner,
|
||||
variant: ExecutionVariant,
|
||||
*,
|
||||
baseline: bool,
|
||||
) -> Generator[SubprocessWorker]:
|
||||
with replay_server() as provider:
|
||||
worker: Final = SubprocessWorker(runner, provider, variant)
|
||||
worker: Final = SubprocessWorker(runner, provider, variant, baseline=baseline)
|
||||
try:
|
||||
yield worker
|
||||
finally:
|
||||
|
|
@ -174,8 +184,8 @@ def execution_worker_pair(
|
|||
baseline: ExecutionVariant,
|
||||
candidate: ExecutionVariant,
|
||||
) -> Generator[tuple[SubprocessWorker, SubprocessWorker]]:
|
||||
with execution_worker(runner, baseline) as baseline_worker:
|
||||
with execution_worker(runner, candidate) as candidate_worker:
|
||||
with execution_worker(runner, baseline, baseline=True) as baseline_worker:
|
||||
with execution_worker(runner, candidate, baseline=False) as candidate_worker:
|
||||
yield baseline_worker, candidate_worker
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -234,14 +234,6 @@ INVALID_OCR_CASES: Final = (
|
|||
expected_status_code=500,
|
||||
expected_message="OCR is not supported for provider: openai",
|
||||
),
|
||||
InvalidOcrCase(
|
||||
name="unsupported_reducto_model",
|
||||
model="reducto/parse-v4",
|
||||
document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_message="OCR is not supported for provider: reducto",
|
||||
),
|
||||
InvalidOcrCase(
|
||||
name="unknown_provider_prefix",
|
||||
model="not_a_provider/model",
|
||||
|
|
@ -254,40 +246,40 @@ INVALID_OCR_CASES: Final = (
|
|||
name="non_object_document",
|
||||
model="mistral/mistral-ocr-latest",
|
||||
document=[],
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_exception_type="litellm.exceptions.BadRequestError",
|
||||
expected_status_code=400,
|
||||
expected_message="document must be a dict",
|
||||
),
|
||||
InvalidOcrCase(
|
||||
name="missing_document_type",
|
||||
model="mistral/mistral-ocr-latest",
|
||||
document={},
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_exception_type="litellm.exceptions.BadRequestError",
|
||||
expected_status_code=400,
|
||||
expected_message="Invalid document type: None",
|
||||
),
|
||||
InvalidOcrCase(
|
||||
name="unsupported_document_type",
|
||||
model="mistral/mistral-ocr-latest",
|
||||
document={"type": "text", "text": "not a document"},
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_exception_type="litellm.exceptions.BadRequestError",
|
||||
expected_status_code=400,
|
||||
expected_message="Invalid document type: text",
|
||||
),
|
||||
InvalidOcrCase(
|
||||
name="missing_document_url",
|
||||
model="azure_ai/doc-intelligence/prebuilt-read",
|
||||
document={"type": "document_url"},
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_exception_type="litellm.exceptions.BadRequestError",
|
||||
expected_status_code=400,
|
||||
expected_message="Document URL is required",
|
||||
),
|
||||
InvalidOcrCase(
|
||||
name="missing_image_url",
|
||||
model="azure_ai/doc-intelligence/prebuilt-read",
|
||||
document={"type": "image_url"},
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_exception_type="litellm.exceptions.BadRequestError",
|
||||
expected_status_code=400,
|
||||
expected_message="Document URL is required",
|
||||
),
|
||||
InvalidOcrCase(
|
||||
|
|
@ -303,8 +295,8 @@ INVALID_OCR_CASES: Final = (
|
|||
name="invalid_document_intelligence_pages",
|
||||
model="azure_ai/doc-intelligence/prebuilt-read",
|
||||
document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_exception_type="litellm.exceptions.BadRequestError",
|
||||
expected_status_code=400,
|
||||
expected_message="`pages` integers must be >= 0",
|
||||
extra_kwargs=(("pages", [-1]),),
|
||||
),
|
||||
|
|
@ -312,8 +304,8 @@ INVALID_OCR_CASES: Final = (
|
|||
name="invalid_document_intelligence_features",
|
||||
model="azure_ai/doc-intelligence/prebuilt-read",
|
||||
document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_exception_type="litellm.exceptions.BadRequestError",
|
||||
expected_status_code=400,
|
||||
expected_message="Invalid `features` for Azure Document Intelligence",
|
||||
extra_kwargs=(("features", [1]),),
|
||||
),
|
||||
|
|
|
|||
|
|
@ -83,3 +83,33 @@ def test_map_failure_leaves_non_upstream_errors_unwrapped() -> None:
|
|||
def test_map_failure_reports_invalid_request_format_as_unsupported_params() -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="Invalid `req_format`: 'markdown'"):
|
||||
raise map_failure(RustFormatError(), REQUEST, "mistral")
|
||||
|
||||
|
||||
def test_failure_before_projection_reports_the_provider_python_reports() -> None:
|
||||
error: Final = ValueError("Document URL is required")
|
||||
error.status_code = 400 # type: ignore[attr-defined]
|
||||
|
||||
mapped: Final = map_failure(error, REQUEST, "")
|
||||
|
||||
assert isinstance(mapped, litellm.BadRequestError)
|
||||
assert mapped.llm_provider == "mistral"
|
||||
assert "BadRequestError: - " not in str(mapped)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider", "expected_type", "expected_message"),
|
||||
[
|
||||
("openai", litellm.APIConnectionError, "OCR is not supported for provider: openai"),
|
||||
("not_a_provider", litellm.BadRequestError, "LLM Provider NOT provided"),
|
||||
],
|
||||
)
|
||||
def test_invalid_provider_failures_follow_the_python_route(
|
||||
provider: str, expected_type: type[Exception], expected_message: str
|
||||
) -> None:
|
||||
error: Final = ValueError(f"invalid provider: {provider}")
|
||||
error.ocr_invalid_provider = provider # type: ignore[attr-defined]
|
||||
|
||||
mapped: Final = map_failure(error, REQUEST, "")
|
||||
|
||||
assert isinstance(mapped, expected_type)
|
||||
assert expected_message in str(mapped)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.rust_bridge import callbacks_legacy_python as legacy
|
||||
from litellm.rust_bridge import callbacks_legacy_python as callbacks
|
||||
from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup
|
||||
|
||||
_OCR_KWARGS: Final = MappingProxyType(
|
||||
|
|
@ -81,12 +81,10 @@ def test_setup_builds_a_logger_when_none_is_supplied(call_type: str, kwargs: Map
|
|||
assert result.logger.litellm_call_id == result.kwargs["litellm_call_id"]
|
||||
|
||||
|
||||
CONTRACT_PATH: Final = (
|
||||
Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json"
|
||||
)
|
||||
CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json"
|
||||
|
||||
|
||||
def test_the_rust_contract_matches_the_shim_signatures() -> None:
|
||||
contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text())
|
||||
|
||||
assert contract == {name: list(inspect.signature(getattr(legacy, name)).parameters) for name in contract}
|
||||
assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract}
|
||||
|
|
|
|||
178
tests/test_litellm/rust_bridge/test_callbacks_v1_python.py
Normal file
178
tests/test_litellm/rust_bridge/test_callbacks_v1_python.py
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
import inspect
|
||||
import warnings
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from litellm.rust_bridge import callbacks_v1_python as callbacks
|
||||
|
||||
ROOT: Final = Path(__file__).parents[3]
|
||||
CONTRACT_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1-python/python_contract.json"
|
||||
GOLDEN_PATH: Final = ROOT / "litellm-rust/crates/callbacks-v1/golden/v1"
|
||||
|
||||
|
||||
def _clear_registry() -> None:
|
||||
for subscriber in callbacks.snapshot():
|
||||
callbacks.unregister(cast(callbacks.Callback, subscriber))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_registry() -> Generator[None, None, None]:
|
||||
_clear_registry()
|
||||
yield
|
||||
_clear_registry()
|
||||
|
||||
|
||||
def test_the_rust_contract_matches_the_shim_signatures() -> None:
|
||||
contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text())
|
||||
assert contract == {name: list(inspect.signature(getattr(callbacks, name)).parameters) for name in contract}
|
||||
|
||||
|
||||
def test_v1_golden_envelopes_match_the_typed_contract() -> None:
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", message="Items .*ReadOnly.*", category=UserWarning)
|
||||
adapter: Final = TypeAdapter(callbacks.EnvelopeV1)
|
||||
envelopes: Final = [adapter.validate_json(path.read_text(), strict=True) for path in GOLDEN_PATH.glob("*.json")]
|
||||
assert {envelope["event"]["type"] for envelope in envelopes} == callbacks.EVENTS
|
||||
assert {envelope["schema"] for envelope in envelopes} == callbacks.SUPPORTED_SCHEMAS
|
||||
|
||||
|
||||
class ValidCallback:
|
||||
name: str = "valid"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset({"call.started"})
|
||||
|
||||
def on_event(self, event: callbacks.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("callback", "error"),
|
||||
[
|
||||
(
|
||||
type(
|
||||
"Unsupported",
|
||||
(),
|
||||
{"name": "unsupported", "schema": 2, "events": frozenset(), "before_send": lambda *_: None},
|
||||
)(),
|
||||
callbacks.UnsupportedSchema,
|
||||
),
|
||||
(
|
||||
type(
|
||||
"Unknown",
|
||||
(),
|
||||
{"name": "unknown", "schema": 1, "events": frozenset({"new.event"}), "on_event": lambda *_: None},
|
||||
)(),
|
||||
callbacks.UnknownEvent,
|
||||
),
|
||||
(
|
||||
type("Missing", (), {"name": "missing", "schema": 1, "events": frozenset()})(),
|
||||
callbacks.MissingHandler,
|
||||
),
|
||||
(
|
||||
type("Empty", (), {"name": "empty", "schema": 1, "events": frozenset(), "on_event": lambda *_: None})(),
|
||||
callbacks.EmptyObserverEvents,
|
||||
),
|
||||
(
|
||||
type(
|
||||
"AsyncFunction",
|
||||
(),
|
||||
{
|
||||
"name": "async-function",
|
||||
"schema": 1,
|
||||
"events": frozenset({"call.started"}),
|
||||
"async_on_event": lambda *_: None,
|
||||
},
|
||||
)(),
|
||||
callbacks.HandlerKindMismatch,
|
||||
),
|
||||
],
|
||||
ids=["schema", "event", "missing", "empty-events", "async-kind"],
|
||||
)
|
||||
def test_register_rejects_invalid_callbacks(callback: object, error: type[ValueError]) -> None:
|
||||
with pytest.raises(error):
|
||||
callbacks.register(callback) # pyright: ignore[reportArgumentType] # invalid callback shapes are the subject of the test
|
||||
|
||||
|
||||
class WrongSyncKind:
|
||||
name: str = "wrong-sync-kind"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset({"call.started"})
|
||||
|
||||
async def on_event(self, event: callbacks.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
def test_register_rejects_coroutine_in_sync_slot() -> None:
|
||||
with pytest.raises(callbacks.HandlerKindMismatch):
|
||||
callbacks.register(cast(callbacks.Callback, WrongSyncKind()))
|
||||
|
||||
|
||||
class AsyncObserver:
|
||||
name: str = "async-observer"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset({"call.started"})
|
||||
|
||||
async def async_on_event(self, event: callbacks.EnvelopeV1) -> None:
|
||||
del event
|
||||
|
||||
|
||||
class SyncInterceptor:
|
||||
name: str = "sync-interceptor"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset()
|
||||
|
||||
def before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None:
|
||||
del request
|
||||
return None
|
||||
|
||||
|
||||
class AsyncInterceptor:
|
||||
name: str = "async-interceptor"
|
||||
schema: int = 1
|
||||
events: frozenset[str] = frozenset()
|
||||
|
||||
async def async_before_send(self, request: callbacks.RequestFactsV1) -> callbacks.WirePatchV1 | None:
|
||||
del request
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("callback", [ValidCallback(), AsyncObserver(), SyncInterceptor(), AsyncInterceptor()])
|
||||
def test_register_accepts_each_handler_family(callback: callbacks.Callback) -> None:
|
||||
callbacks.register(callback)
|
||||
assert len(callbacks.snapshot()) == 1
|
||||
|
||||
|
||||
def test_register_rejects_duplicate_names() -> None:
|
||||
callbacks.register(cast(callbacks.Callback, ValidCallback()))
|
||||
with pytest.raises(callbacks.DuplicateName):
|
||||
callbacks.register(cast(callbacks.Callback, ValidCallback()))
|
||||
|
||||
|
||||
def test_snapshot_is_stable_across_later_registry_writes() -> None:
|
||||
first: Final = ValidCallback()
|
||||
second: Final = AsyncObserver()
|
||||
callbacks.register(cast(callbacks.Callback, first))
|
||||
captured: Final = callbacks.snapshot()
|
||||
callbacks.register(cast(callbacks.Callback, second))
|
||||
callbacks.unregister(cast(callbacks.Callback, first))
|
||||
assert callbacks.snapshot() is not captured
|
||||
assert tuple(subscriber.name for subscriber in captured) == ("valid",)
|
||||
|
||||
|
||||
class ResponseModel(BaseModel):
|
||||
value: int
|
||||
|
||||
|
||||
def test_project_response_supports_pydantic_and_json_values() -> None:
|
||||
assert callbacks.project_response(ResponseModel(value=3)) == {"value": 3}
|
||||
original: Final = {"nested": [1, True, None]}
|
||||
assert callbacks.project_response(original) is original
|
||||
|
||||
|
||||
def test_project_response_rejects_unsupported_objects() -> None:
|
||||
with pytest.raises(TypeError):
|
||||
callbacks.project_response(object())
|
||||
|
|
@ -52,3 +52,18 @@ def test_kwargs_are_handed_to_the_mapper_as_owned_copies(monkeypatch: pytest.Mon
|
|||
assert seen[0]["completion_kwargs"] is not request_kwargs
|
||||
assert seen[0]["model"] == "gpt-4o"
|
||||
assert seen[0]["custom_llm_provider"] == "openai"
|
||||
|
||||
|
||||
def test_native_provider_failure_message_is_the_provider_body_not_the_args_tuple() -> None:
|
||||
class NativeUpstream(Exception):
|
||||
headers = [("retry-after", "7")]
|
||||
|
||||
native_error: Final = NativeUpstream(429, '{"message": "slow down"}')
|
||||
|
||||
mapped: Final = failures.map_native_failure(
|
||||
native_error, "mistral/mistral-ocr-latest", "mistral", MappingProxyType({}), "https://api.mistral.ai/v1"
|
||||
)
|
||||
|
||||
assert isinstance(mapped, litellm.RateLimitError)
|
||||
assert '{"message": "slow down"}' in str(mapped)
|
||||
assert "(429," not in str(mapped)
|
||||
|
|
|
|||
141
tests/test_litellm_rust/ocr/test_callbacks_v1.py
Normal file
141
tests/test_litellm_rust/ocr/test_callbacks_v1.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
from collections.abc import Generator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.rust_bridge import callbacks_v1_python
|
||||
from litellm.rust_bridge.callbacks_v1_python import EVENTS, EnvelopeV1, RequestFactsV1, WirePatchV1
|
||||
from tests.test_litellm_rust.support.callback_recorder import drain_logging
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
|
||||
from tests.test_litellm_rust.support.requests import (
|
||||
OCR_RESPONSE,
|
||||
call_native_aocr,
|
||||
call_native_ocr,
|
||||
request_body,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ocr_server(recording_server: RecordingServer) -> RecordingServer:
|
||||
recording_server.default_response = ResponseSpec(body=OCR_RESPONSE)
|
||||
return recording_server
|
||||
|
||||
|
||||
class Observer:
|
||||
name: Final = "observer"
|
||||
schema: Final = 1
|
||||
events: Final = EVENTS
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.envelopes: Final[list[EnvelopeV1]] = []
|
||||
|
||||
def on_event(self, event: EnvelopeV1) -> None:
|
||||
self.envelopes.append(event)
|
||||
|
||||
|
||||
class Interceptor:
|
||||
name: Final = "interceptor"
|
||||
schema: Final = 1
|
||||
events: Final[frozenset[str]] = frozenset()
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.bodies: Final[list[object]] = []
|
||||
|
||||
def before_send(self, request: RequestFactsV1) -> WirePatchV1 | None:
|
||||
self.bodies.append(request["body"])
|
||||
return {"headers": {"set": [("x-v1-interceptor", "seen")]}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def observer() -> Generator[Observer]:
|
||||
subscriber: Final = Observer()
|
||||
callbacks_v1_python.register(subscriber)
|
||||
try:
|
||||
yield subscriber
|
||||
finally:
|
||||
callbacks_v1_python.unregister(subscriber)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def interceptor() -> Generator[Interceptor]:
|
||||
subscriber: Final = Interceptor()
|
||||
callbacks_v1_python.register(subscriber)
|
||||
try:
|
||||
yield subscriber
|
||||
finally:
|
||||
callbacks_v1_python.unregister(subscriber)
|
||||
|
||||
|
||||
class LegacyCallId(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
self.call_ids: Final[list[str]] = []
|
||||
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
self.call_ids.append(kwargs["litellm_call_id"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_native_ocr_runs_legacy_and_v1_callbacks_on_the_same_call(
|
||||
ocr_server: RecordingServer, observer: Observer, asynchronous: bool
|
||||
) -> None:
|
||||
legacy: Final = LegacyCallId()
|
||||
|
||||
if asynchronous:
|
||||
await call_native_aocr(ocr_server, callbacks=[legacy])
|
||||
else:
|
||||
call_native_ocr(ocr_server, callbacks=[legacy])
|
||||
await drain_logging()
|
||||
|
||||
assert [envelope["event"]["type"] for envelope in observer.envelopes] == [
|
||||
"call.started",
|
||||
"request.sending",
|
||||
"response.received",
|
||||
"call.succeeded",
|
||||
]
|
||||
assert [envelope["seq"] for envelope in observer.envelopes] == sorted(
|
||||
envelope["seq"] for envelope in observer.envelopes
|
||||
)
|
||||
assert len(legacy.call_ids) == 1
|
||||
assert {envelope["call_id"] for envelope in observer.envelopes} == set(legacy.call_ids)
|
||||
|
||||
|
||||
def test_native_ocr_v1_interceptor_sees_the_wire_as_the_legacy_callback_left_it(
|
||||
ocr_server: RecordingServer, interceptor: Interceptor
|
||||
) -> None:
|
||||
class Edit(CustomLogger):
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
request_body(kwargs)["include_image_base64"] = True
|
||||
|
||||
call_native_ocr(ocr_server, callbacks=[Edit()])
|
||||
|
||||
assert len(interceptor.bodies) == 1
|
||||
body: Final = interceptor.bodies[0]
|
||||
assert isinstance(body, dict)
|
||||
assert body["include_image_base64"] is True
|
||||
request: Final = ocr_server.requests[-1]
|
||||
assert request.headers["x-v1-interceptor"] == "seen"
|
||||
|
||||
|
||||
def test_native_ocr_failure_reaches_both_contracts_once(
|
||||
recording_server: RecordingServer, observer: Observer
|
||||
) -> None:
|
||||
recording_server.default_response = ResponseSpec(status=400, body={"message": "bad document"})
|
||||
failures: Final[list[object]] = []
|
||||
|
||||
class Failure(CustomLogger):
|
||||
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
failures.append(kwargs["exception"])
|
||||
|
||||
with pytest.raises(Exception) as raised: # noqa: PT011 # the mapped provider error class is covered elsewhere
|
||||
call_native_ocr(recording_server, callbacks=[Failure()])
|
||||
|
||||
assert failures == [raised.value]
|
||||
terminal: Final = [
|
||||
envelope for envelope in observer.envelopes if envelope["event"]["type"] in {"call.succeeded", "call.failed"}
|
||||
]
|
||||
assert [envelope["event"]["type"] for envelope in terminal] == ["call.failed"]
|
||||
assert observer.envelopes[-1] is terminal[0]
|
||||
Loading…
Add table
Reference in a new issue