refactor(callbacks): move lifecycle dispatch into Rust

This commit is contained in:
Yujong Lee 2026-09-16 19:09:58 -07:00
parent d5acf2f9e1
commit 7073f5c0b8
49 changed files with 5793 additions and 1464 deletions

View file

@ -0,0 +1,857 @@
use super::host::HostPhase;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct CallbackId(pub u64);
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Delivery {
Inline,
Await,
Worker,
Background,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CallbackKind {
CustomLogger,
Callable { internal: bool },
Named { known: bool },
Opaque,
}
impl CallbackKind {
fn runs_sync_handler_for_async_call(self) -> bool {
match self {
Self::Callable { internal } => !internal,
Self::Named { known } => !known,
Self::CustomLogger | Self::Opaque => false,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CallbackMethod {
LogPreApiCall,
LogPostApiCall,
PreCallDeploymentHook,
PostCallSuccessDeploymentHook,
PostCallFailureDeploymentHook,
LoggingHook,
AsyncLoggingHook,
LogSuccessEvent,
AsyncLogSuccessEvent,
LogFailureEvent,
AsyncLogFailureEvent,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CallbackInvocation {
pub target: CallbackId,
pub method: CallbackMethod,
pub delivery: Delivery,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TargetErrorPolicy {
Contain,
Propagate,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LoggedMarker {
SyncSuccess,
AsyncSuccess,
SyncFailure,
AsyncFailure,
}
impl LoggedMarker {
pub const fn key(self) -> &'static str {
match self {
Self::SyncSuccess => "has_logged_sync_success",
Self::AsyncSuccess => "has_logged_async_success",
Self::SyncFailure => "has_logged_sync_failure",
Self::AsyncFailure => "has_logged_async_failure",
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CallbackFamily {
RequestPreCall,
RequestPostCall,
DeploymentPreCall,
DeploymentPostCall,
DeploymentFailure,
SyncSuccess,
AsyncSuccess,
SyncFailure,
AsyncFailure,
}
impl CallbackFamily {
pub const fn dispatch_method(self) -> CallbackMethod {
match self {
Self::RequestPreCall => CallbackMethod::LogPreApiCall,
Self::RequestPostCall => CallbackMethod::LogPostApiCall,
Self::DeploymentPreCall => CallbackMethod::PreCallDeploymentHook,
Self::DeploymentPostCall => CallbackMethod::PostCallSuccessDeploymentHook,
Self::DeploymentFailure => CallbackMethod::PostCallFailureDeploymentHook,
Self::SyncSuccess => CallbackMethod::LogSuccessEvent,
Self::AsyncSuccess => CallbackMethod::AsyncLogSuccessEvent,
Self::SyncFailure => CallbackMethod::LogFailureEvent,
Self::AsyncFailure => CallbackMethod::AsyncLogFailureEvent,
}
}
pub const fn hook_method(self) -> Option<CallbackMethod> {
match self {
Self::SyncSuccess => Some(CallbackMethod::LoggingHook),
Self::AsyncSuccess => Some(CallbackMethod::AsyncLoggingHook),
_ => None,
}
}
pub const fn marker(self) -> Option<LoggedMarker> {
match self {
Self::SyncSuccess => Some(LoggedMarker::SyncSuccess),
Self::AsyncSuccess => Some(LoggedMarker::AsyncSuccess),
Self::SyncFailure => Some(LoggedMarker::SyncFailure),
Self::AsyncFailure => Some(LoggedMarker::AsyncFailure),
_ => None,
}
}
pub const fn prepares_logging(self) -> bool {
self.marker().is_some()
}
pub const fn error_policy(self) -> TargetErrorPolicy {
match self {
Self::DeploymentPreCall | Self::DeploymentPostCall => TargetErrorPolicy::Propagate,
_ => TargetErrorPolicy::Contain,
}
}
pub fn targets(self, global: &[CallbackId], dynamic: Option<&[CallbackId]>) -> Vec<CallbackId> {
match self {
Self::RequestPreCall | Self::RequestPostCall => global
.iter()
.chain(dynamic.unwrap_or_default())
.copied()
.collect(),
Self::DeploymentPreCall | Self::DeploymentPostCall | Self::DeploymentFailure => {
global.to_vec()
}
Self::SyncSuccess | Self::AsyncSuccess | Self::SyncFailure | Self::AsyncFailure => {
let Some(dynamic) = dynamic else {
return global.to_vec();
};
let mut seen = std::collections::HashSet::new();
dynamic
.iter()
.chain(global)
.copied()
.filter(|id| seen.insert(*id))
.collect()
}
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ReleaseGate {
Immediate,
Deferred,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Dispatch {
pub family: CallbackFamily,
pub delivery: Delivery,
pub gate: ReleaseGate,
}
impl Dispatch {
pub const fn immediate(family: CallbackFamily, delivery: Delivery) -> Self {
Self {
family,
delivery,
gate: ReleaseGate::Immediate,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SuccessFacts {
pub asynchronous: bool,
pub internal: bool,
pub fallbacks: bool,
pub deferred: bool,
pub sync_target_kinds: Vec<CallbackKind>,
}
pub fn plan_success(facts: &SuccessFacts) -> Vec<Dispatch> {
if !facts.asynchronous {
return vec![Dispatch {
family: CallbackFamily::SyncSuccess,
delivery: Delivery::Worker,
gate: ReleaseGate::Immediate,
}];
}
let background = (!facts.internal && !facts.fallbacks).then_some(Dispatch {
family: CallbackFamily::AsyncSuccess,
delivery: Delivery::Background,
gate: if facts.deferred {
ReleaseGate::Deferred
} else {
ReleaseGate::Immediate
},
});
let worker = facts
.sync_target_kinds
.iter()
.any(|kind| kind.runs_sync_handler_for_async_call())
.then_some(Dispatch {
family: CallbackFamily::SyncSuccess,
delivery: Delivery::Worker,
gate: ReleaseGate::Immediate,
});
background.into_iter().chain(worker).collect()
}
pub fn plan_failure(phase: HostPhase, asynchronous: bool, internal: bool) -> Option<Dispatch> {
if asynchronous && internal {
return None;
}
match phase {
HostPhase::Failure => Some(Dispatch::immediate(
CallbackFamily::SyncFailure,
Delivery::Inline,
)),
HostPhase::AsyncFailure => Some(Dispatch::immediate(
CallbackFamily::AsyncFailure,
Delivery::Await,
)),
_ => None,
}
}
pub fn plan_request(family: CallbackFamily) -> Dispatch {
let delivery = match family {
CallbackFamily::RequestPreCall | CallbackFamily::RequestPostCall => Delivery::Inline,
_ => Delivery::Await,
};
Dispatch::immediate(family, delivery)
}
pub fn object_target_eligible(
sync_request: bool,
method: CallbackMethod,
kind: CallbackKind,
) -> bool {
match (method, kind) {
(
CallbackMethod::LogSuccessEvent | CallbackMethod::LogFailureEvent,
CallbackKind::CustomLogger | CallbackKind::Callable { .. },
) => sync_request,
_ => true,
}
}
pub trait DispatchFacts {
fn eligible(&mut self, target: CallbackId, method: CallbackMethod) -> bool;
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DispatchStep {
PrepareLogging,
Invoke(CallbackInvocation),
MarkLogged(LoggedMarker),
Complete { aborted: bool },
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum InvocationOutcome {
Completed,
Failed,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Position {
Prepare,
Hook(usize),
Mark,
Dispatch(usize),
Complete { aborted: bool },
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CursorFacts {
pub already_logged: bool,
pub stream: bool,
pub sync_request: bool,
}
pub struct DispatchCursor {
dispatch: Dispatch,
targets: Vec<CallbackId>,
facts: CursorFacts,
position: Position,
}
impl DispatchCursor {
pub fn start(dispatch: Dispatch, targets: Vec<CallbackId>, facts: CursorFacts) -> Self {
let family = dispatch.family;
let position = if family.marker().is_some() && facts.already_logged {
Position::Complete { aborted: false }
} else if family.prepares_logging() {
Position::Prepare
} else {
Position::Dispatch(0)
};
Self {
dispatch,
targets,
facts,
position,
}
}
pub fn object_target_eligible(&self, method: CallbackMethod, kind: CallbackKind) -> bool {
object_target_eligible(self.facts.sync_request, method, kind)
}
pub const fn family(&self) -> CallbackFamily {
self.dispatch.family
}
pub const fn delivery(&self) -> Delivery {
self.dispatch.delivery
}
pub fn targets(&self) -> &[CallbackId] {
&self.targets
}
pub fn accept(&mut self, outcome: InvocationOutcome) {
if outcome == InvocationOutcome::Failed
&& self.dispatch.family.error_policy() == TargetErrorPolicy::Propagate
{
self.position = Position::Complete { aborted: true };
}
}
pub fn next(&mut self, facts: &mut dyn DispatchFacts) -> DispatchStep {
loop {
match self.position {
Position::Prepare => {
self.position = self.after_prepare();
return DispatchStep::PrepareLogging;
}
Position::Hook(index) => {
let Some(method) = self.dispatch.family.hook_method() else {
self.position = Position::Mark;
continue;
};
let Some(target) = self.targets.get(index).copied() else {
self.position = Position::Mark;
continue;
};
self.position = Position::Hook(index + 1);
if facts.eligible(target, method) {
return DispatchStep::Invoke(self.invocation(target, method));
}
}
Position::Mark => {
self.position = Position::Dispatch(0);
match self.dispatch.family.marker() {
Some(marker) if !self.facts.stream => {
return DispatchStep::MarkLogged(marker);
}
_ => continue,
}
}
Position::Dispatch(index) => {
let Some(target) = self.targets.get(index).copied() else {
self.position = Position::Complete { aborted: false };
continue;
};
self.position = Position::Dispatch(index + 1);
let method = self.dispatch.family.dispatch_method();
if facts.eligible(target, method) {
return DispatchStep::Invoke(self.invocation(target, method));
}
}
Position::Complete { aborted } => return DispatchStep::Complete { aborted },
}
}
}
fn after_prepare(&self) -> Position {
if self.dispatch.family.hook_method().is_some() {
Position::Hook(0)
} else {
Position::Mark
}
}
const fn invocation(&self, target: CallbackId, method: CallbackMethod) -> CallbackInvocation {
CallbackInvocation {
target,
method,
delivery: self.dispatch.delivery,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
struct AllEligible;
impl DispatchFacts for AllEligible {
fn eligible(&mut self, _: CallbackId, _: CallbackMethod) -> bool {
true
}
}
struct Gate<F: FnMut(CallbackId, CallbackMethod) -> bool>(F);
impl<F: FnMut(CallbackId, CallbackMethod) -> bool> DispatchFacts for Gate<F> {
fn eligible(&mut self, target: CallbackId, method: CallbackMethod) -> bool {
(self.0)(target, method)
}
}
fn drain(cursor: &mut DispatchCursor, facts: &mut dyn DispatchFacts) -> Vec<DispatchStep> {
let mut steps = Vec::new();
loop {
let step = cursor.next(facts);
steps.push(step);
match step {
DispatchStep::Complete { .. } => return steps,
DispatchStep::Invoke(_) => cursor.accept(InvocationOutcome::Completed),
_ => {}
}
}
}
fn ids(values: &[u64]) -> Vec<CallbackId> {
values.iter().copied().map(CallbackId).collect()
}
fn delivery_for(family: CallbackFamily) -> Delivery {
match family {
CallbackFamily::SyncSuccess => Delivery::Worker,
CallbackFamily::AsyncSuccess => Delivery::Background,
CallbackFamily::SyncFailure
| CallbackFamily::RequestPreCall
| CallbackFamily::RequestPostCall => Delivery::Inline,
_ => Delivery::Await,
}
}
fn start(
family: CallbackFamily,
targets: Vec<CallbackId>,
already_logged: bool,
stream: bool,
) -> DispatchCursor {
DispatchCursor::start(
Dispatch::immediate(family, delivery_for(family)),
targets,
CursorFacts {
already_logged,
stream,
sync_request: true,
},
)
}
#[test]
fn terminal_families_order_dynamic_before_global_and_keep_first_duplicate() {
let combined = CallbackFamily::SyncSuccess.targets(&ids(&[3, 1, 4]), Some(&ids(&[1, 2])));
assert_eq!(combined, ids(&[1, 2, 3, 4]));
}
#[test]
fn terminal_families_copy_global_without_dedup_when_dynamic_is_absent() {
let combined = CallbackFamily::AsyncFailure.targets(&ids(&[3, 3, 1]), None);
assert_eq!(combined, ids(&[3, 3, 1]));
}
#[test]
fn request_families_order_global_before_dynamic_without_dedup() {
let combined = CallbackFamily::RequestPreCall.targets(&ids(&[1, 2]), Some(&ids(&[2, 3])));
assert_eq!(combined, ids(&[1, 2, 2, 3]));
}
#[test]
fn success_runs_every_hook_before_any_dispatch_and_marks_between_passes() {
let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1, 2]), false, false);
let steps = drain(&mut cursor, &mut AllEligible);
let invocation = |target, method| {
DispatchStep::Invoke(CallbackInvocation {
target: CallbackId(target),
method,
delivery: Delivery::Worker,
})
};
assert_eq!(
steps,
vec![
DispatchStep::PrepareLogging,
invocation(1, CallbackMethod::LoggingHook),
invocation(2, CallbackMethod::LoggingHook),
DispatchStep::MarkLogged(LoggedMarker::SyncSuccess),
invocation(1, CallbackMethod::LogSuccessEvent),
invocation(2, CallbackMethod::LogSuccessEvent),
DispatchStep::Complete { aborted: false },
]
);
}
#[test]
fn async_success_uses_async_leaf_methods_and_background_delivery() {
let mut cursor = start(CallbackFamily::AsyncSuccess, ids(&[7]), false, false);
let steps = drain(&mut cursor, &mut AllEligible);
let methods: Vec<_> = steps
.iter()
.filter_map(|step| match step {
DispatchStep::Invoke(invocation) => {
assert_eq!(invocation.delivery, Delivery::Background);
Some(invocation.method)
}
_ => None,
})
.collect();
assert_eq!(
methods,
[
CallbackMethod::AsyncLoggingHook,
CallbackMethod::AsyncLogSuccessEvent
]
);
}
#[test]
fn failure_and_request_families_have_no_hook_pass() {
for (family, delivery, method) in [
(
CallbackFamily::SyncFailure,
Delivery::Inline,
CallbackMethod::LogFailureEvent,
),
(
CallbackFamily::AsyncFailure,
Delivery::Await,
CallbackMethod::AsyncLogFailureEvent,
),
] {
let mut cursor = start(family, ids(&[1, 2]), false, false);
let steps = drain(&mut cursor, &mut AllEligible);
assert_eq!(steps[0], DispatchStep::PrepareLogging);
assert!(matches!(steps[1], DispatchStep::MarkLogged(_)));
assert_eq!(
&steps[2..],
&[
DispatchStep::Invoke(CallbackInvocation {
target: CallbackId(1),
method,
delivery
}),
DispatchStep::Invoke(CallbackInvocation {
target: CallbackId(2),
method,
delivery
}),
DispatchStep::Complete { aborted: false },
]
);
}
let mut cursor = start(CallbackFamily::RequestPreCall, ids(&[1]), false, false);
let steps = drain(&mut cursor, &mut AllEligible);
assert_eq!(
steps,
vec![
DispatchStep::Invoke(CallbackInvocation {
target: CallbackId(1),
method: CallbackMethod::LogPreApiCall,
delivery: Delivery::Inline,
}),
DispatchStep::Complete { aborted: false },
]
);
}
#[test]
fn already_logged_marker_skips_the_whole_terminal_family_but_not_request_families() {
let mut cursor = start(CallbackFamily::AsyncSuccess, ids(&[1]), true, false);
assert_eq!(
cursor.next(&mut AllEligible),
DispatchStep::Complete { aborted: false }
);
let mut cursor = start(CallbackFamily::RequestPostCall, ids(&[1]), true, false);
assert!(matches!(
cursor.next(&mut AllEligible),
DispatchStep::Invoke(_)
));
}
#[test]
fn streaming_skips_the_marker_write_but_still_dispatches() {
let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1]), false, true);
let steps = drain(&mut cursor, &mut AllEligible);
assert!(
!steps
.iter()
.any(|step| matches!(step, DispatchStep::MarkLogged(_)))
);
assert_eq!(
steps
.iter()
.filter(|step| matches!(step, DispatchStep::Invoke(_)))
.count(),
2
);
}
#[test]
fn ineligible_targets_are_skipped_per_method_without_affecting_others() {
let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1, 2]), false, false);
let mut facts = Gate(|target, method| {
!(target == CallbackId(1) && method == CallbackMethod::LoggingHook)
&& !(target == CallbackId(2) && method == CallbackMethod::LogSuccessEvent)
});
let invoked: Vec<_> = drain(&mut cursor, &mut facts)
.into_iter()
.filter_map(|step| match step {
DispatchStep::Invoke(invocation) => Some((invocation.target.0, invocation.method)),
_ => None,
})
.collect();
assert_eq!(
invoked,
[
(2, CallbackMethod::LoggingHook),
(1, CallbackMethod::LogSuccessEvent)
]
);
}
#[test]
fn contained_failures_continue_and_propagating_failures_abort() {
let mut cursor = start(CallbackFamily::SyncFailure, ids(&[1, 2]), false, false);
assert_eq!(cursor.next(&mut AllEligible), DispatchStep::PrepareLogging);
assert!(matches!(
cursor.next(&mut AllEligible),
DispatchStep::MarkLogged(_)
));
assert!(matches!(
cursor.next(&mut AllEligible),
DispatchStep::Invoke(_)
));
cursor.accept(InvocationOutcome::Failed);
assert!(matches!(
cursor.next(&mut AllEligible),
DispatchStep::Invoke(CallbackInvocation {
target: CallbackId(2),
..
})
));
let mut cursor = start(
CallbackFamily::DeploymentPreCall,
ids(&[1, 2]),
false,
false,
);
assert!(matches!(
cursor.next(&mut AllEligible),
DispatchStep::Invoke(_)
));
cursor.accept(InvocationOutcome::Failed);
assert_eq!(
cursor.next(&mut AllEligible),
DispatchStep::Complete { aborted: true }
);
}
#[test]
fn sync_sdk_success_selects_one_worker_dispatch() {
let plan = plan_success(&SuccessFacts {
asynchronous: false,
internal: true,
fallbacks: true,
deferred: true,
sync_target_kinds: vec![],
});
assert_eq!(
plan,
[Dispatch {
family: CallbackFamily::SyncSuccess,
delivery: Delivery::Worker,
gate: ReleaseGate::Immediate,
}]
);
}
#[test]
fn async_sdk_success_enqueues_background_then_worker_only_for_external_sync_targets() {
let base = SuccessFacts {
asynchronous: true,
internal: false,
fallbacks: false,
deferred: false,
sync_target_kinds: vec![
CallbackKind::CustomLogger,
CallbackKind::Named { known: true },
],
};
assert_eq!(
plan_success(&base),
[Dispatch {
family: CallbackFamily::AsyncSuccess,
delivery: Delivery::Background,
gate: ReleaseGate::Immediate,
}]
);
let with_external = SuccessFacts {
sync_target_kinds: vec![
CallbackKind::CustomLogger,
CallbackKind::Callable { internal: false },
],
deferred: true,
..base.clone()
};
assert_eq!(
plan_success(&with_external),
[
Dispatch {
family: CallbackFamily::AsyncSuccess,
delivery: Delivery::Background,
gate: ReleaseGate::Deferred,
},
Dispatch {
family: CallbackFamily::SyncSuccess,
delivery: Delivery::Worker,
gate: ReleaseGate::Immediate,
},
]
);
let internal_or_fallback = SuccessFacts {
internal: true,
sync_target_kinds: vec![CallbackKind::Named { known: false }],
..base
};
assert_eq!(
plan_success(&internal_or_fallback),
[Dispatch {
family: CallbackFamily::SyncSuccess,
delivery: Delivery::Worker,
gate: ReleaseGate::Immediate,
}]
);
}
#[test]
fn opaque_and_internal_targets_never_trigger_the_worker_for_async_calls() {
let plan = plan_success(&SuccessFacts {
asynchronous: true,
internal: false,
fallbacks: true,
deferred: false,
sync_target_kinds: vec![
CallbackKind::Opaque,
CallbackKind::Callable { internal: true },
],
});
assert!(plan.is_empty());
}
#[test]
fn failure_families_follow_the_phase_and_skip_internal_async_calls() {
assert_eq!(
plan_failure(HostPhase::Failure, false, true),
Some(Dispatch::immediate(
CallbackFamily::SyncFailure,
Delivery::Inline
))
);
assert_eq!(
plan_failure(HostPhase::AsyncFailure, true, false),
Some(Dispatch::immediate(
CallbackFamily::AsyncFailure,
Delivery::Await
))
);
assert_eq!(plan_failure(HostPhase::Failure, true, true), None);
assert_eq!(plan_failure(HostPhase::Success, false, false), None);
}
#[test]
fn delivery_is_selected_by_the_plan_and_carried_on_every_invocation() {
let mut cursor = DispatchCursor::start(
Dispatch::immediate(CallbackFamily::SyncFailure, Delivery::Worker),
ids(&[1]),
CursorFacts {
already_logged: false,
stream: false,
sync_request: true,
},
);
let deliveries: Vec<_> = drain(&mut cursor, &mut AllEligible)
.into_iter()
.filter_map(|step| match step {
DispatchStep::Invoke(invocation) => Some(invocation.delivery),
_ => None,
})
.collect();
assert_eq!(deliveries, [Delivery::Worker]);
assert_eq!(
plan_request(CallbackFamily::RequestPreCall).delivery,
Delivery::Inline
);
assert_eq!(
plan_request(CallbackFamily::DeploymentPreCall).delivery,
Delivery::Await
);
}
#[test]
fn sync_leaf_methods_skip_object_targets_on_async_requests() {
let asynchronous = DispatchCursor::start(
Dispatch::immediate(CallbackFamily::SyncSuccess, Delivery::Worker),
ids(&[1]),
CursorFacts {
already_logged: false,
stream: false,
sync_request: false,
},
);
for kind in [
CallbackKind::CustomLogger,
CallbackKind::Callable { internal: false },
] {
assert!(!asynchronous.object_target_eligible(CallbackMethod::LogSuccessEvent, kind));
assert!(!asynchronous.object_target_eligible(CallbackMethod::LogFailureEvent, kind));
assert!(asynchronous.object_target_eligible(CallbackMethod::LoggingHook, kind));
assert!(
asynchronous.object_target_eligible(CallbackMethod::AsyncLogSuccessEvent, kind)
);
}
assert!(asynchronous.object_target_eligible(
CallbackMethod::LogSuccessEvent,
CallbackKind::Named { known: false }
));
let synchronous = start(CallbackFamily::SyncSuccess, ids(&[1]), false, false);
assert!(
synchronous.object_target_eligible(
CallbackMethod::LogSuccessEvent,
CallbackKind::CustomLogger
)
);
}
}

View file

@ -223,9 +223,11 @@ mod tests {
] {
assert_eq!(lifecycle.phase(), phase);
assert!(
lifecycle.accept(Err(HostFailure::Error(crate::ocr::Error::InvalidRequest(
"callback".into()
)))).is_none()
lifecycle
.accept(Err(HostFailure::Error(crate::ocr::Error::InvalidRequest(
"callback".into()
))))
.is_none()
);
}
assert_eq!(lifecycle.phase(), HostPhase::Complete);

View file

@ -1,9 +1,17 @@
use std::future::Future;
use std::time::{Instant, SystemTime, UNIX_EPOCH};
pub mod callbacks;
pub mod host;
pub mod registration;
pub mod types;
pub use callbacks::{
CallbackFamily, CallbackId, CallbackInvocation, CallbackKind, CallbackMethod, CursorFacts,
Delivery, Dispatch, DispatchCursor, DispatchFacts, DispatchStep, InvocationOutcome,
LoggedMarker, ReleaseGate, SuccessFacts, TargetErrorPolicy, object_target_eligible,
plan_failure, plan_request, plan_success,
};
pub use types::{
CallLifecycleContext, CallLifecyclePhase, CallLifecyclePhaseTiming, CallLifecycleRequest,
CallLifecycleTiming,

View file

@ -0,0 +1,374 @@
use std::collections::HashSet;
use super::callbacks::CallbackId;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Registry {
Input,
AsyncInput,
Success,
AsyncSuccess,
Failure,
AsyncFailure,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Registration {
Named { known: bool, async_only: bool },
Object { asynchronous: bool },
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Entry {
pub id: CallbackId,
pub registration: Registration,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Candidate {
pub resolved: Option<Entry>,
pub duplicate_type: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Default)]
pub struct RegistrationFacts {
pub candidates: Vec<Candidate>,
pub input: Vec<Entry>,
pub success: Vec<Entry>,
pub failure: Vec<Entry>,
pub async_success: Vec<CallbackId>,
pub async_failure: Vec<CallbackId>,
pub bootstrap_pending: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum NamedEvent {
Success,
Failure,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RegistryMutation {
Append(Registry, CallbackId),
Remove(Registry, CallbackId),
ExpandNamed(NamedEvent, CallbackId),
Bootstrap,
}
struct Planner {
input: Vec<Entry>,
success: Vec<Entry>,
failure: Vec<Entry>,
async_success: HashSet<CallbackId>,
async_failure: HashSet<CallbackId>,
mutations: Vec<RegistryMutation>,
}
impl Planner {
fn contains(&self, registry: Registry, id: CallbackId) -> bool {
match registry {
Registry::Input => self.input.iter().any(|entry| entry.id == id),
Registry::Success => self.success.iter().any(|entry| entry.id == id),
Registry::Failure => self.failure.iter().any(|entry| entry.id == id),
Registry::AsyncSuccess => self.async_success.contains(&id),
Registry::AsyncFailure => self.async_failure.contains(&id),
Registry::AsyncInput => false,
}
}
fn append(&mut self, registry: Registry, entry: Entry) {
if self.contains(registry, entry.id) {
return;
}
match registry {
Registry::Input => self.input.push(entry),
Registry::Success => self.success.push(entry),
Registry::Failure => self.failure.push(entry),
Registry::AsyncSuccess => {
self.async_success.insert(entry.id);
}
Registry::AsyncFailure => {
self.async_failure.insert(entry.id);
}
Registry::AsyncInput => {}
}
self.mutations
.push(RegistryMutation::Append(registry, entry.id));
}
fn record(&mut self, mutation: RegistryMutation) {
self.mutations.push(mutation);
}
}
fn is_asynchronous(registration: Registration) -> bool {
matches!(registration, Registration::Object { asynchronous: true })
}
pub fn plan_registration(facts: &RegistrationFacts) -> Vec<RegistryMutation> {
let mut planner = Planner {
input: facts.input.clone(),
success: facts.success.clone(),
failure: facts.failure.clone(),
async_success: facts.async_success.iter().copied().collect(),
async_failure: facts.async_failure.iter().copied().collect(),
mutations: Vec::new(),
};
for candidate in &facts.candidates {
let Some(entry) = candidate.resolved else {
continue;
};
if candidate.duplicate_type {
continue;
}
planner.append(Registry::Input, entry);
if !is_asynchronous(entry.registration) {
planner.append(Registry::Success, entry);
planner.append(Registry::Failure, entry);
}
planner.append(Registry::AsyncSuccess, entry);
planner.append(Registry::AsyncFailure, entry);
}
if facts.bootstrap_pending
&& !(planner.input.is_empty() && planner.success.is_empty() && planner.failure.is_empty())
{
planner.record(RegistryMutation::Bootstrap);
}
let input = planner.input.clone();
for entry in input
.iter()
.filter(|entry| is_asynchronous(entry.registration))
{
planner.record(RegistryMutation::Append(Registry::AsyncInput, entry.id));
planner.record(RegistryMutation::Remove(Registry::Input, entry.id));
}
let success = planner.success.clone();
for entry in &success {
match entry.registration {
Registration::Object { asynchronous: true }
| Registration::Named {
async_only: true, ..
} => {
planner.append(Registry::AsyncSuccess, *entry);
planner.record(RegistryMutation::Remove(Registry::Success, entry.id));
}
Registration::Named { known: true, .. } => {
planner.record(RegistryMutation::ExpandNamed(NamedEvent::Success, entry.id));
}
_ => {}
}
}
let failure = planner.failure.clone();
for entry in &failure {
match entry.registration {
Registration::Object { asynchronous: true } => {
planner.append(Registry::AsyncFailure, *entry);
planner.record(RegistryMutation::Remove(Registry::Failure, entry.id));
}
Registration::Named { known: true, .. } => {
planner.record(RegistryMutation::ExpandNamed(NamedEvent::Failure, entry.id));
}
_ => {}
}
}
planner.mutations
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DynamicSuccessSlot {
Sync,
Async,
}
pub fn classify_dynamic_success(entry: Entry, named_async: bool) -> DynamicSuccessSlot {
match entry.registration {
Registration::Object { asynchronous: true } => DynamicSuccessSlot::Async,
Registration::Named { .. } if named_async => DynamicSuccessSlot::Async,
_ => DynamicSuccessSlot::Sync,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn object(id: u64, asynchronous: bool) -> Entry {
Entry {
id: CallbackId(id),
registration: Registration::Object { asynchronous },
}
}
fn named(id: u64, known: bool, async_only: bool) -> Entry {
Entry {
id: CallbackId(id),
registration: Registration::Named { known, async_only },
}
}
fn candidate(entry: Entry) -> Candidate {
Candidate {
resolved: Some(entry),
duplicate_type: false,
}
}
#[test]
fn sync_callback_in_callbacks_registers_in_every_list_once() {
let facts = RegistrationFacts {
candidates: vec![candidate(object(1, false)), candidate(object(1, false))],
..RegistrationFacts::default()
};
assert_eq!(
plan_registration(&facts),
[
RegistryMutation::Append(Registry::Input, CallbackId(1)),
RegistryMutation::Append(Registry::Success, CallbackId(1)),
RegistryMutation::Append(Registry::Failure, CallbackId(1)),
RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(1)),
RegistryMutation::Append(Registry::AsyncFailure, CallbackId(1)),
]
);
}
#[test]
fn async_callable_in_callbacks_skips_sync_lists_and_moves_out_of_input() {
let facts = RegistrationFacts {
candidates: vec![candidate(object(2, true))],
..RegistrationFacts::default()
};
assert_eq!(
plan_registration(&facts),
[
RegistryMutation::Append(Registry::Input, CallbackId(2)),
RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(2)),
RegistryMutation::Append(Registry::AsyncFailure, CallbackId(2)),
RegistryMutation::Append(Registry::AsyncInput, CallbackId(2)),
RegistryMutation::Remove(Registry::Input, CallbackId(2)),
]
);
}
#[test]
fn unresolved_and_duplicate_type_named_candidates_are_skipped() {
let facts = RegistrationFacts {
candidates: vec![
Candidate {
resolved: None,
duplicate_type: false,
},
Candidate {
resolved: Some(object(3, false)),
duplicate_type: true,
},
],
..RegistrationFacts::default()
};
assert!(plan_registration(&facts).is_empty());
}
#[test]
fn already_registered_callbacks_are_not_appended_again() {
let facts = RegistrationFacts {
candidates: vec![candidate(object(1, false))],
input: vec![object(1, false)],
success: vec![object(1, false)],
failure: vec![object(1, false)],
async_success: vec![CallbackId(1)],
async_failure: vec![CallbackId(1)],
..RegistrationFacts::default()
};
assert!(plan_registration(&facts).is_empty());
}
#[test]
fn bootstrap_runs_once_when_any_public_list_is_populated() {
let empty = RegistrationFacts {
bootstrap_pending: true,
..RegistrationFacts::default()
};
assert!(plan_registration(&empty).is_empty());
let populated = RegistrationFacts {
bootstrap_pending: true,
candidates: vec![candidate(object(1, false))],
..RegistrationFacts::default()
};
assert!(plan_registration(&populated).contains(&RegistryMutation::Bootstrap));
let already = RegistrationFacts {
bootstrap_pending: false,
success: vec![object(1, false)],
..RegistrationFacts::default()
};
assert!(!plan_registration(&already).contains(&RegistryMutation::Bootstrap));
}
#[test]
fn success_safety_net_moves_async_and_async_only_names_and_expands_known_names() {
let facts = RegistrationFacts {
success: vec![
object(1, true),
named(2, false, true),
named(3, true, false),
named(4, false, false),
object(5, false),
],
..RegistrationFacts::default()
};
assert_eq!(
plan_registration(&facts),
[
RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(1)),
RegistryMutation::Remove(Registry::Success, CallbackId(1)),
RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(2)),
RegistryMutation::Remove(Registry::Success, CallbackId(2)),
RegistryMutation::ExpandNamed(NamedEvent::Success, CallbackId(3)),
]
);
}
#[test]
fn failure_safety_net_ignores_async_only_names() {
let facts = RegistrationFacts {
failure: vec![
object(1, true),
named(2, false, true),
named(3, true, false),
],
async_failure: vec![CallbackId(1)],
..RegistrationFacts::default()
};
assert_eq!(
plan_registration(&facts),
[
RegistryMutation::Remove(Registry::Failure, CallbackId(1)),
RegistryMutation::ExpandNamed(NamedEvent::Failure, CallbackId(3)),
]
);
}
#[test]
fn dynamic_success_split_follows_async_callables_and_selected_names() {
assert_eq!(
classify_dynamic_success(object(1, true), false),
DynamicSuccessSlot::Async
);
assert_eq!(
classify_dynamic_success(object(1, false), false),
DynamicSuccessSlot::Sync
);
assert_eq!(
classify_dynamic_success(named(2, true, false), true),
DynamicSuccessSlot::Async
);
assert_eq!(
classify_dynamic_success(named(2, true, true), false),
DynamicSuccessSlot::Sync
);
}
}

View file

@ -816,7 +816,8 @@ mod tests {
"type":"document_url",
"document_url":"https://example.com/document.pdf"
}))
.unwrap().into();
.unwrap()
.into();
perform_ocr(request).await.unwrap();
server.await.unwrap();

View file

@ -361,10 +361,12 @@ mod tests {
}
}),
);
let request = request.with_document(serde_json::from_value(json!({
"type":"image_url","image_url":"https://example.com/original.png"
}))
.unwrap());
let request = request.with_document(
serde_json::from_value(json!({
"type":"image_url","image_url":"https://example.com/original.png"
}))
.unwrap(),
);
let request = crate::ocr::prepare::prepare_request(request);
let http = CohereParseConfig
.prepare_request(&request, &crate::ocr::test_support::ocr_client())
@ -503,10 +505,12 @@ mod tests {
"https://example.com",
json!({"output_format":null,"req_format":null}),
);
let request = request.with_document(serde_json::from_value(
let request = request.with_document(
serde_json::from_value(
json!({"type":"image_url","image_url":"https://example.com/a.png"}),
)
.unwrap());
.unwrap(),
);
assert_eq!(
request.response_format().unwrap(),
crate::ocr::types::OcrResponseFormat::Litellm

View file

@ -738,7 +738,8 @@ mod tests {
"result":{"chunks":[]}
}))])
.await;
let request = crate::ocr::test_support::with_source(wire_request(model, &base, options), source);
let request =
crate::ocr::test_support::with_source(wire_request(model, &base, options), source);
perform_ocr(request).await.unwrap();
server.await.unwrap();
@ -857,7 +858,9 @@ mod tests {
#[tokio::test]
async fn rejects_invalid_document_sources_before_network(#[case] source: &str) {
let request = crate::ocr::test_support::with_source(
wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})), source);
wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})),
source,
);
assert!(perform_ocr(request).await.is_err());
}
@ -913,7 +916,9 @@ mod tests {
let raw = json!({"job_id":"job-1","result":{"chunks":[]}});
let (base, seen, server) = mock_server(vec![MockResponse::json(raw)]).await;
let mut request = crate::ocr::test_support::with_source(
wire_request("reducto/parse-v3", &base, json!({})), "reducto://ready.pdf");
wire_request("reducto/parse-v3", &base, json!({})),
"reducto://ready.pdf",
);
request.transport.extra_headers = vec![("authorization".into(), "Bearer existing".into())];
let response = perform_ocr(request).await.unwrap();
@ -984,6 +989,7 @@ mod tests {
request: OcrDuringCallRequest,
) -> OcrHookFuture<'_, OcrDuringCallRequest> {
Box::pin(async move {
assert_eq!(request.optional_params["use_cache"], json!(true));
assert_eq!(
request.body["document_url"],
"data:application/pdf;base64,YWJj"
@ -1000,7 +1006,7 @@ mod tests {
async fn guardrail_rewrites_document_before_upload() {
let (base, seen, server) =
mock_server(vec![MockResponse::json(json!({"result":{"chunks":[]}}))]).await;
let mut request = wire_request("reducto/parse-v3", &base, json!({}));
let mut request = wire_request("reducto/parse-v3", &base, json!({"use_cache":true}));
request.hooks = Arc::new(RewriteDocument);
perform_ocr(request).await.unwrap();

View file

@ -338,8 +338,12 @@ mod tests {
options.clone(),
);
let vertex = wire_request("vertex_ai/mistral-ocr-maas", "https://vertex.test", options);
let direct = crate::ocr::prepare::prepare_request(crate::ocr::test_support::resolved_request(direct));
let vertex = crate::ocr::prepare::prepare_request(crate::ocr::test_support::resolved_request(vertex));
let direct = crate::ocr::prepare::prepare_request(
crate::ocr::test_support::resolved_request(direct),
);
let vertex = crate::ocr::prepare::prepare_request(
crate::ocr::test_support::resolved_request(vertex),
);
let direct_http = MistralOCRConfig
.prepare_request(&direct, &client)
.await

View file

@ -39,9 +39,10 @@ impl OcrClient {
) -> Result<LiteLLMOcrResponse, crate::ocr::Error> {
use super::{
NativeOutcome, OcrAdmission, OcrCall, OcrCallStep, OcrHookHost, OcrHost,
OcrHostOperation, OcrHostResult,
OcrHostOperation, OcrHostResult, OcrProjectedRequest,
};
let intercepts_requests = request.hooks.intercepts_requests();
let host = OcrHookHost::new(request.hooks.clone());
let mut request = Some(request);
let NativeOutcome::Completed(mut call) = OcrCall::admit(self.clone(), OcrAdmission::all())
@ -54,14 +55,15 @@ impl OcrClient {
loop {
match call.resume(result.take()).await? {
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().ok_or_else(|| {
result = Some(OcrHostResult::Request(Ok(OcrProjectedRequest {
request: Box::new(request.take().ok_or_else(|| {
crate::ocr::Error::InvalidRequest(
"OCR request was already projected".into(),
)
})?),
false,
))))
intercepts_requests,
host_token_provider: false,
})))
}
OcrCallStep::Host(operation) => result = Some(host.invoke(operation).await),
OcrCallStep::Complete(response) => return Ok(response),

View file

@ -15,7 +15,10 @@ pub(crate) fn read_path_document(
) -> Result<OcrDocument, super::Error> {
let mut bytes = Vec::new();
std::fs::File::open(path)
.and_then(|file| file.take(OCR_INLINE_MAX_BYTES as u64 + 1).read_to_end(&mut bytes))
.and_then(|file| {
file.take(OCR_INLINE_MAX_BYTES as u64 + 1)
.read_to_end(&mut bytes)
})
.map_err(|source| super::Error::FileRead {
path: path.to_owned(),
source: std::sync::Arc::new(source),
@ -201,9 +204,14 @@ mod tests {
#[test]
fn path_preparation_preserves_io_causes_and_enforces_the_inline_limit() {
let path = std::env::temp_dir().join(format!("ocr-document-{:032x}.png", rand::random::<u128>()));
let path =
std::env::temp_dir().join(format!("ocr-document-{:032x}.png", rand::random::<u128>()));
let error = read_path_document(&path, None).unwrap_err();
let super::super::Error::FileRead { path: failed_path, source } = error else {
let super::super::Error::FileRead {
path: failed_path,
source,
} = error
else {
panic!("missing path must produce a typed file error");
};
assert_eq!(failed_path, path);
@ -215,9 +223,16 @@ mod tests {
let image = read_path_document(&path, None).unwrap();
let overridden = read_path_document(&path, Some("application/pdf")).unwrap();
std::fs::remove_file(&path).unwrap();
assert!(matches!(oversized, Err(super::super::Error::InlineDocumentTooLarge)));
assert!(matches!(image, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2UgYnl0ZXM="));
assert!(matches!(overridden, OcrDocument::DocumentUrl { document_url, .. } if document_url == "data:application/pdf;base64,aW1hZ2UgYnl0ZXM="));
assert!(matches!(
oversized,
Err(super::super::Error::InlineDocumentTooLarge)
));
assert!(
matches!(image, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2UgYnl0ZXM=")
);
assert!(
matches!(overridden, OcrDocument::DocumentUrl { document_url, .. } if document_url == "data:application/pdf;base64,aW1hZ2UgYnl0ZXM=")
);
}
fn document(source: &str) -> OcrDocument {

View file

@ -8,6 +8,8 @@ pub enum Error {
},
#[error("File is empty or could not be read")]
EmptyFile,
#[error("Host OCR document read failed")]
HostDocumentRead,
#[error("Failed to read OCR file {}: {source}", path.display())]
FileRead {
path: std::path::PathBuf,

View file

@ -2,7 +2,7 @@ use std::sync::Arc;
use super::OcrClient;
use super::hooks::{OcrHooks, OcrLifecycleHooks, OcrPostCallRequest};
use super::types::{ResolvedOcrRequest, LiteLLMOcrResponse, PreparedOcrRequest};
use super::types::{LiteLLMOcrResponse, PreparedOcrRequest, ResolvedOcrRequest};
use crate::call_lifecycle::{CallLifecycle, CallLifecycleContext};
use crate::llms::base_llm::ocr::transformation::OcrResponseContext;

View file

@ -22,6 +22,7 @@ pub struct OcrPreCallRequest {
pub struct OcrDuringCallRequest {
pub model: String,
pub custom_llm_provider: String,
pub optional_params: Value,
pub api_key: Option<String>,
pub url: String,
pub headers: Vec<(String, String)>,

View file

@ -1,6 +1,7 @@
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::{mpsc, oneshot};
@ -9,8 +10,8 @@ use super::hooks::{
OcrDuringCallRequest, OcrHookFuture, OcrHooks, OcrLogFuture, OcrPostCallRequest,
OcrPreCallRequest,
};
use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient, OcrDocumentInput, OcrFileContent};
use super::types::ResolvedOcrRequest;
use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient, OcrDocumentInput, OcrFileContent};
use crate::call_lifecycle::host::{
HostCall, HostCallFuture, HostCallStep, HostFailure, HostLifecycle, HostPhase,
};
@ -82,8 +83,14 @@ impl OcrHostOperation {
}
}
pub struct OcrProjectedRequest {
pub request: Box<LiteLLMOcrRequest>,
pub intercepts_requests: bool,
pub host_token_provider: bool,
}
pub enum OcrHostResult {
Request(Result<(Box<LiteLLMOcrRequest>, bool), super::Error>),
Request(Result<OcrProjectedRequest, super::Error>),
Document(Result<OcrFileContent, super::Error>),
Lifecycle(Result<(), HostFailure<super::Error>>),
AzureAdToken(Result<ResolvedCredential, litellm_auth::Error>),
@ -160,9 +167,10 @@ impl OcrCall {
Some(OcrHostResult::Request(result)) if self.projecting => {
self.projecting = false;
match result {
Ok((request, azure_ad_token_provider)) => {
self.execution.request = Some(*request);
self.execution.azure_ad_token_provider = azure_ad_token_provider;
Ok(projected) => {
self.execution.set_request(*projected.request);
self.execution.intercepts_requests = projected.intercepts_requests;
self.execution.host_token_provider = projected.host_token_provider;
}
Err(error) => self.accept(Err(HostFailure::Error(error))),
}
@ -268,6 +276,15 @@ impl OcrCall {
fn accept(&mut self, result: Result<(), HostFailure<super::Error>>) {
let cancelled = matches!(&result, Err(HostFailure::Cancelled(_)));
if let Some(error) = self.lifecycle.accept(result) {
if let Some((_, timing)) = self
.execution
.terminal
.lock()
.unwrap_or_else(|error| error.into_inner())
.as_mut()
{
timing.end_time = epoch_seconds();
}
if cancelled {
self.error = Some(error);
} else {
@ -333,11 +350,33 @@ struct OcrExecution {
pending_result: Option<oneshot::Sender<OcrHostResult>>,
execution: Option<tokio::task::JoinHandle<Result<LiteLLMOcrResponse, super::Error>>>,
completed: bool,
azure_ad_token_provider: bool,
intercepts_requests: bool,
host_token_provider: bool,
terminal: Arc<std::sync::Mutex<Option<(CallLifecycleContext, CallLifecycleTiming)>>>,
}
impl OcrExecution {
fn set_request(&mut self, mut request: LiteLLMOcrRequest) {
let call_id = request
.litellm_call_id
.clone()
.unwrap_or_else(|| format!("ocr-{:032x}", rand::random::<u128>()));
let context = CallLifecycleContext::new(
"ocr",
request.model.clone(),
request.provider_name(),
call_id.clone(),
);
request.litellm_call_id = Some(call_id);
let start = epoch_seconds();
*self
.terminal
.lock()
.unwrap_or_else(|error| error.into_inner()) =
Some((context, CallLifecycleTiming::new(start, start, Vec::new())));
self.request = Some(request);
}
fn new(client: OcrClient) -> Self {
let (operations_tx, operations_rx) = mpsc::unbounded_channel();
Self {
@ -350,7 +389,8 @@ impl OcrExecution {
pending_result: None,
execution: None,
completed: false,
azure_ad_token_provider: false,
intercepts_requests: false,
host_token_provider: false,
terminal: Arc::default(),
}
}
@ -366,11 +406,16 @@ impl OcrExecution {
}
let result = if self.reading {
let Some(OcrHostResult::Document(content)) = result else {
return Err(super::Error::InvalidRequest("OCR document read result is required".into()));
return Err(super::Error::InvalidRequest(
"OCR document read result is required".into(),
));
};
self.reading = false;
let content = content?;
let request = self.request.take().expect("pending document read has a request");
let request = self
.request
.take()
.expect("pending document read has a request");
let OcrDocumentInput::HostReader { mime_type } = &request.document else {
unreachable!("only host readers request document reads");
};
@ -428,7 +473,10 @@ impl OcrExecution {
async fn prepare(&mut self) -> Result<Option<OcrHostOperation>, super::Error> {
if self.preparation.is_none() {
let request = self.request.take().expect("admitted OCR call has a request");
let request = self
.request
.take()
.expect("admitted OCR call has a request");
if matches!(request.document, OcrDocumentInput::HostReader { .. }) {
self.request = Some(request);
self.reading = true;
@ -444,15 +492,25 @@ impl OcrExecution {
OcrDocumentInput::Path { path, mime_type } => {
super::document::read_path_document(path, mime_type.as_deref())?
}
OcrDocumentInput::Bytes { bytes, file_name, mime_type } => {
super::document::encode_file_document(bytes, file_name.as_deref(), mime_type.as_deref())?
}
OcrDocumentInput::Bytes {
bytes,
file_name,
mime_type,
} => super::document::encode_file_document(
bytes,
file_name.as_deref(),
mime_type.as_deref(),
)?,
_ => unreachable!("only native file inputs require preparation"),
};
Ok(request.with_document(document))
}));
}
let result = self.preparation.as_mut().expect("document preparation started").await;
let result = self
.preparation
.as_mut()
.expect("document preparation started")
.await;
self.preparation = None;
let request = result.map_err(|error| super::Error::DocumentTask(Arc::new(error)))??;
self.start(request);
@ -461,8 +519,7 @@ impl OcrExecution {
fn start(&mut self, mut request: ResolvedOcrRequest) {
let client = self.client.take().expect("admitted OCR call has a client");
let intercepts_requests = request.hooks.intercepts_requests();
if self.azure_ad_token_provider {
if self.host_token_provider {
request.azure_ad_token_provider = Some(TokenProviderHandle::new(Arc::new(
OcrAzureAdTokenProvider {
operations: self.operations_tx.clone(),
@ -471,7 +528,7 @@ impl OcrExecution {
}
request.hooks = Arc::new(ProtocolHooks {
operations: self.operations_tx.clone(),
intercepts_requests,
intercepts_requests: self.intercepts_requests,
terminal: self.terminal.clone(),
});
self.execution = Some(tokio::spawn(async move {
@ -517,6 +574,13 @@ struct ProtocolHooks {
terminal: Arc<std::sync::Mutex<Option<(CallLifecycleContext, CallLifecycleTiming)>>>,
}
fn epoch_seconds() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs_f64()
}
#[derive(Debug)]
struct OcrAzureAdTokenProvider {
operations: mpsc::UnboundedSender<PendingOperation>,
@ -743,36 +807,60 @@ mod tests {
use crate::ocr::test_support::{MockResponse, mock_server, perform_ocr, wire_request};
use crate::ocr::{
LiteLLMOcrRequest, NativeOutcome, NoopOcrHost, OcrAdmission, OcrCall, OcrCallStep,
OcrDecline, OcrDocument, OcrHost, OcrHostOperation, OcrHostResult,
OcrDecline, OcrDocument, OcrHost, OcrHostOperation, OcrHostResult, OcrProjectedRequest,
};
fn projected_request(request: LiteLLMOcrRequest) -> OcrHostResult {
OcrHostResult::Request(Ok(OcrProjectedRequest {
intercepts_requests: request.hooks.intercepts_requests(),
request: Box::new(request),
host_token_provider: false,
}))
}
#[tokio::test]
async fn sdk_paths_are_read_during_execution_and_hooks_observe_normalized_documents() {
struct CaptureDocument(Arc<Mutex<Vec<OcrDocument>>>);
impl OcrHooks for CaptureDocument {
fn intercepts_requests(&self) -> bool { true }
fn intercepts_requests(&self) -> bool {
true
}
fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> {
self.0.lock().unwrap().push(request.document.clone());
Box::pin(async { Ok(request) })
}
}
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
let path = std::env::temp_dir().join(format!("ocr-sdk-{:032x}.pdf", rand::random::<u128>()));
let path =
std::env::temp_dir().join(format!("ocr-sdk-{:032x}.pdf", rand::random::<u128>()));
let documents = Arc::new(Mutex::new(Vec::new()));
let request = LiteLLMOcrRequest::from_inputs(
"mistral/model".into(), path.clone(), None, Default::default(),
crate::ocr::OcrConnectionInputs { api_base: Some(base), api_key: Some("test-key".into()), ..Default::default() },
).unwrap().with_host_hooks(Arc::new(CaptureDocument(documents.clone())), None);
"mistral/model".into(),
path.clone(),
None,
Default::default(),
crate::ocr::OcrConnectionInputs {
api_base: Some(base),
api_key: Some("test-key".into()),
..Default::default()
},
)
.unwrap()
.with_host_hooks(Arc::new(CaptureDocument(documents.clone())), None);
std::fs::write(&path, b"sdk document").unwrap();
let result = perform_ocr(request).await;
std::fs::remove_file(&path).unwrap();
result.unwrap();
server.await.unwrap();
let expected = json!({"type":"document_url","document_url":"data:application/pdf;base64,c2RrIGRvY3VtZW50"});
assert_eq!(serde_json::to_value(&documents.lock().unwrap()[0]).unwrap(), expected);
assert_eq!(
serde_json::to_value(&documents.lock().unwrap()[0]).unwrap(),
expected
);
let requests = seen.lock().unwrap();
assert_eq!(requests.len(), 1);
let body: Value = serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap();
let body: Value =
serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap();
assert_eq!(body["document"], expected);
}
@ -781,9 +869,18 @@ mod tests {
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
let request = wire_request("mistral/model", &base, json!({}))
.with_document(crate::ocr::OcrDocumentInput::HostReader { mime_type: None })
.with_host_hooks(Arc::new(AdmissionSpy { effects: Arc::new(Mutex::new(0)) }), None);
.with_host_hooks(
Arc::new(AdmissionSpy {
effects: Arc::new(Mutex::new(0)),
}),
None,
);
let mut request = Some(request);
let NativeOutcome::Completed(mut call) = OcrCall::admit(crate::ocr::test_support::ocr_client(), OcrAdmission::all()) else { panic!("admission declined"); };
let NativeOutcome::Completed(mut call) =
OcrCall::admit(crate::ocr::test_support::ocr_client(), OcrAdmission::all())
else {
panic!("admission declined");
};
let mut result = None;
let mut reads = 0;
let mut pre_calls = 0;
@ -791,18 +888,21 @@ mod tests {
result = Some(match operation {
OcrHostOperation::ProjectRequest => {
assert_eq!(reads, 0);
OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false)))
projected_request(request.take().unwrap())
}
OcrHostOperation::ReadDocument => {
reads += 1;
OcrHostResult::Document(Ok(crate::ocr::OcrFileContent {
bytes: bytes::Bytes::from_static(b"image"), file_name: Some("scan.png".into()),
bytes: bytes::Bytes::from_static(b"image"),
file_name: Some("scan.png".into()),
}))
}
OcrHostOperation::PreCall(request) => {
assert_eq!(reads, 1);
pre_calls += 1;
assert!(matches!(&request.document, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2U="));
assert!(
matches!(&request.document, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2U=")
);
OcrHostResult::PreCall(Ok(request))
}
operation => NoopOcrHost.invoke(operation).await,
@ -812,6 +912,37 @@ mod tests {
assert_eq!((reads, pre_calls, seen.lock().unwrap().len()), (1, 1, 1));
}
#[tokio::test]
async fn sdk_file_failure_dispatches_the_typed_error_once() {
struct CaptureFailure(Arc<Mutex<Vec<crate::ocr::Error>>>);
impl OcrHooks for CaptureFailure {
fn failure<'a>(
&'a self,
_: &'a CallLifecycleContext,
error: &'a crate::ocr::Error,
_: &'a CallLifecycleTiming,
) -> OcrLogFuture<'a> {
self.0.lock().unwrap().push(error.clone());
Box::pin(async {})
}
}
let path =
std::env::temp_dir().join(format!("missing-ocr-{:032x}.pdf", rand::random::<u128>()));
let failures = Arc::new(Mutex::new(Vec::new()));
let request = wire_request("mistral/model", "http://127.0.0.1:1", json!({}))
.with_document(path.clone().into())
.with_host_hooks(Arc::new(CaptureFailure(failures.clone())), None);
let error = perform_ocr(request).await.unwrap_err();
assert!(
matches!(error, crate::ocr::Error::FileRead { path: actual, .. } if actual == path)
);
let failures = failures.lock().unwrap();
assert_eq!(failures.len(), 1);
assert!(
matches!(&failures[0], crate::ocr::Error::FileRead { source, .. } if source.kind() == std::io::ErrorKind::NotFound)
);
}
#[tokio::test]
async fn cancellation_acknowledges_blocking_preparation_completion() {
use std::sync::atomic::{AtomicBool, Ordering};
@ -831,7 +962,8 @@ mod tests {
std::future::poll_fn(|cx| {
assert!(stop.as_mut().poll(cx).is_pending());
std::task::Poll::Ready(())
}).await;
})
.await;
assert!(!finished.load(Ordering::SeqCst));
release_tx.send(()).unwrap();
stop.await;
@ -1029,6 +1161,8 @@ mod tests {
mut request: OcrDuringCallRequest,
) -> OcrHookFuture<'_, OcrDuringCallRequest> {
Box::pin(async move {
assert_eq!(request.optional_params["pages"], json!([0]));
assert_eq!(request.optional_params["extra_body"]["pages"], json!([2]));
assert_eq!(request.body["pages"], json!([2]));
assert_eq!(request.body.get("future"), Some(&Value::Null));
request.body.as_object_mut().unwrap().remove("future");
@ -1282,10 +1416,7 @@ mod tests {
result = Some(OcrHostResult::Lifecycle(Ok(())))
}
OcrHostOperation::ProjectRequest => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().unwrap()),
false,
))))
result = Some(projected_request(request.take().unwrap()))
}
OcrHostOperation::AcquireAzureAdToken => {
panic!("test request has no token provider")
@ -1307,7 +1438,9 @@ mod tests {
Ok(request)
}));
}
OcrHostOperation::PostCall(_) | OcrHostOperation::ReadDocument => panic!("transport should not be reached"),
OcrHostOperation::PostCall(_) | OcrHostOperation::ReadDocument => {
panic!("transport should not be reached")
}
},
Err(error) => break error,
Ok(OcrCallStep::Complete(_)) => panic!("failed call completed"),
@ -1340,10 +1473,7 @@ mod tests {
let error = loop {
match call.resume(result.take()).await {
Ok(OcrCallStep::Host(OcrHostOperation::ProjectRequest)) => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().unwrap()),
false,
))));
result = Some(projected_request(request.take().unwrap()));
}
Ok(OcrCallStep::Host(operation)) => {
if let OcrHostOperation::PostCall(request) = &operation {
@ -1407,7 +1537,7 @@ mod tests {
});
result = Some(match operation {
OcrHostOperation::ProjectRequest => {
OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false)))
projected_request(request.take().unwrap())
}
operation => host.invoke(operation).await,
});
@ -1467,7 +1597,9 @@ mod tests {
OcrHostResult::Lifecycle(Err(HostFailure::Error(selected.clone())))
}
OcrHostOperation::Failure { error, .. } => {
assert!(matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed"));
assert!(
matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed")
);
failures.push("sync");
OcrHostResult::Lifecycle(Err(HostFailure::Error(
crate::ocr::Error::InvalidRequest("failure callback failed".into()),
@ -1483,7 +1615,7 @@ mod tests {
panic!("finalization failure used provider/success dispatch")
}
OcrHostOperation::ProjectRequest => {
OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false)))
projected_request(request.take().unwrap())
}
operation => host.invoke(operation).await,
});
@ -1493,7 +1625,9 @@ mod tests {
}
};
server.await.unwrap();
assert!(matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed"));
assert!(
matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed")
);
assert_eq!(failures, ["sync", "async"]);
assert_eq!(seen.lock().unwrap().len(), 1);
}
@ -1520,10 +1654,7 @@ mod tests {
match call.resume(result.take()).await.unwrap() {
OcrCallStep::Host(OcrHostOperation::PreCall(_)) => break,
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().unwrap()),
false,
))))
result = Some(projected_request(request.take().unwrap()))
}
OcrCallStep::Host(operation) => result = Some(host.invoke(operation).await),
OcrCallStep::Complete(_) => panic!("provider executed before pre-call result"),
@ -1746,7 +1877,7 @@ mod tests {
_ = entered.notified() => break,
step = call.resume(result.take()) => {
result = Some(match step.unwrap() {
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false))),
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => projected_request(request.take().unwrap()),
OcrCallStep::Host(operation) => NoopOcrHost.invoke(operation).await,
OcrCallStep::Complete(_) => panic!("pending provider completed"),
});
@ -1773,7 +1904,9 @@ mod tests {
)
.await
.unwrap();
assert!(matches!(result, Err(crate::ocr::Error::InvalidRequest(message)) if message == "cancelled"));
assert!(
matches!(result, Err(crate::ocr::Error::InvalidRequest(message)) if message == "cancelled")
);
assert!(
dropped.load(Ordering::SeqCst),
"cancellation returned while provider captures were still alive"

View file

@ -18,12 +18,13 @@ pub use client::{OcrClient, ocr};
pub use document::{encode_file_document, mime_type_for_name, upload_mime_type};
pub use lifecycle::{
NativeOutcome, NativeResult, NoopOcrHost, OcrAdmission, OcrCall, OcrCallStep, OcrDecline,
OcrHookHost, OcrHost, OcrHostOperation, OcrHostResult,
OcrHookHost, OcrHost, OcrHostOperation, OcrHostResult, OcrProjectedRequest,
};
pub use provider_config::{get_api_key_env_var, get_health_check_document};
pub use types::{
LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrConnectionInputs, OcrCredentialInputs,
OcrDocument, OcrDocumentInput, OcrFileContent, OcrPage, OcrPageDimensions, OcrPageImage, OcrTransportConfig, OcrUsageInfo,
OcrDocument, OcrDocumentInput, OcrFileContent, OcrPage, OcrPageDimensions, OcrPageImage,
OcrTransportConfig, OcrUsageInfo,
};
#[cfg(test)]

View file

@ -3,7 +3,7 @@ use serde_json::Value;
use super::OcrClient;
use super::hooks::OcrDuringCallRequest;
use super::types::{ResolvedOcrRequest, OcrConnection, OcrDocument, PreparedOcrRequest};
use super::types::{OcrConnection, OcrDocument, PreparedOcrRequest, ResolvedOcrRequest};
pub(crate) async fn transform_request_body<B>(
client: &OcrClient,
@ -28,6 +28,7 @@ where
.during_call(OcrDuringCallRequest {
model: request.model.clone(),
custom_llm_provider: request.provider_name().into(),
optional_params: Value::Object(request.optional_params.clone().into()),
api_key: request.connection.api_key.clone(),
url: url.into(),
headers: headers.to_vec(),
@ -78,6 +79,7 @@ pub(crate) async fn guardrail_document(
.during_call(OcrDuringCallRequest {
model: request.model.clone(),
custom_llm_provider: request.provider_name().into(),
optional_params: Value::Object(request.optional_params.clone().into()),
api_key: request.connection.api_key.clone(),
url: url.into(),
headers: headers.to_vec(),

View file

@ -3,8 +3,8 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use serde::{Deserialize, Serialize};
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use serde_with::serde_as;
@ -93,7 +93,10 @@ impl From<OcrDocument> for OcrDocumentInput {
impl From<PathBuf> for OcrDocumentInput {
fn from(path: PathBuf) -> Self {
Self::Path { path, mime_type: None }
Self::Path {
path,
mime_type: None,
}
}
}
@ -324,7 +327,6 @@ impl LiteLLMOcrRequest {
config,
})
}
}
impl<D> LiteLLMOcrRequest<D> {
@ -383,7 +385,6 @@ impl<D> LiteLLMOcrRequest<D> {
..self
}
}
}
impl LiteLLMOcrRequest {

View file

@ -1,6 +1,6 @@
use litellm_python_interop::release_count;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::types::{PyDict, PyTuple};
#[pyfunction]
fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
@ -9,6 +9,27 @@ fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
Ok(stats.into_any().unbind())
}
#[pyfunction]
fn _debug_setup(
py: Python<'_>,
call_type: String,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
start: Py<PyAny>,
asynchronous: bool,
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
let leaked: &'static str = Box::leak(call_type.into_boxed_str());
let result = crate::lifecycle::debug_setup(
py,
leaked,
&args.unbind(),
&kwargs.unbind(),
&start,
asynchronous,
)?;
Ok(result)
}
#[cfg(feature = "panic-test")]
#[pyfunction]
fn _panic_for_test() {
@ -17,6 +38,7 @@ fn _panic_for_test() {
pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_function(wrap_pyfunction!(gil_stats, module)?)?;
module.add_function(wrap_pyfunction!(_debug_setup, module)?)?;
#[cfg(feature = "panic-test")]
module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?;
Ok(())

View file

@ -1,7 +1,6 @@
use pyo3::exceptions::PyBaseException;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyTuple};
use pyo3::types::PyDict;
#[derive(FromPyObject)]
pub(crate) struct PythonLogger(Py<PyAny>);
@ -19,34 +18,6 @@ impl PythonLogger {
visit.call(&self.0)
}
pub(crate) fn callbacks_needed(&self, py: Python<'_>, phase: &str) -> PyResult<bool> {
if !self
.object(py)
.getattr("_native_callback_fast_path")
.is_ok_and(|value| value.is_truthy().unwrap_or(false))
{
return Ok(true);
}
py.import("litellm.rust_bridge.lifecycle")?
.getattr("callbacks_needed")?
.call1((self.object(py), phase))?
.extract()
}
pub(super) fn success_bookkeeping(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
asynchronous: bool,
) -> PyResult<()> {
py.import("litellm.rust_bridge.lifecycle")?
.getattr("success_bookkeeping")?
.call1((self.object(py), response, start, end, asynchronous))?;
Ok(())
}
pub(super) fn defers_async_logging(&self, py: Python<'_>) -> bool {
self.object(py)
.getattr("_defer_async_logging")
@ -56,150 +27,10 @@ impl PythonLogger {
pub(super) fn defer_success(
&self,
py: Python<'_>,
pending: Py<super::PendingLogging>,
pending: Py<super::DeferredSuccess>,
) -> PyResult<()> {
self.object(py).setattr("_native_pending_logging", pending)
}
pub(super) fn sync_success_for_async_call(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
) -> PyResult<()> {
if !self.callbacks_needed(py, "sync_success_async")? {
return Ok(());
}
self.object(py).call_method1(
"handle_sync_success_callbacks_for_async_calls",
(response, start, end),
)?;
Ok(())
}
pub(super) fn failure(
&self,
py: Python<'_>,
error: &Py<PyBaseException>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
asynchronous: bool,
) -> PyResult<Option<Py<PyAny>>> {
if !self.callbacks_needed(
py,
if asynchronous {
"async_failure"
} else {
"sync_failure"
},
)? {
py.import("litellm.rust_bridge.lifecycle")?
.getattr("failure_bookkeeping")?
.call1((self.object(py), error, start, end, asynchronous))?;
return Ok(None);
}
let trace = py
.import("traceback")?
.getattr("format_exception")?
.call1((error,))?;
let trace = pyo3::types::PyString::new(py, "").call_method1("join", (trace,))?;
let value = self.object(py).call_method1(
if asynchronous {
"async_failure_handler"
} else {
"failure_handler"
},
(error, trace, start, end),
)?;
Ok(asynchronous.then(|| value.unbind()))
}
pub(super) fn restore_context(&self, py: Python<'_>) -> PyResult<()> {
py.import("litellm.utils")?
.getattr("_restore_correlation_context_if_supported")?
.call1((self.object(py),))?;
Ok(())
}
pub(super) fn submit_success(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
) -> PyResult<()> {
if !self.callbacks_needed(py, "sync_success")? {
return self.success_bookkeeping(py, response, start, end, false);
}
let context = py.import("contextvars")?.call_method0("copy_context")?;
py.import("litellm.litellm_core_utils.litellm_logging")?
.getattr("executor")?
.call_method1(
"submit",
(
context.getattr("run")?,
self.object(py).getattr("success_handler")?,
response,
start,
end,
),
)?;
Ok(())
}
pub(super) fn enqueue_success(
&self,
py: Python<'_>,
response: &Option<Py<PyAny>>,
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
) -> PyResult<()> {
if !self.callbacks_needed(py, "async_success")? {
return self.success_bookkeeping(py, response, start, end, true);
}
let context = py.import("contextvars")?.call_method0("copy_context")?;
let worker = py
.import("litellm.litellm_core_utils.logging_worker")?
.getattr("GLOBAL_LOGGING_WORKER")?
.getattr("ensure_initialized_and_enqueue")?;
let coroutine = self
.object(py)
.call_method1("async_success_handler", (response, start, end))?;
let enqueue = context.call_method1("run", (worker, &coroutine));
if enqueue.is_err()
&& let Err(error) = coroutine.call_method0("close")
{
error.write_unraisable(py, Some(&coroutine));
}
enqueue.map(|_| ())
}
}
pub(super) struct SetupResult<'py>(Bound<'py, PyAny>);
impl SetupResult<'_> {
pub(super) fn logger(&self) -> PyResult<PythonLogger> {
self.0.getattr("logger")?.extract()
}
pub(super) fn kwargs(&self) -> PyResult<Py<PyDict>> {
Ok(self.0.getattr("kwargs")?.extract()?)
}
}
pub(super) fn setup<'py>(
py: Python<'py>,
call_type: &str,
args: &Py<PyTuple>,
kwargs: &Py<PyDict>,
start: &Py<PyAny>,
asynchronous: bool,
) -> PyResult<SetupResult<'py>> {
py.import("litellm.rust_bridge.lifecycle")?
.getattr("setup")?
.call1((call_type, args, kwargs, start, asynchronous))
.map(SetupResult)
}
pub(super) fn finalize(
@ -210,9 +41,11 @@ pub(super) fn finalize(
start: &Py<PyAny>,
end: &Option<Py<PyAny>>,
) -> PyResult<()> {
py.import("litellm.rust_bridge.lifecycle")?
.getattr("finalize")?
.call1((response, logger.object(py), kwargs, start, end))?;
let model = kwargs.bind(py).get_item("model")?;
let model = model.filter(|value| value.is_instance_of::<pyo3::types::PyString>());
py.import("litellm.litellm_core_utils.llm_response_utils.response_metadata")?
.getattr("update_response_metadata")?
.call1((response, logger.object(py), model, kwargs, start, end))?;
Ok(())
}
@ -222,170 +55,3 @@ pub(super) fn is_internal_call(py: Python<'_>) -> PyResult<bool> {
.call_method0("get")?
.extract()
}
pub(super) struct DeploymentHooks;
impl DeploymentHooks {
pub(super) fn needed(py: Python<'_>) -> PyResult<bool> {
py.import("litellm.rust_bridge.lifecycle")?
.getattr("deployment_callbacks_needed")?
.call0()?
.extract()
}
pub(super) fn before_call(
py: Python<'_>,
kwargs: &Py<PyDict>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
py.import("litellm.utils")?
.getattr("async_pre_call_deployment_hook")?
.call1((kwargs, call_type))
.map(Bound::unbind)
}
pub(super) fn after_success(
py: Python<'_>,
kwargs: &Py<PyDict>,
response: &Option<Py<PyAny>>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
py.import("litellm.utils")?
.getattr("async_post_call_success_deployment_hook")?
.call1((kwargs, response, call_type))
.map(Bound::unbind)
}
pub(super) fn after_failure(
py: Python<'_>,
kwargs: &Py<PyDict>,
error: &Py<PyBaseException>,
call_type: &str,
) -> PyResult<Py<PyAny>> {
py.import("litellm.utils")?
.getattr("async_post_call_failure_deployment_hook")?
.call1((kwargs, error, call_type))
.map(Bound::unbind)
}
}
#[cfg(test)]
mod tests {
use super::*;
use pyo3::exceptions::PyTypeError;
#[test]
fn setup_fields_are_checked_in_order_without_eager_logger_method_reads() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
pyo3::ffi::c_str!(
r#"
reads = []
class Logger:
def __getattribute__(self, name):
reads.append(name)
raise AssertionError('logger methods must remain lazy')
logger = Logger()
class Setup:
@property
def logger(self):
reads.append('logger')
return logger
@property
def kwargs(self):
reads.append('kwargs')
return []
result = Setup()
"#
),
Some(&locals),
Some(&locals),
)
.unwrap();
let result = SetupResult(locals.get_item("result").unwrap().unwrap());
let logger = result.logger().unwrap();
assert!(
logger
.object(py)
.is(locals.get_item("logger").unwrap().unwrap())
);
assert_eq!(
locals
.get_item("reads")
.unwrap()
.unwrap()
.extract::<Vec<String>>()
.unwrap(),
["logger"]
);
assert!(
result
.kwargs()
.unwrap_err()
.is_instance_of::<PyTypeError>(py)
);
assert_eq!(
locals
.get_item("reads")
.unwrap()
.unwrap()
.extract::<Vec<String>>()
.unwrap(),
["logger", "kwargs"]
);
});
}
#[test]
fn logger_resolves_each_callback_at_invocation_and_preserves_arguments() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
pyo3::ffi::c_str!(
r#"
calls = []
response, start, end = object(), object(), object()
class Logger:
@property
def handle_sync_success_callbacks_for_async_calls(self):
generation = len(calls)
def callback(*args):
assert args == (response, start, end)
calls.append(generation)
return callback
logger = Logger()
"#
),
Some(&locals),
Some(&locals),
)
.unwrap();
let logger: PythonLogger = locals
.get_item("logger")
.unwrap()
.unwrap()
.extract()
.unwrap();
let response = Some(locals.get_item("response").unwrap().unwrap().unbind());
let start = locals.get_item("start").unwrap().unwrap().unbind();
let end = Some(locals.get_item("end").unwrap().unwrap().unbind());
for _ in 0..2 {
logger
.sync_success_for_async_call(py, &response, &start, &end)
.unwrap();
}
assert_eq!(
locals
.get_item("calls")
.unwrap()
.unwrap()
.extract::<Vec<usize>>()
.unwrap(),
[0, 1]
);
});
}
}

View file

@ -0,0 +1,168 @@
use litellm_core::call_lifecycle::{
CallbackFamily, Delivery, ReleaseGate, SuccessFacts, plan_success,
};
use pyo3::prelude::*;
use super::bindings::PythonLogger;
use super::{PythonCallState, dispatch, missing_state};
pub(super) fn dispatch_success(
py: Python<'_>,
state: &PythonCallState,
logger: &PythonLogger,
) -> PyResult<()> {
let facts = SuccessFacts {
asynchronous: state.asynchronous,
internal: state.internal,
fallbacks: !state
.kwargs
.bind(py)
.get_item("fallbacks")?
.is_none_or(|value| value.is_none()),
deferred: logger.defers_async_logging(py),
sync_target_kinds: sync_kinds(py, logger)?,
};
let leaves = dispatch::leaves(py)?;
let object = logger.object(py);
for selected in plan_success(&facts) {
match (selected.family, selected.delivery, selected.gate) {
(CallbackFamily::SyncSuccess, Delivery::Worker, _) => {
let handler = object.getattr("success_handler")?;
let bound = py.import("functools")?.getattr("partial")?.call1((
handler,
&state.response,
&state.start,
&state.end,
))?;
leaves.getattr("submit_worker")?.call1((bound,))?;
}
(CallbackFamily::AsyncSuccess, Delivery::Background, ReleaseGate::Immediate) => {
let coroutine = object.call_method1(
"async_success_handler",
(&state.response, &state.start, &state.end),
)?;
let enqueue = leaves.getattr("enqueue_background")?.call1((&coroutine,));
if enqueue.is_err()
&& let Err(error) = coroutine.call_method0("close")
{
error.write_unraisable(py, Some(&coroutine));
}
enqueue?;
}
(CallbackFamily::AsyncSuccess, Delivery::Background, ReleaseGate::Deferred) => {
let pending = Py::new(
py,
SuppliedDeferred {
logger: Some(logger.clone_ref(py)),
response: state.response.as_ref().map(|value| value.clone_ref(py)),
start: state.start.clone_ref(py),
end: state.end.as_ref().map(|value| value.clone_ref(py)),
},
)?;
object.setattr("_native_pending_logging", pending)?;
}
_ => return Err(missing_state()),
}
}
Ok(())
}
fn sync_kinds(
py: Python<'_>,
logger: &PythonLogger,
) -> PyResult<Vec<litellm_core::call_lifecycle::CallbackKind>> {
let (targets, ids) = dispatch::family_targets(py, logger, CallbackFamily::SyncSuccess)?;
Ok(targets.kinds(&ids))
}
pub(super) fn dispatch_failure(
py: Python<'_>,
state: &PythonCallState,
family: CallbackFamily,
) -> PyResult<Option<Py<PyAny>>> {
let logger = state.logger()?.object(py);
let error = state.error.as_ref().ok_or_else(missing_state)?;
let trace = py
.import("traceback")?
.getattr("format_exception")?
.call1((error,))?;
let trace = pyo3::types::PyString::new(py, "").call_method1("join", (trace,))?;
match family {
CallbackFamily::SyncFailure => {
logger.call_method1("failure_handler", (error, trace, &state.start, &state.end))?;
Ok(None)
}
CallbackFamily::AsyncFailure => Ok(Some(
logger
.call_method1(
"async_failure_handler",
(error, trace, &state.start, &state.end),
)?
.unbind(),
)),
_ => Err(missing_state()),
}
}
#[pyclass]
struct SuppliedDeferred {
logger: Option<PythonLogger>,
response: Option<Py<PyAny>>,
start: Py<PyAny>,
end: Option<Py<PyAny>>,
}
#[pymethods]
impl SuppliedDeferred {
fn __call__(slf: &Bound<'_, Self>, py: Python<'_>) -> PyResult<()> {
let logger = slf.borrow_mut().logger.take();
let Some(logger) = logger else {
return Ok(());
};
let (response, start, end) = {
let this = slf.borrow();
(
this.response.as_ref().map(|value| value.clone_ref(py)),
this.start.clone_ref(py),
this.end.as_ref().map(|value| value.clone_ref(py)),
)
};
let object = logger.object(py);
let coroutine = object.call_method1("async_success_handler", (response, start, end))?;
let enqueue = dispatch::leaves(py)?
.getattr("enqueue_background")?
.call1((&coroutine,));
if enqueue.is_err()
&& let Err(error) = coroutine.call_method0("close")
{
error.write_unraisable(py, Some(&coroutine));
}
match enqueue {
Err(error) if error.is_instance_of::<pyo3::exceptions::PyException>(py) => {
error.write_unraisable(py, Some(object));
Ok(())
}
result => result.map(|_| ()),
}
}
fn __traverse__(&self, visit: pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> {
if let Some(logger) = &self.logger {
logger.traverse(&visit)?;
}
visit.call(&self.response)?;
visit.call(&self.start)?;
visit.call(&self.end)
}
fn close(slf: &Bound<'_, Self>) {
let mut this = slf.borrow_mut();
this.logger = None;
this.response = None;
this.end = None;
}
fn __clear__(slf: &Bound<'_, Self>) {
Self::close(slf);
}
}

File diff suppressed because it is too large Load diff

View file

@ -1,41 +1,40 @@
use std::sync::Arc;
use std::task::Poll;
use futures_util::future::{AbortHandle, Abortable};
#[cfg(test)]
use litellm_core::call_lifecycle::host::HostCallFuture;
use litellm_core::call_lifecycle::host::{
HostCall as NativeCall, HostCallStep as NativeCallStep, HostFailure, HostPhase, HostStep,
};
use pyo3::exceptions::{PyBaseException, PyException, PyRuntimeError};
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyTuple};
use futures_util::future::{AbortHandle, Abortable};
use tokio::sync::Mutex;
use crate::execution::{poll_async_value, run_async_value, run_sync_value};
use litellm_core::call_lifecycle::host::{
HostCall as NativeCall, HostCallStep as NativeCallStep, HostFailure, HostPhase, HostStep,
};
use litellm_core::call_lifecycle::{
CallbackFamily, Delivery, Dispatch, ReleaseGate, SuccessFacts, plan_failure, plan_success,
};
mod arguments;
mod bindings;
mod compat;
mod dispatch;
mod handle;
mod preparation;
mod setup;
use crate::execution::{poll_async_value, run_async_value, run_sync_value};
pub(crate) use arguments::{BoundArguments, Signature};
use bindings::DeploymentHooks;
pub(crate) use bindings::PythonLogger;
use handle::{Execution, ExecutionBody, ExecutionStep};
pub(crate) enum OperationClass {
Phase(HostPhase),
Route,
}
pub(crate) trait PythonRoute: Send + Sync {
type Call: NativeCall + 'static;
fn state(&self) -> &PythonCallState;
fn state_mut(&mut self) -> &mut PythonCallState;
fn classify(operation: &<Self::Call as NativeCall>::Operation) -> OperationClass;
fn phase(operation: &<Self::Call as NativeCall>::Operation) -> Option<HostPhase>;
fn lifecycle_result() -> <Self::Call as NativeCall>::Result;
fn map_error(error: <Self::Call as NativeCall>::Error) -> PyErr;
fn host_error(message: String) -> <Self::Call as NativeCall>::Error;
@ -98,6 +97,18 @@ pub(crate) fn run_call<R: PythonRoute + 'static>(
}
}
pub(crate) fn debug_setup(
py: Python<'_>,
call_type: &'static str,
args: &Py<PyTuple>,
kwargs: &Py<PyDict>,
start: &Py<PyAny>,
asynchronous: bool,
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
let result = setup::setup(py, call_type, args, kwargs, start, asynchronous)?;
Ok((result.logger.object(py).clone().unbind(), result.kwargs))
}
pub(crate) fn missing_state() -> PyErr {
pyo3::exceptions::PyRuntimeError::new_err("missing native call state")
}
@ -167,12 +178,12 @@ impl<R: PythonRoute> PythonLifecycle<R> {
HostFailure::Cancelled(native)
};
let state = self.route.state_mut();
if state.error.is_none() || (cancelled && phase != Some(HostPhase::DeploymentFailure)) {
state.retain_error(py, error);
}
if state.end.is_none() {
state.end = now(py).ok();
}
state.retain_first_error(
py,
error,
cancelled && phase != Some(HostPhase::DeploymentFailure),
);
let _ = state.finish(py);
failure
}
@ -215,10 +226,7 @@ impl<R: PythonRoute> PythonLifecycle<R> {
}
HostStep::Ready(NativeCallStep::Host(operation)) => operation,
};
let phase = match R::classify(&operation) {
OperationClass::Phase(phase) => Some(phase),
OperationClass::Route => None,
};
let phase = R::phase(&operation);
let result = match phase {
Some(phase) => match self.route.state_mut().invoke(py, phase) {
Ok(HostStep::Suspend(awaitable)) => {
@ -295,6 +303,7 @@ pub(crate) struct PythonCallState {
pub error: Option<Py<PyBaseException>>,
pub asynchronous: bool,
pub internal: bool,
pub supplied: bool,
pub call_type: &'static str,
}
@ -314,42 +323,34 @@ impl PythonCallState {
match phase {
HostPhase::Setup => self.setup(py)?,
HostPhase::DeploymentPreCall => {
if !DeploymentHooks::needed(py)? {
return Ok(HostStep::Ready(self.kwargs.clone_ref(py).into_any()));
}
return Ok(HostStep::Suspend(DeploymentHooks::before_call(
let copied = self.kwargs.bind(py).copy()?.into_any().unbind();
return Ok(HostStep::Suspend(self.deployment(
py,
&self.kwargs,
self.call_type,
CallbackFamily::DeploymentPreCall,
copied,
)?));
}
HostPhase::Prepare => self.prepare(py)?,
HostPhase::DeploymentPostCall => {
if !DeploymentHooks::needed(py)? {
return self
.response
.as_ref()
.map(|value| HostStep::Ready(value.clone_ref(py)))
.ok_or_else(missing_state);
}
return Ok(HostStep::Suspend(DeploymentHooks::after_success(
let response = self
.response
.as_ref()
.map(|value| value.clone_ref(py))
.unwrap_or_else(|| py.None());
return Ok(HostStep::Suspend(self.deployment(
py,
&self.kwargs,
&self.response,
self.call_type,
CallbackFamily::DeploymentPostCall,
response,
)?));
}
HostPhase::Finalize => self.finalize(py)?,
HostPhase::Success => self.dispatch_success(py)?,
HostPhase::DeploymentFailure => {
if let Some(error) = &self.error
&& DeploymentHooks::needed(py)?
{
return Ok(HostStep::Suspend(DeploymentHooks::after_failure(
if self.error.is_some() {
return Ok(HostStep::Suspend(self.deployment(
py,
&self.kwargs,
error,
self.call_type,
CallbackFamily::DeploymentFailure,
py.None(),
)?));
}
}
@ -368,6 +369,24 @@ impl PythonCallState {
Ok(HostStep::Ready(py.None()))
}
fn deployment(
&self,
py: Python<'_>,
family: CallbackFamily,
current: Py<PyAny>,
) -> PyResult<Py<PyAny>> {
let body = dispatch::DeploymentBody::start(
py,
self.logger()?,
family,
self.call_type,
&self.kwargs,
current,
self.error.as_ref(),
)?;
dispatch::deployment_coroutine(py, body)
}
fn accept(&mut self, py: Python<'_>, phase: HostPhase, value: Py<PyAny>) -> PyResult<()> {
match phase {
HostPhase::DeploymentPreCall => {
@ -396,6 +415,7 @@ impl PythonCallState {
error: None,
asynchronous,
internal: false,
supplied: false,
call_type,
})
}
@ -409,7 +429,7 @@ impl PythonCallState {
pub fn setup(&mut self, py: Python<'_>) -> PyResult<()> {
self.start = now(py)?;
self.internal = bindings::is_internal_call(py)?;
let result = bindings::setup(
let result = setup::setup(
py,
self.call_type,
&self.args,
@ -417,8 +437,9 @@ impl PythonCallState {
&self.start,
self.asynchronous,
)?;
self.logger = Some(result.logger()?);
self.kwargs = result.kwargs()?;
self.logger = Some(result.logger);
self.kwargs = result.kwargs;
self.supplied = result.supplied;
Ok(())
}
@ -438,6 +459,16 @@ impl PythonCallState {
)
}
pub fn dispatch_request(&self, py: Python<'_>, family: CallbackFamily) -> PyResult<()> {
dispatch::dispatch_request(
py,
dispatch::RequestJob {
logger: self.logger()?,
family,
},
)
}
pub fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
match self.try_dispatch_success(py) {
Err(error) if error.is_instance_of::<PyException>(py) => {
@ -448,51 +479,76 @@ impl PythonCallState {
}
}
fn job(&self, py: Python<'_>, selected: Dispatch) -> PyResult<dispatch::Job> {
let logger = self.logger()?;
let (targets, ids) = dispatch::family_targets(py, logger, selected.family)?;
let sync_request = dispatch::leaves(py)?
.getattr("is_sync_request")?
.call1((logger.object(py),))?
.extract()?;
Ok(dispatch::Job {
logger: logger.clone_ref(py),
targets,
ids,
dispatch: selected,
sync_request,
response: self.response.as_ref().map(|value| value.clone_ref(py)),
error: self.error.as_ref().map(|value| value.clone_ref(py)),
start: self.start.clone_ref(py),
end: self
.end
.as_ref()
.map(|value| value.clone_ref(py))
.unwrap_or_else(|| py.None()),
stream: false,
})
}
fn try_dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
let logger = self.logger()?;
let pending = || PendingSuccess {
logger: logger.clone_ref(py),
response: self.response.as_ref().map(|value| value.clone_ref(py)),
start: self.start.clone_ref(py),
end: self.end.as_ref().map(|value| value.clone_ref(py)),
if self.supplied {
return compat::dispatch_success(py, self, logger);
}
let (sync_targets, sync_ids) =
dispatch::family_targets(py, logger, CallbackFamily::SyncSuccess)?;
let facts = SuccessFacts {
asynchronous: self.asynchronous,
internal: self.internal,
fallbacks: !self
.kwargs
.bind(py)
.get_item("fallbacks")?
.is_none_or(|value| value.is_none()),
deferred: logger.defers_async_logging(py),
sync_target_kinds: sync_targets.kinds(&sync_ids),
};
if !self.asynchronous {
if !logger.callbacks_needed(py, "sync_success")? {
return logger.success_bookkeeping(
py,
&self.response,
&self.start,
&self.end,
false,
);
}
pending().sync(py)
} else {
if !self.internal
&& self
.kwargs
.bind(py)
.get_item("fallbacks")?
.is_none_or(|value| value.is_none())
{
if !logger.callbacks_needed(py, "async_success")? {
logger.success_bookkeeping(py, &self.response, &self.start, &self.end, true)?;
} else if logger.defers_async_logging(py) {
for selected in plan_success(&facts) {
let runner = dispatch::Runner::start(py, self.job(py, selected)?)?;
match (selected.delivery, selected.gate) {
(Delivery::Worker, _) => {
let job = Py::new(py, dispatch::WorkerJob::new(runner))?;
dispatch::leaves(py)?
.getattr("submit_worker")?
.call1((job,))?;
}
(Delivery::Background, ReleaseGate::Immediate) => {
DeferredSuccess::release(py, runner)?;
}
(Delivery::Background, ReleaseGate::Deferred) => {
logger.defer_success(
py,
Py::new(
py,
PendingLogging {
pending: Some(pending()),
DeferredSuccess {
runner: Some(runner),
},
)?,
)?;
} else {
pending().asynchronous(py)?;
}
(Delivery::Inline | Delivery::Await, _) => return Err(missing_state()),
}
logger.sync_success_for_async_call(py, &self.response, &self.start, &self.end)
}
Ok(())
}
pub fn dispatch_failure(
@ -500,19 +556,39 @@ impl PythonCallState {
py: Python<'_>,
asynchronous: bool,
) -> PyResult<Option<Py<PyAny>>> {
if self.logger.is_none() || (self.asynchronous && self.internal) {
if self.logger.is_none() || self.error.is_none() {
return Ok(None);
}
let Some(error) = &self.error else {
let phase = if asynchronous {
HostPhase::AsyncFailure
} else {
HostPhase::Failure
};
let Some(selected) = plan_failure(phase, self.asynchronous, self.internal) else {
return Ok(None);
};
self.logger()?
.failure(py, error, &self.start, &self.end, asynchronous)
if self.supplied {
return compat::dispatch_failure(py, self, selected.family);
}
let mut runner = dispatch::Runner::start(py, self.job(py, selected)?)?;
match selected.delivery {
Delivery::Inline => match runner.resume(py, None)? {
dispatch::Step::Done => Ok(None),
dispatch::Step::Await(_) => Err(missing_state()),
},
Delivery::Await => Ok(Some(dispatch::coroutine(py, runner)?)),
Delivery::Worker | Delivery::Background => Err(missing_state()),
}
}
pub fn cleanup(&mut self, py: Python<'_>) {
if let Some(logger) = self.logger.take()
&& let Err(error) = logger.restore_context(py)
&& let Err(error) = dispatch::leaves(py).and_then(|leaves| {
leaves
.getattr("restore_correlation_context")?
.call1((logger.object(py),))
.map(|_| ())
})
{
error.write_unraisable(py, None);
}
@ -522,6 +598,19 @@ impl PythonCallState {
self.error = Some(error.into_value(py));
}
pub fn retain_first_error(&mut self, py: Python<'_>, error: PyErr, replace: bool) {
if self.error.is_none() || replace {
self.retain_error(py, error);
}
}
pub fn finish(&mut self, py: Python<'_>) -> PyResult<()> {
if self.end.is_none() {
self.end = Some(now(py)?);
}
Ok(())
}
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.args)?;
visit.call(&self.kwargs)?;
@ -535,74 +624,96 @@ impl PythonCallState {
}
}
struct PendingSuccess {
logger: PythonLogger,
response: Option<Py<PyAny>>,
start: Py<PyAny>,
end: Option<Py<PyAny>>,
}
impl PendingSuccess {
fn sync(&self, py: Python<'_>) -> PyResult<()> {
self.logger
.submit_success(py, &self.response, &self.start, &self.end)
}
fn asynchronous(&self, py: Python<'_>) -> PyResult<()> {
self.logger
.enqueue_success(py, &self.response, &self.start, &self.end)
}
}
#[pyclass]
struct PendingLogging {
pending: Option<PendingSuccess>,
struct DeferredSuccess {
runner: Option<dispatch::Runner>,
}
impl DeferredSuccess {
fn release(py: Python<'_>, runner: dispatch::Runner) -> PyResult<()> {
let coroutine = dispatch::coroutine(py, runner)?;
let enqueue = dispatch::leaves(py)?
.getattr("enqueue_background")?
.call1((&coroutine,));
if enqueue.is_err()
&& let Err(error) = coroutine.call_method0(py, "close")
{
error.write_unraisable(py, Some(coroutine.bind(py)));
}
enqueue.map(|_| ())
}
}
#[pymethods]
impl PendingLogging {
fn release(slf: &Bound<'_, Self>, py: Python<'_>, success: bool) -> PyResult<()> {
let pending = slf.borrow_mut().pending.take();
if let Some(pending) = pending
&& success
{
match pending.asynchronous(py) {
Err(error) if error.is_instance_of::<PyException>(py) => {
error.write_unraisable(py, Some(pending.logger.object(py)));
}
result => return result,
impl DeferredSuccess {
fn __call__(slf: &Bound<'_, Self>, py: Python<'_>) -> PyResult<()> {
let runner = slf.borrow_mut().runner.take();
let Some(runner) = runner else {
return Ok(());
};
let logger = runner.logger(py).clone().unbind();
match Self::release(py, runner) {
Err(error) if error.is_instance_of::<PyException>(py) => {
error.write_unraisable(py, Some(logger.bind(py)));
Ok(())
}
result => result,
}
Ok(())
}
fn __traverse__(&self, visit: pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> {
if let Some(pending) = &self.pending {
pending.logger.traverse(&visit)?;
visit.call(&pending.response)?;
visit.call(&pending.start)?;
visit.call(&pending.end)?;
match &self.runner {
Some(runner) => runner.traverse(&visit),
None => Ok(()),
}
Ok(())
}
fn close(slf: &Bound<'_, Self>) {
let runner = slf.borrow_mut().runner.take();
drop(runner);
}
fn __clear__(slf: &Bound<'_, Self>) {
let pending = slf.borrow_mut().pending.take();
drop(pending);
Self::close(slf);
}
}
#[cfg(test)]
mod tests {
use super::*;
use pyo3::types::PyDict;
use std::sync::Mutex;
use litellm_core::call_lifecycle::host::HostCallFuture;
static PYTHON_GLOBALS: Mutex<()> = Mutex::new(());
fn install_logging_worker(py: Python<'_>, worker: &Bound<'_, PyAny>) -> PyResult<()> {
py.import("litellm.litellm_core_utils.logging_worker")?
.setattr("GLOBAL_LOGGING_WORKER", worker)
fn load_lifecycle_module(py: Python<'_>) -> Bound<'_, PyModule> {
py.run(
pyo3::ffi::c_str!(
r#"
import sys
import types
for name in ("litellm", "litellm.rust_bridge"):
sys.modules.setdefault(name, types.ModuleType(name))
"#
),
None,
None,
)
.unwrap();
let source = std::ffi::CString::new(include_str!(
"../../../../../litellm/rust_bridge/lifecycle.py"
))
.unwrap();
PyModule::from_code(
py,
&source,
pyo3::ffi::c_str!("lifecycle.py"),
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
)
.unwrap()
}
struct RetainingHost {
@ -715,8 +826,8 @@ mod tests {
&mut self.0
}
fn classify(_: &()) -> OperationClass {
OperationClass::Route
fn phase(_: &()) -> Option<HostPhase> {
None
}
fn lifecycle_result() {}
@ -774,17 +885,7 @@ mod tests {
.unwrap_or_else(|error| error.into_inner());
Python::initialize();
Python::attach(|py| {
let source = std::ffi::CString::new(include_str!(
"../../../../../litellm/rust_bridge/lifecycle.py"
))
.unwrap();
PyModule::from_code(
py,
&source,
pyo3::ffi::c_str!("lifecycle.py"),
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
)
.unwrap();
load_lifecycle_module(py);
let route = SyntheticRoute(
PythonCallState::new(
py,
@ -820,17 +921,7 @@ mod tests {
Python::initialize();
Python::attach(|py| {
py.import("asyncio").unwrap();
let source = std::ffi::CString::new(include_str!(
"../../../../../litellm/rust_bridge/lifecycle.py"
))
.unwrap();
let module = PyModule::from_code(
py,
&source,
pyo3::ffi::c_str!("lifecycle.py"),
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
)
.unwrap();
let module = load_lifecycle_module(py);
let locals = PyDict::new(py);
locals
.set_item("drive", module.getattr("drive").unwrap())
@ -938,64 +1029,11 @@ assert reference() is None
error: None,
asynchronous,
internal: false,
supplied: false,
call_type: "test",
}
}
#[test]
fn success_dispatch_reports_ordinary_failures_without_replacing_response() {
let _guard = PYTHON_GLOBALS
.lock()
.unwrap_or_else(|error| error.into_inner());
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
pyo3::ffi::c_str!(
r#"
import sys
response = object()
failure = ValueError('terminal diagnostic')
diagnostics = []
old_hook = sys.unraisablehook
sys.unraisablehook = lambda event: diagnostics.append(event.exc_value)
class Logger:
def handle_sync_success_callbacks_for_async_calls(self, *args):
raise failure
logger = Logger()
"#
),
Some(&locals),
Some(&locals),
)
.unwrap();
let response = locals.get_item("response").unwrap().unwrap().unbind();
let mut lifecycle_state = state(
py,
locals.get_item("logger").unwrap().unwrap().unbind(),
response.clone_ref(py),
true,
);
lifecycle_state.internal = true;
lifecycle_state.dispatch_success(py).unwrap();
assert!(lifecycle_state.response.as_ref().unwrap().is(&response));
py.run(
pyo3::ffi::c_str!(
r#"
assert diagnostics == [failure]
sys.unraisablehook = old_hook
"#
),
Some(&locals),
Some(&locals),
)
.unwrap();
});
}
#[test]
fn retained_failure_preserves_exception_identity() {
Python::initialize();
@ -1011,139 +1049,6 @@ sys.unraisablehook = old_hook
});
}
#[test]
fn deferred_release_uses_release_context_and_allows_reentry_once() {
let _guard = PYTHON_GLOBALS
.lock()
.unwrap_or_else(|error| error.into_inner());
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
pyo3::ffi::c_str!(
r#"
import sys
import types
from contextvars import ContextVar
litellm = types.ModuleType('litellm')
core_utils = types.ModuleType('litellm.litellm_core_utils')
logging_worker = types.ModuleType('litellm.litellm_core_utils.logging_worker')
litellm.litellm_core_utils = core_utils
core_utils.logging_worker = logging_worker
sys.modules['litellm'] = litellm
sys.modules['litellm.litellm_core_utils'] = core_utils
sys.modules['litellm.litellm_core_utils.logging_worker'] = logging_worker
marker = ContextVar('marker', default='unset')
observed = []
class Coroutine:
def close(self):
observed.append('closed')
class Worker:
def ensure_initialized_and_enqueue(self, coroutine):
observed.append(marker.get())
pending.release(True)
coroutine.close()
class Logger:
def async_success_handler(self, *args):
observed.append('created')
return Coroutine()
worker = Worker()
logger = Logger()
"#
),
Some(&locals),
Some(&locals),
)
.unwrap();
install_logging_worker(py, &locals.get_item("worker").unwrap().unwrap()).unwrap();
let pending = Py::new(
py,
PendingLogging {
pending: Some(PendingSuccess {
logger: locals
.get_item("logger")
.unwrap()
.unwrap()
.extract()
.unwrap(),
response: Some(py.None()),
start: py.None(),
end: Some(py.None()),
}),
},
)
.unwrap();
locals.set_item("pending", &pending).unwrap();
py.run(
pyo3::ffi::c_str!(
r#"
marker.set('release')
pending.release(True)
pending.release(True)
assert observed == ['created', 'release', 'closed']
"#
),
Some(&locals),
Some(&locals),
)
.unwrap();
});
}
#[test]
fn deferred_logging_collects_cycles_through_typed_logger() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
pyo3::ffi::c_str!("class Logger: pass\nlogger = Logger()"),
Some(&locals),
Some(&locals),
)
.unwrap();
let pending = Py::new(
py,
PendingLogging {
pending: Some(PendingSuccess {
logger: locals
.get_item("logger")
.unwrap()
.unwrap()
.extract()
.unwrap(),
response: None,
start: py.None(),
end: None,
}),
},
)
.unwrap();
locals.set_item("pending", pending).unwrap();
py.run(
pyo3::ffi::c_str!(
r#"
import gc
import weakref
logger.pending = pending
reference = weakref.ref(logger)
del logger, pending
gc.collect()
assert reference() is None
"#
),
Some(&locals),
Some(&locals),
)
.unwrap();
});
}
#[test]
fn coroutine_collects_cycles_retained_by_bridge_host() {
Python::initialize();

View file

@ -0,0 +1,337 @@
use litellm_core::call_lifecycle::CallbackId;
use litellm_core::call_lifecycle::registration::{
Candidate, DynamicSuccessSlot, Entry, NamedEvent, Registration, RegistrationFacts, Registry,
RegistryMutation, classify_dynamic_success, plan_registration,
};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyList, PyString, PyTuple};
use super::bindings::PythonLogger;
const SETUP_MODULE: &str = "litellm.rust_bridge.setup";
struct Targets<'py> {
objects: Vec<Bound<'py, PyAny>>,
}
impl<'py> Targets<'py> {
fn new() -> Self {
Self {
objects: Vec::new(),
}
}
fn intern(&mut self, object: &Bound<'py, PyAny>) -> CallbackId {
if let Some(index) = self
.objects
.iter()
.position(|existing| existing.is(object) || existing.eq(object).unwrap_or(false))
{
return CallbackId(index as u64);
}
self.objects.push(object.clone());
CallbackId((self.objects.len() - 1) as u64)
}
fn get(&self, id: CallbackId) -> PyResult<&Bound<'py, PyAny>> {
self.objects
.get(id.0 as usize)
.ok_or_else(super::missing_state)
}
}
fn registration<'py>(
setup: &Bound<'py, PyModule>,
object: &Bound<'py, PyAny>,
) -> PyResult<Registration> {
if let Ok(name) = object.cast::<PyString>() {
let name = name.to_str()?;
let known = setup.getattr("is_known_name")?.call1((name,))?.extract()?;
return Ok(Registration::Named {
known,
async_only: matches!(name, "dynamodb" | "openmeter"),
});
}
let asynchronous = setup
.getattr("is_async_callable")?
.call1((object,))?
.extract()?;
Ok(Registration::Object { asynchronous })
}
fn entries<'py>(
setup: &Bound<'py, PyModule>,
targets: &mut Targets<'py>,
list: &Bound<'py, PyAny>,
) -> PyResult<Vec<Entry>> {
list.try_iter()?
.map(|object| {
let object = object?;
Ok(Entry {
id: targets.intern(&object),
registration: registration(setup, &object)?,
})
})
.collect()
}
fn registry_name(registry: Registry) -> &'static str {
match registry {
Registry::Input => "input",
Registry::AsyncInput => "async_input",
Registry::Success => "success",
Registry::AsyncSuccess => "async_success",
Registry::Failure => "failure",
Registry::AsyncFailure => "async_failure",
}
}
fn read_registry<'py>(setup: &Bound<'py, PyModule>, name: &str) -> PyResult<Bound<'py, PyAny>> {
setup.getattr("registry")?.call1((name,))
}
fn read_candidates<'py>(
setup: &Bound<'py, PyModule>,
targets: &mut Targets<'py>,
dynamic: Option<Bound<'py, PyAny>>,
) -> PyResult<Vec<Candidate>> {
let mut candidates = Vec::new();
let global = read_registry(setup, "callbacks")?;
let sources = std::iter::once(global).chain(dynamic);
for source in sources {
for object in source.try_iter()? {
let object = object?;
let candidate = if object.is_instance_of::<PyString>() {
let resolved = setup
.getattr("resolve_named_integration")?
.call1((&object,))?;
if resolved.is_none() {
Candidate {
resolved: None,
duplicate_type: false,
}
} else {
let duplicate_type = setup
.getattr("async_success_registry_has_type")?
.call1((&resolved,))?
.extract()?;
Candidate {
resolved: Some(Entry {
id: targets.intern(&resolved),
registration: registration(setup, &resolved)?,
}),
duplicate_type,
}
}
} else {
Candidate {
resolved: Some(Entry {
id: targets.intern(&object),
registration: registration(setup, &object)?,
}),
duplicate_type: false,
}
};
candidates.push(candidate);
}
}
Ok(candidates)
}
fn apply<'py>(
setup: &Bound<'py, PyModule>,
targets: &Targets<'py>,
mutations: &[RegistryMutation],
function_id: Option<&Bound<'py, PyAny>>,
) -> PyResult<()> {
for mutation in mutations {
match mutation {
RegistryMutation::Append(registry, id) => {
setup
.getattr("append_registry")?
.call1((registry_name(*registry), targets.get(*id)?))?;
}
RegistryMutation::Remove(registry, id) => {
setup
.getattr("remove_registry")?
.call1((registry_name(*registry), targets.get(*id)?))?;
}
RegistryMutation::ExpandNamed(event, id) => {
let event = match event {
NamedEvent::Success => "success",
NamedEvent::Failure => "failure",
};
setup
.getattr("expand_named")?
.call1((targets.get(*id)?, event))?;
}
RegistryMutation::Bootstrap => {
setup.getattr("bootstrap")?.call1((function_id,))?;
}
}
}
Ok(())
}
struct DynamicLists<'py> {
success: Option<Bound<'py, PyList>>,
async_success: Option<Bound<'py, PyList>>,
failure: Option<Bound<'py, PyList>>,
}
fn split_dynamic<'py>(
py: Python<'py>,
setup: &Bound<'py, PyModule>,
targets: &mut Targets<'py>,
kwargs: &Bound<'py, PyDict>,
) -> PyResult<DynamicLists<'py>> {
let success = match kwargs.get_item("success_callback")? {
Some(value) if value.is_instance_of::<PyList>() => {
let list = value.cast_into::<PyList>()?;
let sync = PyList::empty(py);
let asynchronous = PyList::empty(py);
for object in list.iter() {
let entry = Entry {
id: targets.intern(&object),
registration: registration(setup, &object)?,
};
let named_async = object
.cast::<PyString>()
.ok()
.and_then(|name| {
name.to_str()
.ok()
.map(|name| matches!(name, "dynamodb" | "s3"))
})
.unwrap_or(false);
match classify_dynamic_success(entry, named_async) {
DynamicSuccessSlot::Sync => sync.append(&object)?,
DynamicSuccessSlot::Async => asynchronous.append(&object)?,
}
}
kwargs.del_item("success_callback")?;
Some((sync, (!asynchronous.is_empty()).then_some(asynchronous)))
}
_ => None,
};
let failure = match kwargs.get_item("failure_callback")? {
Some(value) if value.is_instance_of::<PyList>() => {
kwargs.del_item("failure_callback")?;
Some(value.cast_into::<PyList>()?)
}
_ => None,
};
let (success, async_success) = match success {
Some((sync, asynchronous)) => (Some(sync), asynchronous),
None => (None, None),
};
Ok(DynamicLists {
success,
async_success,
failure,
})
}
pub(super) struct Setup {
pub logger: PythonLogger,
pub kwargs: Py<PyDict>,
pub supplied: bool,
}
pub(super) fn setup(
py: Python<'_>,
call_type: &str,
args: &Py<PyTuple>,
kwargs: &Py<PyDict>,
start: &Py<PyAny>,
asynchronous: bool,
) -> PyResult<Setup> {
let setup = py.import(SETUP_MODULE)?;
let kwargs = kwargs.bind(py).copy()?;
if !kwargs.contains("litellm_call_id")? {
let call_id = py.import("uuid")?.call_method0("uuid4")?.str()?;
kwargs.set_item("litellm_call_id", call_id)?;
}
if let Some(supplied) = kwargs.get_item("litellm_logging_obj")? {
let logging_class = py
.import("litellm.litellm_core_utils.litellm_logging")?
.getattr("Logging")?;
if supplied.is_instance(&logging_class)? {
return Ok(Setup {
logger: supplied.extract()?,
kwargs: kwargs.unbind(),
supplied: true,
});
}
}
setup.getattr("prepare_environment")?.call0()?;
let guardrails = setup.getattr("applied_guardrails")?.call1((&kwargs,))?;
let function_id = kwargs.get_item("id")?;
let mut targets = Targets::new();
let dynamic = match kwargs.get_item("callbacks")? {
Some(value) => {
kwargs.del_item("callbacks")?;
(!value.is_none()).then_some(value)
}
None => None,
};
let candidates = read_candidates(&setup, &mut targets, dynamic)?;
let facts = RegistrationFacts {
candidates,
input: entries(&setup, &mut targets, &read_registry(&setup, "input")?)?,
success: entries(&setup, &mut targets, &read_registry(&setup, "success")?)?,
failure: entries(&setup, &mut targets, &read_registry(&setup, "failure")?)?,
async_success: entries(
&setup,
&mut targets,
&read_registry(&setup, "async_success")?,
)?
.into_iter()
.map(|entry| entry.id)
.collect(),
async_failure: entries(
&setup,
&mut targets,
&read_registry(&setup, "async_failure")?,
)?
.into_iter()
.map(|entry| entry.id)
.collect(),
bootstrap_pending: setup.getattr("bootstrap_pending")?.call0()?.extract()?,
};
apply(
&setup,
&targets,
&plan_registration(&facts),
function_id.as_ref(),
)?;
let dynamic = split_dynamic(py, &setup, &mut targets, &kwargs)?;
setup.getattr("breadcrumb")?.call1((&kwargs,))?;
if let Some(logger_fn) = kwargs.get_item("logger_fn")? {
setup.getattr("logger_fn")?.call1((logger_fn,))?;
}
let model = match args.bind(py).get_item(0) {
Ok(model) => Some(model),
Err(_) => kwargs.get_item("model")?,
};
let build = setup.getattr("build_logging")?;
let build_kwargs = PyDict::new(py);
build_kwargs.set_item("call_type", call_type)?;
build_kwargs.set_item("model", model)?;
build_kwargs.set_item("kwargs", &kwargs)?;
build_kwargs.set_item("start_time", start)?;
build_kwargs.set_item("asynchronous", asynchronous)?;
build_kwargs.set_item("dynamic_success", dynamic.success)?;
build_kwargs.set_item("dynamic_async_success", dynamic.async_success)?;
build_kwargs.set_item("dynamic_failure", dynamic.failure)?;
build_kwargs.set_item("guardrails", guardrails)?;
let logger = build.call((), Some(&build_kwargs))?;
Ok(Setup {
logger: logger.extract()?,
kwargs: kwargs.unbind(),
supplied: false,
})
}

View file

@ -13,6 +13,11 @@ use litellm_python_interop::from_py_preserving_errors as from_py;
use crate::auth::{AZURE_AD_TOKEN_PROVIDER, PythonTokenProvider};
use crate::lifecycle::BoundArguments;
pub(crate) struct Projection<Native, Retained> {
pub native: Native,
pub retained: Retained,
}
/// Fields every lifecycle route reads from its bound `*args, **kwargs` before
/// asking core to build the typed request. Route-specific inputs (for example
/// the OCR `document`) are read separately by the route.

View file

@ -3,158 +3,94 @@ use pyo3::types::PyDict;
use serde_json::Value;
use litellm_core::ocr::LiteLLMOcrResponse;
use litellm_core::ocr::hooks::OcrPreCallRequest;
use litellm_core::ocr::hooks::OcrDuringCallRequest;
use litellm_python_interop::to_py_preserving_errors as to_py;
use crate::lifecycle::PythonLogger;
use litellm_core::call_lifecycle::CallbackFamily;
pub(super) struct OcrLoggingFields {
model: String,
custom_llm_provider: String,
optional_params: Value,
use super::host::PythonPayload;
use crate::lifecycle::{PythonCallState, PythonLogger};
pub(super) fn update_logging(
py: Python<'_>,
logger: &PythonLogger,
kwargs: &Py<PyDict>,
request: &OcrDuringCallRequest,
secret_fields: &[&str],
) -> PyResult<()> {
py.import("litellm.rust_bridge.ocr")?
.getattr("update_logging")?
.call1((
logger.object(py),
kwargs,
&request.model,
&request.custom_llm_provider,
to_py(py, &request.optional_params)?,
secret_fields,
&request.url,
))?;
Ok(())
}
impl From<&OcrPreCallRequest> for OcrLoggingFields {
fn from(request: &OcrPreCallRequest) -> Self {
Self {
model: request.model.clone(),
custom_llm_provider: request.custom_llm_provider.clone(),
optional_params: request.optional_params.clone(),
}
}
}
impl PythonLogger {
pub(super) fn update_ocr(
&self,
py: Python<'_>,
kwargs: &Py<PyDict>,
pre_call: &OcrLoggingFields,
secret_fields: &[&str],
url: &str,
) -> PyResult<()> {
let update = PyDict::new(py);
update.set_item("kwargs", redact(py, kwargs.bind(py), secret_fields)?)?;
update.set_item("model", &pre_call.model)?;
update.set_item(
"optional_params",
redact(
py,
&to_py(py, &pre_call.optional_params)?
.into_bound(py)
.cast_into::<PyDict>()?,
secret_fields,
)?,
)?;
let params = PyDict::new(py);
params.set_item(
"litellm_call_id",
kwargs.bind(py).get_item("litellm_call_id")?,
)?;
params.set_item("api_base", url)?;
for name in ["logger_fn", "litellm_request_debug"] {
if let Some(value) = kwargs.bind(py).get_item(name)? {
params.set_item(name, value)?;
}
}
for name in custom_pricing_fields(py)? {
if let Some(value) = kwargs.bind(py).get_item(&name)?
&& !value.is_none()
{
params.set_item(name, value)?;
}
}
update.set_item("litellm_params", params)?;
update.set_item("custom_llm_provider", &pre_call.custom_llm_provider)?;
self.object(py)
.call_method("update_from_kwargs", (), Some(&update))?;
Ok(())
}
pub(crate) fn pre_ocr(
&self,
py: Python<'_>,
api_key: Option<&str>,
body: &Bound<'_, PyDict>,
headers: &Bound<'_, PyDict>,
url: &str,
) -> PyResult<()> {
let additional = PyDict::new(py);
additional.set_item("complete_input_dict", body)?;
additional.set_item("headers", headers)?;
additional.set_item("api_base", url)?;
pub(super) fn pre_call(
py: Python<'_>,
state: &PythonCallState,
request: &OcrDuringCallRequest,
payload: &PythonPayload,
) -> PyResult<()> {
let logger = state.logger()?;
if state.supplied {
let additional_args = PyDict::new(py);
additional_args.set_item("complete_input_dict", &payload.body)?;
additional_args.set_item("headers", &payload.headers)?;
additional_args.set_item("api_base", &request.url)?;
let kwargs = PyDict::new(py);
kwargs.set_item("input", "OCR document processing")?;
kwargs.set_item("api_key", api_key)?;
kwargs.set_item("additional_args", &additional)?;
if self.callbacks_needed(py, "input")? {
self.object(py).call_method("pre_call", (), Some(&kwargs))?;
} else {
self.object(py)
.call_method("_pre_call", (), Some(&kwargs))?;
self.object(py).call_method0("record_api_call_start_time")?;
}
Ok(())
}
pub(crate) fn post_ocr(
&self,
py: Python<'_>,
original_response: &Value,
body: Option<&Py<PyDict>>,
headers: Option<&Py<PyDict>>,
) -> PyResult<()> {
let additional = PyDict::new(py);
additional.set_item("complete_input_dict", body)?;
additional.set_item("headers", headers)?;
if self.callbacks_needed(py, "input")? {
let kwargs = PyDict::new(py);
kwargs.set_item("original_response", to_py(py, original_response)?)?;
kwargs.set_item("additional_args", &additional)?;
self.object(py)
.call_method("post_call", (), Some(&kwargs))?;
} else {
let response = py
.import("json")?
.call_method1("dumps", (to_py(py, original_response)?,))?;
self.object(py).call_method1(
"record_post_call",
(response, py.None(), py.None(), additional),
)?;
}
Ok(())
kwargs.set_item("api_key", request.api_key.as_deref())?;
kwargs.set_item("additional_args", additional_args)?;
logger
.object(py)
.call_method("pre_call", (), Some(&kwargs))?;
return Ok(());
}
let kwargs = PyDict::new(py);
kwargs.set_item("api_key", request.api_key.as_deref())?;
kwargs.set_item("body", &payload.body)?;
kwargs.set_item("headers", &payload.headers)?;
kwargs.set_item("url", &request.url)?;
py.import("litellm.rust_bridge.leaves")?
.getattr("record_pre_call")?
.call((logger.object(py),), Some(&kwargs))?;
state.dispatch_request(py, CallbackFamily::RequestPreCall)
}
fn custom_pricing_fields(py: Python<'_>) -> PyResult<Vec<String>> {
py.import("litellm.types.utils")?
.getattr("CustomPricingLiteLLMParams")?
.getattr("model_fields")?
.cast_into::<PyDict>()?
.keys()
.iter()
.map(|name| name.extract::<String>())
.collect()
}
fn redact(
pub(super) fn post_call(
py: Python<'_>,
params: &Bound<'_, PyDict>,
secret_fields: &[&str],
) -> PyResult<Py<PyDict>> {
let redacted = PyDict::new(py);
for (name, value) in params {
let name = name.extract::<String>()?;
if name == "proxy_server_request" {
continue;
}
if secret_fields.contains(&name.as_str()) {
redacted.set_item(name, "****")?;
} else {
redacted.set_item(name, value)?;
}
state: &PythonCallState,
original_response: &Value,
payload: &PythonPayload,
) -> PyResult<()> {
let logger = state.logger()?;
if state.supplied {
let additional_args = PyDict::new(py);
additional_args.set_item("complete_input_dict", &payload.body)?;
additional_args.set_item("headers", &payload.headers)?;
let kwargs = PyDict::new(py);
kwargs.set_item("original_response", to_py(py, original_response)?)?;
kwargs.set_item("additional_args", additional_args)?;
logger
.object(py)
.call_method("post_call", (), Some(&kwargs))?;
return Ok(());
}
Ok(redacted.unbind())
let kwargs = PyDict::new(py);
kwargs.set_item("original_response", to_py(py, original_response)?)?;
kwargs.set_item("body", &payload.body)?;
kwargs.set_item("headers", &payload.headers)?;
py.import("litellm.rust_bridge.leaves")?
.getattr("record_post_call")?
.call((logger.object(py),), Some(&kwargs))?;
state.dispatch_request(py, CallbackFamily::RequestPostCall)
}
pub(super) fn response(py: Python<'_>, response: &LiteLLMOcrResponse) -> PyResult<Py<PyAny>> {

View file

@ -36,11 +36,11 @@ fn extract_bytes(value: &Bound<'_, PyAny>) -> PyResult<Bytes> {
return Ok(Bytes::from_owner(value.extract::<PyBackedBytes>()?));
}
// Bytes subclasses may retain GC edges that a native Bytes owner cannot traverse.
Ok(Bytes::copy_from_slice(value.extract::<PyBackedBytes>()?.as_ref()))
Ok(Bytes::copy_from_slice(
value.extract::<PyBackedBytes>()?.as_ref(),
))
}
pub(super) struct FileDocumentInput {
pub input: OcrDocumentInput,
pub reader: Option<PythonFileReader>,
@ -56,9 +56,11 @@ impl FromPyObject<'_, '_> for FileDocumentInput {
Err(error) if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py) => None,
Err(error) => return Err(error),
};
let missing = || PyValueError::new_err(
"document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes",
);
let missing = || {
PyValueError::new_err(
"document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes",
)
};
let file = document.get_item("file").map_err(|error| {
if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py) {
missing()
@ -93,7 +95,9 @@ impl FromPyObject<'_, '_> for FileDocumentInput {
reader: None,
});
}
let reader = file.getattr_opt("read")?.filter(|value| value.is_callable());
let reader = file
.getattr_opt("read")?
.filter(|value| value.is_callable());
let Some(reader) = reader else {
return Err(PyValueError::new_err(format!(
"Unsupported file input type: {}. Expected pathlib.Path, bytes, or a file-like object.",
@ -107,7 +111,10 @@ impl FromPyObject<'_, '_> for FileDocumentInput {
.transpose()?;
Ok(Self {
input: OcrDocumentInput::HostReader { mime_type },
reader: Some(PythonFileReader { reader: reader.unbind(), name }),
reader: Some(PythonFileReader {
reader: reader.unbind(),
name,
}),
})
}
}
@ -127,12 +134,21 @@ mod tests {
let error = document.extract::<FileDocumentInput>().err().unwrap();
assert!(error.is_instance_of::<PyValueError>(py));
}
for expression in [c"{'file': b'abc', 'mime_type': None}", c"{'file': b'abc', 'mime_type': 7}"] {
let error = py.eval(expression, None, None).unwrap().extract::<FileDocumentInput>().err().unwrap();
for expression in [
c"{'file': b'abc', 'mime_type': None}",
c"{'file': b'abc', 'mime_type': 7}",
] {
let error = py
.eval(expression, None, None)
.unwrap()
.extract::<FileDocumentInput>()
.err()
.unwrap();
assert!(error.is_instance_of::<PyTypeError>(py));
}
let locals = PyDict::new(py);
py.run(c"from pathlib import Path
py.run(
c"from pathlib import Path
failure = KeyError('reader failed')
class Reader:
def __init__(self):
@ -143,12 +159,31 @@ class Reader:
reader = Reader()
document = {'file': reader}
path_document = {'file': Path('/nonexistent/ocr-projection-test.pdf')}
", Some(&locals), Some(&locals)).unwrap();
",
Some(&locals),
Some(&locals),
)
.unwrap();
let document = locals.get_item("document").unwrap().unwrap();
let input: FileDocumentInput = document.extract().unwrap();
assert_eq!(locals.get_item("reader").unwrap().unwrap().getattr("reads").unwrap().extract::<usize>().unwrap(), 0);
assert_eq!(
locals
.get_item("reader")
.unwrap()
.unwrap()
.getattr("reads")
.unwrap()
.extract::<usize>()
.unwrap(),
0
);
assert!(input.reader.is_some());
let path: FileDocumentInput = locals.get_item("path_document").unwrap().unwrap().extract().unwrap();
let path: FileDocumentInput = locals
.get_item("path_document")
.unwrap()
.unwrap()
.extract()
.unwrap();
assert!(matches!(path.input, OcrDocumentInput::Path { .. }));
});
}

View file

@ -103,9 +103,13 @@ pub(super) fn public_exception(
std::io::ErrorKind::PermissionDenied => 13,
_ => 5,
});
return Ok(PyErr::from_value(py.import("builtins")?.getattr("OSError")?.call1((
errno, source.to_string(), path,
))?));
return Ok(PyErr::from_value(
py.import("builtins")?.getattr("OSError")?.call1((
errno,
source.to_string(),
path.into_pyobject(py)?.call_method0("__fspath__")?,
))?,
));
}
raise_public(py, classify(error), model, provider, None)
}

View file

@ -1,11 +1,11 @@
use std::sync::Arc;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::PyDict;
use litellm_auth::ResolvedCredential;
use litellm_core::call_lifecycle::host::HostPhase;
use litellm_core::ocr::hooks::{OcrDuringCallRequest, OcrPostCallRequest};
use litellm_core::ocr::{OcrCall, OcrHostOperation, OcrHostResult};
use litellm_core::ocr::{OcrCall, OcrHostOperation, OcrHostResult, OcrProjectedRequest};
use litellm_python_interop::{
from_py_preserving_errors as from_py, to_py_preserving_errors as to_py,
};
@ -13,75 +13,100 @@ use litellm_python_interop::{
use super::document::PythonFileReader;
use super::errors::to_pyerr as ocr_error_to_pyerr;
use super::project::project;
use super::{ASYNC_SIGNATURE, SIGNATURE, callbacks, errors};
use super::{callbacks, errors};
use crate::auth::PythonTokenProvider;
use crate::lifecycle::{OperationClass, PythonCallState, PythonRoute, missing_state, now};
use crate::lifecycle::{PythonCallState, PythonRoute, Signature, missing_state};
use crate::marshal::Projection;
pub(super) struct PythonOcrHost {
state: PythonCallState,
projected: Option<ProjectedOcrHost>,
signature: &'static Signature,
retained: Option<OcrRetained>,
}
struct ProjectedOcrHost {
model: String,
provider: &'static str,
secret_fields: Vec<&'static str>,
azure_ad_token_provider: Option<PythonTokenProvider>,
pre_call: Option<callbacks::OcrLoggingFields>,
reader: Option<PythonFileReader>,
reader_failed: bool,
body: Option<Py<PyDict>>,
headers: Option<Py<PyDict>>,
pub(super) struct OcrRetained {
pub model: String,
pub provider: &'static str,
pub secret_fields: Vec<&'static str>,
pub azure_ad_token_provider: Option<PythonTokenProvider>,
pub payload: Option<PythonPayload>,
pub reader: Option<PythonFileReader>,
}
pub(super) struct PythonPayload {
pub body: Py<PyDict>,
pub headers: Py<PyDict>,
}
impl PythonPayload {
fn from_request(py: Python<'_>, request: &OcrDuringCallRequest) -> PyResult<Self> {
let body = to_py(py, &request.body)?
.into_bound(py)
.cast_into::<PyDict>()?;
let headers = PyDict::new(py);
for (name, value) in &request.headers {
headers.set_item(name, value)?;
}
Ok(Self {
body: body.unbind(),
headers: headers.unbind(),
})
}
fn write_back(
&self,
py: Python<'_>,
mut request: OcrDuringCallRequest,
) -> PyResult<OcrDuringCallRequest> {
request.body = from_py(self.body.bind(py))?;
request.headers = self
.headers
.bind(py)
.iter()
.map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
.collect::<PyResult<Vec<_>>>()?;
Ok(request)
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.body)?;
visit.call(&self.headers)
}
}
impl PythonOcrHost {
pub(super) fn new(state: PythonCallState) -> Self {
pub(super) fn new(state: PythonCallState, signature: &'static Signature) -> Self {
Self {
state,
projected: None,
signature,
retained: None,
}
}
fn projected(&self) -> PyResult<&ProjectedOcrHost> {
self.projected.as_ref().ok_or_else(missing_state)
fn retained(&self) -> PyResult<&OcrRetained> {
self.retained.as_ref().ok_or_else(missing_state)
}
fn projected_mut(&mut self) -> PyResult<&mut ProjectedOcrHost> {
self.projected.as_mut().ok_or_else(missing_state)
fn retained_mut(&mut self) -> PyResult<&mut OcrRetained> {
self.retained.as_mut().ok_or_else(missing_state)
}
fn project(&mut self, py: Python<'_>) -> PyResult<OcrHostResult> {
let signature = if self.state.asynchronous {
&ASYNC_SIGNATURE
} else {
&SIGNATURE
};
let arguments = signature.bind(self.state.args.bind(py), self.state.kwargs.bind(py))?;
let projected = project(py, &arguments)?;
let has_token_provider = projected.azure_ad_token_provider.is_some();
self.projected = Some(ProjectedOcrHost {
model: projected.request.model.clone(),
provider: projected.request.provider_name(),
secret_fields: projected.secret_fields,
azure_ad_token_provider: projected.azure_ad_token_provider,
pre_call: None,
reader: projected.reader,
reader_failed: false,
body: None,
headers: None,
});
Ok(OcrHostResult::Request(Ok((
Box::new(
projected
.request
.with_host_hooks(Arc::new(BridgeOcrHooks), None),
),
has_token_provider,
))))
let arguments = self
.signature
.bind(self.state.args.bind(py), self.state.kwargs.bind(py))?;
let Projection { native, retained } = project(py, &arguments)?;
let host_token_provider = retained.azure_ad_token_provider.is_some();
self.retained = Some(retained);
Ok(OcrHostResult::Request(Ok(OcrProjectedRequest {
request: Box::new(native),
intercepts_requests: true,
host_token_provider,
})))
}
fn acquire_azure_ad_token(&self, py: Python<'_>) -> PyResult<ResolvedCredential> {
self.projected()?
self.retained()?
.azure_ad_token_provider
.as_ref()
.ok_or_else(missing_state)?
@ -91,34 +116,21 @@ impl PythonOcrHost {
fn during_call(
&mut self,
py: Python<'_>,
mut request: OcrDuringCallRequest,
request: OcrDuringCallRequest,
) -> PyResult<OcrDuringCallRequest> {
let projected = self.projected()?;
let pre_call = projected.pre_call.as_ref().ok_or_else(missing_state)?;
let retained = self.retained()?;
let logger = self.state.logger()?;
logger.update_ocr(
callbacks::update_logging(
py,
logger,
&self.state.kwargs,
pre_call,
&projected.secret_fields,
&request.url,
&request,
&retained.secret_fields,
)?;
let body = to_py(py, &request.body)?
.into_bound(py)
.cast_into::<PyDict>()?;
let headers = PyDict::new(py);
for (name, value) in &request.headers {
headers.set_item(name, value)?;
}
logger.pre_ocr(py, request.api_key.as_deref(), &body, &headers, &request.url)?;
request.body = from_py(&body)?;
request.headers = headers
.iter()
.map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
.collect::<PyResult<Vec<_>>>()?;
let projected = self.projected_mut()?;
projected.body = Some(body.unbind());
projected.headers = Some(headers.unbind());
let payload = PythonPayload::from_request(py, &request)?;
callbacks::pre_call(py, &self.state, &request, &payload)?;
let request = payload.write_back(py, request)?;
self.retained_mut()?.payload = Some(payload);
Ok(request)
}
@ -127,26 +139,23 @@ impl PythonOcrHost {
py: Python<'_>,
request: OcrPostCallRequest,
) -> PyResult<OcrPostCallRequest> {
let projected = self.projected()?;
self.state.logger()?.post_ocr(
py,
&request.original_response,
projected.body.as_ref(),
projected.headers.as_ref(),
)?;
let payload = self
.retained()?
.payload
.as_ref()
.ok_or_else(missing_state)?;
callbacks::post_call(py, &self.state, &request.original_response, payload)?;
Ok(request)
}
fn map_failure(&mut self, py: Python<'_>, error: litellm_core::ocr::Error) -> PyResult<()> {
if self.state.end.is_none() {
self.state.end = Some(now(py)?);
}
let (model, provider) = match &self.projected {
Some(projected) => (projected.model.as_str(), projected.provider),
self.state.finish(py)?;
let (model, provider) = match &self.retained {
Some(retained) => (retained.model.as_str(), retained.provider),
None => ("", ""),
};
let mapped = match self.state.error.take() {
Some(host_error) if self.projected.as_ref().is_some_and(|host| host.reader_failed) => {
Some(host_error) if matches!(error, litellm_core::ocr::Error::HostDocumentRead) => {
PyErr::from_value(host_error.into_bound(py).into_any())
}
Some(host_error) => errors::public_host_exception(py, &host_error, model, provider)?,
@ -168,10 +177,8 @@ impl PythonRoute for PythonOcrHost {
&mut self.state
}
fn classify(operation: &OcrHostOperation) -> OperationClass {
operation
.phase()
.map_or(OperationClass::Route, OperationClass::Phase)
fn phase(operation: &OcrHostOperation) -> Option<HostPhase> {
operation.phase()
}
fn lifecycle_result() -> OcrHostResult {
@ -190,18 +197,24 @@ impl PythonRoute for PythonOcrHost {
Ok(match operation {
OcrHostOperation::ProjectRequest => self.project(py)?,
OcrHostOperation::ReadDocument => {
let reader = self.projected_mut()?.reader.take().ok_or_else(missing_state)?;
let result = reader.read(py);
self.projected_mut()?.reader_failed = result.is_err();
OcrHostResult::Document(Ok(result?))
let reader = self
.retained_mut()?
.reader
.take()
.ok_or_else(missing_state)?;
match reader.read(py) {
Ok(content) => OcrHostResult::Document(Ok(content)),
Err(error) if error.is_instance_of::<pyo3::exceptions::PyException>(py) => {
self.state.retain_first_error(py, error, false);
OcrHostResult::Document(Err(litellm_core::ocr::Error::HostDocumentRead))
}
Err(error) => return Err(error),
}
}
OcrHostOperation::AcquireAzureAdToken => {
OcrHostResult::AzureAdToken(Ok(self.acquire_azure_ad_token(py)?))
}
OcrHostOperation::PreCall(request) => {
self.projected_mut()?.pre_call = Some((&request).into());
OcrHostResult::PreCall(Ok(request))
}
OcrHostOperation::PreCall(request) => OcrHostResult::PreCall(Ok(request)),
OcrHostOperation::DuringCall(request) => {
OcrHostResult::DuringCall(Ok(self.during_call(py, request)?))
}
@ -209,7 +222,7 @@ impl PythonRoute for PythonOcrHost {
OcrHostResult::PostCall(Ok(self.post_call(py, request)?))
}
OcrHostOperation::ConstructResponse(response) => {
self.state.end = Some(now(py)?);
self.state.finish(py)?;
self.state.response = Some(callbacks::response(py, response.as_ref())?);
OcrHostResult::Lifecycle(Ok(()))
}
@ -224,28 +237,22 @@ impl PythonRoute for PythonOcrHost {
}
fn cleanup(&mut self) {
self.projected = None;
self.retained = None;
}
fn traverse(&self, visit: &pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> {
let Some(projected) = &self.projected else {
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
let Some(retained) = &self.retained else {
return Ok(());
};
if let Some(provider) = &projected.azure_ad_token_provider {
if let Some(provider) = &retained.azure_ad_token_provider {
provider.traverse(visit)?;
}
if let Some(reader) = &projected.reader {
if let Some(reader) = &retained.reader {
reader.traverse(visit)?;
}
visit.call(&projected.body)?;
visit.call(&projected.headers)
}
}
struct BridgeOcrHooks;
impl litellm_core::ocr::hooks::OcrHooks for BridgeOcrHooks {
fn intercepts_requests(&self) -> bool {
true
if let Some(payload) = &retained.payload {
payload.traverse(visit)?;
}
Ok(())
}
}

View file

@ -62,13 +62,16 @@ fn call(
..OcrAdmission::all()
},
))?;
let host = PythonOcrHost::new(PythonCallState::new(
py,
args.unbind(),
kwargs.copy()?.unbind(),
asynchronous,
signature.name,
)?);
let host = PythonOcrHost::new(
PythonCallState::new(
py,
args.unbind(),
kwargs.copy()?.unbind(),
asynchronous,
signature.name,
)?,
signature,
);
run_call(py, call, host)
}

View file

@ -10,54 +10,38 @@ use litellm_core::ocr::{
};
use litellm_python_interop::from_py_preserving_errors as from_py;
use super::document::FileDocumentInput;
use super::errors::to_pyerr as ocr_error_to_pyerr;
use super::document::{FileDocumentInput, PythonFileReader};
use crate::auth::PythonTokenProvider;
use super::host::OcrRetained;
use crate::lifecycle::BoundArguments;
use crate::marshal::BoundRouteInputs;
use crate::marshal::{BoundRouteInputs, Projection};
/// Positional parameters of `ocr()` that are never projected into
/// `optional_params`.
const BOUND_FIELDS: &[&str] = &["model", "document", "timeout", "input_sources"];
pub(super) struct ProjectedOcrCall {
pub request: LiteLLMOcrRequest,
pub azure_ad_token_provider: Option<PythonTokenProvider>,
pub secret_fields: Vec<&'static str>,
pub reader: Option<PythonFileReader>,
}
enum ProjectedDocument {
File(FileDocumentInput),
Url(serde_json::Value),
}
impl ProjectedDocument {
fn into_native(self) -> Result<FileDocumentInput, litellm_core::ocr::Error> {
match self {
Self::File(file) => Ok(file),
Self::Url(value) => Ok(FileDocumentInput {
input: OcrDocument::try_from(value)?.into(),
reader: None,
}),
}
}
}
fn project_document(document: &Bound<'_, PyAny>) -> PyResult<ProjectedDocument> {
fn project_document(
document: &Bound<'_, PyAny>,
) -> PyResult<Result<FileDocumentInput, litellm_core::ocr::Error>> {
let kind: String = document.get_item("type")?.extract()?;
if kind != "file" {
return Ok(ProjectedDocument::Url(from_py(document)?));
let value: serde_json::Value = from_py(document)?;
return Ok(
OcrDocument::try_from(value).map(|document| FileDocumentInput {
input: document.into(),
reader: None,
}),
);
}
document.extract().map(ProjectedDocument::File)
document.extract().map(Ok)
}
/// Pure core assembly; every failure here is a typed `ocr::Error`.
fn build_request(
inputs: BoundRouteInputs,
document: ProjectedDocument,
) -> Result<ProjectedOcrCall, litellm_core::ocr::Error> {
let document = document.into_native()?;
document: Result<FileDocumentInput, litellm_core::ocr::Error>,
) -> Result<Projection<LiteLLMOcrRequest, OcrRetained>, litellm_core::ocr::Error> {
let document = document?;
let BoundRouteInputs {
model,
custom_llm_provider,
@ -83,18 +67,23 @@ fn build_request(
input_sources,
},
)?;
Ok(ProjectedOcrCall {
request,
azure_ad_token_provider,
secret_fields,
reader: document.reader,
Ok(Projection {
retained: OcrRetained {
model: request.model.clone(),
provider: request.provider_name(),
azure_ad_token_provider,
secret_fields,
reader: document.reader,
payload: None,
},
native: request,
})
}
pub(super) fn project(
py: Python<'_>,
arguments: &BoundArguments<'_>,
) -> PyResult<ProjectedOcrCall> {
) -> PyResult<Projection<LiteLLMOcrRequest, OcrRetained>> {
let model: String = arguments.extract("model")?;
let custom_llm_provider: Option<String> = arguments.optional("custom_llm_provider")?;
let document = project_document(&arguments.required("document")?)?;
@ -142,7 +131,7 @@ mod tests {
)
.unwrap();
assert!(matches!(
project_document(&file).unwrap().into_native().unwrap().input,
project_document(&file).unwrap().unwrap().input,
litellm_core::ocr::OcrDocumentInput::Bytes { bytes, mime_type, .. }
if bytes == b"%PDF-1.4"[..] && mime_type.as_deref() == Some("application/pdf")
));
@ -155,7 +144,7 @@ mod tests {
)
.unwrap();
assert!(matches!(
project_document(&original).unwrap().into_native().unwrap().input,
project_document(&original).unwrap().unwrap().input,
litellm_core::ocr::OcrDocumentInput::Document(OcrDocument::DocumentUrl { document_url, .. })
if document_url == "https://example.com/a.pdf"
));
@ -169,7 +158,7 @@ mod tests {
let document = py
.eval(c"{'type': 'mystery', 'mystery': 'x'}", None, None)
.unwrap();
let error = project_document(&document).unwrap().into_native().err().unwrap();
let error = project_document(&document).unwrap().err().unwrap();
assert!(error.to_string().contains("document"));
});
}
@ -181,14 +170,16 @@ mod tests {
let missing = py.eval(c"{}", None, None).unwrap();
assert!(
project_document(&missing)
.err().unwrap()
.err()
.unwrap()
.is_instance_of::<PyKeyError>(py)
);
let non_string = py.eval(c"{'type': 1}", None, None).unwrap();
assert!(
project_document(&non_string)
.err().unwrap()
.err()
.unwrap()
.is_instance_of::<PyTypeError>(py)
);
@ -202,8 +193,9 @@ class Document:
document = Document()
",
);
let error =
project_document(&locals.get_item("document").unwrap().unwrap()).err().unwrap();
let error = project_document(&locals.get_item("document").unwrap().unwrap())
.err()
.unwrap();
assert!(
error
.value(py)
@ -232,8 +224,11 @@ document = Document()
",
);
let document = locals.get_item("document").unwrap().unwrap();
let projected = project_document(&document).unwrap().into_native().unwrap();
assert!(matches!(projected.input, litellm_core::ocr::OcrDocumentInput::Bytes { .. }));
let projected = project_document(&document).unwrap().unwrap();
assert!(matches!(
projected.input,
litellm_core::ocr::OcrDocumentInput::Bytes { .. }
));
let reads: Vec<String> = document.getattr("reads").unwrap().extract().unwrap();
assert_eq!(reads, ["type", "mime_type", "file"]);
});

View file

@ -573,7 +573,6 @@ class Logging(LiteLLMLoggingBaseClass):
self.streaming_chunks: list[Any] = [] # for generating complete stream response
self.sync_streaming_chunks: list[Any] = [] # for generating complete stream response
self.log_raw_request_response = log_raw_request_response
self._native_callback_fast_path: bool = False
# Initialize dynamic callbacks
self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks

View file

@ -39,6 +39,7 @@ async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: pr
)
return await fallback(*args, **kwargs)
def _decline_types() -> tuple[type[BaseException], ...]:
exception_types: Final = native_exception_types()
return (exception_types[0],) if exception_types is not None else ()

View file

@ -3307,7 +3307,10 @@ class ProxyBaseLLMRequestProcessing:
pending: Final = getattr(logging_obj, "_native_pending_logging", None)
if pending is not None:
logging_obj._native_pending_logging = None # rebind-ok: consume the native OCR release signal once
pending.release(not exception_raised)
if exception_raised:
pending.close()
else:
pending()
_enqueue_fn: Final = getattr(logging_obj, "_enqueue_deferred_logging", None)
if _enqueue_fn is None:
return

View file

@ -29,10 +29,12 @@ def _build_document_from_upload(
filename: str | None,
content_type: str | None,
) -> dict[str, str]:
mime_type: Final = content_type.split(";")[0].strip() if content_type else None
if not mime_type or mime_type == "application/octet-stream":
if filename:
mime_type = get_mime_type(filename)
supplied_mime: Final = content_type.split(";")[0].strip() if content_type else None
mime_type: Final = (
get_mime_type(filename)
if filename and (not supplied_mime or supplied_mime == "application/octet-stream")
else supplied_mime
)
return convert_file_document_to_url_document(
{ # mutable-ok: OCR file document TypedDict handed to the converter

View file

@ -1,3 +1,4 @@
import datetime
from asyncio import Future
from collections.abc import Coroutine, Mapping
from typing import Never, final
@ -115,11 +116,20 @@ class TokenCounter:
def acount_request(self, body: bytes) -> Future[dict[str, object]]: ...
def gil_stats() -> dict[str, int]: ...
def _debug_setup(
call_type: str,
args: tuple[object, ...],
kwargs: dict[str, object],
start: datetime.datetime,
asynchronous: bool,
) -> tuple[object, dict[str, object]]: ...
__all__ = [
"ResponsesWebSocketConnection",
"RustBridgeDeclined",
"RustUpstreamError",
"TokenCounter",
"_debug_setup",
"achat_completions",
"amessages",
"aocr",

View file

@ -47,3 +47,8 @@ def native_exception_types() -> tuple[type[BaseException], type[BaseException]]
if not isinstance(declined, type) or not isinstance(upstream, type):
return None
return declined, upstream
def native_decline_types() -> tuple[type[BaseException], ...]:
exceptions: Final = native_exception_types()
return (exceptions[0],) if exceptions is not None else ()

View file

@ -0,0 +1,913 @@
"""Leaf helpers invoked by the native callback dispatcher.
Rust selects every target, delivery and sequence. Nothing here chooses a callback,
reads a registry, or decides whether an event fires. Each function performs one
integration-specific or interop-specific action on values Rust hands it.
Labeled leaf helpers pending native migration (see rust-callback-inventory.md):
`prepare_success_logging`, `prepare_failure_logging`, `dispatch_named_success`,
`dispatch_named_failure`, `dispatch_callable`. They are deleted when cost and
payload construction move into core and when each string integration becomes a
CustomLogger.
"""
from __future__ import annotations
import datetime
import json
import traceback
from collections.abc import Awaitable, Callable, Mapping
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Final,
Literal,
Protocol,
TypeAlias,
cast, # noqa: TID251 # narrows the untyped legacy Logging and CustomLogger surfaces once
)
from litellm.integrations.custom_logger import CustomLogger
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import StandardCallbackDynamicParams
TerminalFamily: TypeAlias = Literal["sync_success", "async_success", "sync_failure", "async_failure"]
Family: TypeAlias = Literal["request", "deployment", TerminalFamily]
Details: TypeAlias = dict[
str, object
] # mutable-ok: model_call_details is the shared mutable envelope callbacks write to
Timestamp: TypeAlias = datetime.datetime
LegacyCall: TypeAlias = Callable[..., object]
class LoggerView(Protocol):
model: str | None
messages: object
call_type: str
start_time: Timestamp
litellm_call_id: str
completion_start_time: Timestamp | None
model_call_details: Details
log_raw_request_response: bool
standard_callback_dynamic_params: StandardCallbackDynamicParams
standard_built_in_tools_params: object
def record_api_call_start_time(self) -> None: ...
def record_post_call(
self, original_response: object, input: object, api_key: object, additional_args: Details
) -> None: ...
def should_run_callback(self, callback: object, litellm_params: Details, event_hook: str) -> bool: ...
def _pre_call(self, input: str, api_key: str | None, model: str | None, additional_args: Details) -> None: ...
def _print_llm_call_debugging_log(
self, api_base: str, headers: Mapping[str, str], additional_args: Details
) -> None: ...
def _get_request_curl_command(
self, api_base: str, headers: Mapping[str, str] | None, additional_args: Details, data: object
) -> str: ...
def _get_masked_api_base(self, api_base: str) -> str: ...
def _get_raw_request_body(self, data: object) -> Details: ...
def _get_masked_headers(self, headers: Mapping[str, str]) -> Details: ...
def _response_cost_calculator(self, result: object) -> float | None: ...
def _build_standard_logging_payload(
self, init_response_obj: object, start_time: Timestamp, end_time: Timestamp
) -> object: ...
def _handle_callback_failure(self, callback: object) -> None: ...
class IntegrationView(Protocol):
def log_pre_api_call(self, model: str | None, messages: object, kwargs: Details) -> None: ...
def log_post_api_call(
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp | None
) -> None: ...
def log_success_event(
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp
) -> None: ...
def log_failure_event(
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp
) -> None: ...
def async_log_success_event(
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp
) -> Awaitable[None]: ...
def async_log_failure_event(
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp
) -> Awaitable[None]: ...
def logging_hook(self, kwargs: Details, result: object, call_type: str) -> tuple[Details, object]: ...
def async_logging_hook(
self, kwargs: Details, result: object, call_type: str
) -> Awaitable[tuple[Details, object]]: ...
def redact_standard_logging_payload_from_model_call_details(self, model_call_details: Details) -> Details: ...
def log_input_event(
self, model: str | None, messages: object, kwargs: Details, print_verbose: LegacyCall, callback_func: LegacyCall
) -> None: ...
def log_event(
self,
kwargs: Details,
response_obj: object,
start_time: Timestamp,
end_time: Timestamp,
print_verbose: LegacyCall,
callback_func: LegacyCall,
) -> None: ...
def async_log_event(
self,
kwargs: Details,
response_obj: object,
start_time: Timestamp,
end_time: Timestamp,
print_verbose: LegacyCall,
callback_func: LegacyCall,
) -> Awaitable[None]: ...
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
def _details_of(value: object) -> Details:
return cast(Details, value) # cast-ok: legacy model_call_details nesting is untyped and shared by callbacks
def _call_of(value: object) -> LegacyCall:
return cast(LegacyCall, value) # cast-ok: legacy integration entry points are untyped
def _attribute(target: object, name: str) -> object:
value: Final[object] = getattr(target, name) # pyright: ignore[reportAny] # legacy attributes are untyped by definition
return value
def _seed(details: Details, key: str) -> Details:
existing: Final[object] = details.get(key)
if isinstance(existing, dict):
return _details_of(existing) # pyright: ignore[reportUnknownArgumentType] # isinstance yields dict[Unknown, Unknown]
seeded: Final[Details] = {} # mutable-ok: legacy envelope seed shared with callbacks
details[key] = seeded # rebind-ok: seeding a nested dict on the shared envelope is the legacy contract
return seeded
def _logger(logger: Logging) -> LoggerView:
return cast(LoggerView, logger) # cast-ok: legacy Logging is untyped; this protocol names the attributes we read
def _integration(callback: CustomLogger) -> IntegrationView:
return cast(IntegrationView, callback) # cast-ok: legacy CustomLogger methods are untyped
def _singleton(name: str) -> object:
from litellm.litellm_core_utils import litellm_logging
return getattr(litellm_logging, name, None) # pyright: ignore[reportAny] # legacy module globals are rebound at runtime
def _print_verbose() -> LegacyCall:
from litellm.litellm_core_utils import litellm_logging
return cast(LegacyCall, litellm_logging.print_verbose) # cast-ok: legacy debug printer is untyped
def _redact_string(value: str) -> str:
from litellm.litellm_core_utils import litellm_logging
redact: Final = _call_of(litellm_logging._redact_string) # pyright: ignore[reportPrivateUsage] # private legacy helper
return str(redact(value))
def _redact_result(details: Details, result: object) -> object:
from litellm.litellm_core_utils import redact_messages
redact: Final = cast(LegacyCall, redact_messages.redact_message_input_output_from_logging) # cast-ok: legacy helper
return redact(model_call_details=details, result=result)
def record_pre_call(
logger: Logging,
*,
api_key: str | None,
body: Details,
headers: Mapping[str, str],
url: str,
) -> None:
view: Final = _logger(logger)
additional_args: Final[Details] = { # mutable-ok: legacy additional_args is rebound by callbacks in place
"complete_input_dict": body,
"headers": headers,
"api_base": url,
}
view._pre_call(input="OCR document processing", api_key=api_key, model=None, additional_args=additional_args) # pyright: ignore[reportPrivateUsage] # legacy state writer
view._print_llm_call_debugging_log(api_base=url, headers=headers, additional_args=additional_args) # pyright: ignore[reportPrivateUsage] # legacy debug output
_capture_raw_request(view, additional_args)
_run_logger_fn(logger)
view.record_api_call_start_time()
def _capture_raw_request(view: LoggerView, additional_args: Details) -> None:
import litellm
from litellm.types.utils import RawRequestTypedDict
if not (view.log_raw_request_response or litellm.log_raw_request_response):
return
details: Final = view.model_call_details
metadata: Final = _seed(_seed(details, "litellm_params"), "metadata")
if litellm.turn_off_message_logging:
metadata["raw_request"] = "redacted by litellm. 'litellm.turn_off_message_logging=True'"
return
api_base: Final = str(additional_args.get("api_base") or "")
headers: Final = cast(Mapping[str, str], additional_args.get("headers") or _EMPTY) # cast-ok: legacy nested dict
body: Final = additional_args.get("complete_input_dict", _EMPTY)
try:
curl: Final = view._get_request_curl_command( # pyright: ignore[reportPrivateUsage] # legacy debug formatter
api_base=api_base, headers=headers, additional_args=additional_args, data=body
)
metadata["raw_request"] = _redact_string(str(curl))
details["raw_request_typed_dict"] = RawRequestTypedDict(
raw_request_api_base=view._get_masked_api_base(api_base), # pyright: ignore[reportPrivateUsage] # legacy masking
raw_request_body=view._get_raw_request_body(body), # pyright: ignore[reportPrivateUsage] # legacy masking
raw_request_headers=view._get_masked_headers(headers), # pyright: ignore[reportPrivateUsage] # legacy masking
error=None,
)
except Exception as error: # noqa: BLE001 # raw-request capture is best effort by contract
details["raw_request_typed_dict"] = RawRequestTypedDict(error=str(error))
metadata["raw_request"] = _redact_string(f"Unable to Log raw request: {error}")
def _run_logger_fn(logger: Logging) -> None:
from litellm._logging import verbose_logger
logger_fn: Final = getattr(logger, "logger_fn", None)
if not callable(logger_fn):
return
try:
_call_of(logger_fn)(_logger(logger).model_call_details)
except Exception as error: # noqa: BLE001 # user logger_fn failures never fail the request
verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", error)
def record_post_call(logger: Logging, *, original_response: object, body: Details, headers: Mapping[str, str]) -> None:
view: Final = _logger(logger)
serialized: Final = (
json.dumps(original_response, default=str) if isinstance(original_response, dict) else original_response
)
view.record_post_call(
original_response=serialized,
input=None,
api_key=None,
additional_args={"complete_input_dict": body, "headers": headers}, # mutable-ok: rebound in place by callbacks
)
_run_logger_fn(logger)
_redact_result(view.model_call_details, serialized)
def log_pre_api_call(logger: Logging, callback: CustomLogger) -> None:
view: Final = _logger(logger)
_integration(callback).log_pre_api_call(model=view.model, messages=view.messages, kwargs=view.model_call_details)
def log_post_api_call(logger: Logging, callback: CustomLogger) -> None:
view: Final = _logger(logger)
_integration(callback).log_post_api_call(
kwargs=view.model_call_details, response_obj=None, start_time=view.start_time, end_time=None
)
def dispatch_named_request(logger: Logging, name: str, event: Literal["pre_api_call", "post_api_call"]) -> None:
from litellm.integrations.supabase import Supabase
view: Final = _logger(logger)
details: Final = view.model_call_details
match name:
case "supabase" if event == "pre_api_call" and isinstance(client := _singleton("supabaseClient"), Supabase):
client.input_log_event(
model=view.model,
messages=view.messages,
end_user=details.get("user", "default"),
litellm_call_id=details["litellm_call_id"],
print_verbose=_print_verbose(),
)
case "sentry" if callable(add_breadcrumb := _singleton("add_breadcrumb")):
add_breadcrumb(category="litellm.llm_call", message=f"Model Call Details {event}: {details}", level="info")
case _:
return
def dispatch_callable_request(logger: Logging, callback: LegacyCall) -> None:
custom: Final = _singleton("customLogger")
if not isinstance(custom, CustomLogger):
return
view: Final = _logger(logger)
_integration(custom).log_input_event(
model=view.model,
messages=view.messages,
kwargs=view.model_call_details,
print_verbose=_print_verbose(),
callback_func=callback,
)
def _typed_call_type(call_type: str) -> object:
from litellm.types.utils import CallTypes
try:
return CallTypes(call_type)
except ValueError:
return None
def _awaitable(value: object) -> Awaitable[object]:
return cast(Awaitable[object], value) # cast-ok: legacy async hooks return coroutines
def pre_call_deployment_hook(callback: CustomLogger, kwargs: Details, call_type: str) -> Awaitable[object]:
hook: Final = _call_of(_attribute(callback, "async_pre_call_deployment_hook"))
return _awaitable(hook(kwargs, _typed_call_type(call_type)))
def post_call_success_deployment_hook(
callback: CustomLogger, kwargs: Details, response: object, call_type: str
) -> Awaitable[object]:
hook: Final = _call_of(_attribute(callback, "async_post_call_success_deployment_hook"))
return _awaitable(hook(kwargs, response, _typed_call_type(call_type)))
def failure_deployment_hook_view(
kwargs: Details, exception: BaseException
) -> tuple[Mapping[str, object], BaseException, int | None]:
from litellm import utils
raw_depth: Final = kwargs.get("fallback_depth")
depth: Final = raw_depth if isinstance(raw_depth, int) else None
safe_request: Final = MappingProxyType({key: value for key, value in kwargs.items() if key != "attempted_targets"})
snapshot: Final = _call_of(utils._snapshot_exception_for_hook)(exception) # pyright: ignore[reportPrivateUsage] # legacy snapshot helper
return safe_request, cast(BaseException, snapshot), depth # cast-ok: snapshot is a same-class copy of the exception
def post_call_failure_deployment_hook(
callback: CustomLogger,
request: Mapping[str, object],
exception: BaseException,
call_type: str,
fallback_depth: int | None,
) -> Awaitable[object]:
from litellm import utils
hook: Final = _call_of(_attribute(callback, "async_post_call_failure_deployment_hook"))
accepts_depth: Final = _call_of(utils._accepts_fallback_depth_kwarg_for_class)(type(callback)) # pyright: ignore[reportPrivateUsage] # legacy signature probe
typed: Final = _typed_call_type(call_type)
if accepts_depth:
return _awaitable(hook(request, exception, typed, fallback_depth=fallback_depth))
return _awaitable(hook(request, exception, typed))
def report_deployment_failure_hook_error(callback: object, error: BaseException) -> None:
from litellm._logging import verbose_logger
verbose_logger.debug("async_post_call_failure_deployment_hook error in %s: %s", type(callback).__name__, error)
def report_target_failure(logger: Logging, callback: object, family: Family, error: BaseException) -> None:
from litellm._logging import verbose_logger
verbose_logger.error(
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while %s logging with %s: %s",
family,
callback,
"".join(traceback.format_exception(error)),
)
capture: Final = _singleton("capture_exception")
if callable(capture) and family in ("request", "sync_success", "sync_failure"):
capture(error)
if family not in ("request", "deployment", "sync_failure"):
_logger(logger)._handle_callback_failure(callback=callback) # pyright: ignore[reportPrivateUsage] # legacy prometheus counter
def prepare_success_logging(logger: Logging, response: object, start_time: Timestamp, end_time: Timestamp) -> object:
from litellm.litellm_core_utils.litellm_logging import emit_standard_logging_payload
from litellm.types.utils import StandardLoggingPayload
view: Final = _logger(logger)
details: Final = view.model_call_details
if view.completion_start_time is None:
view.completion_start_time = end_time
details["completion_start_time"] = end_time
details["log_event_type"] = "successful_api_call"
details["end_time"] = end_time
details["cache_hit"] = None
hidden: Final = _details_of(getattr(response, "_hidden_params", None) or {}) # mutable-ok: legacy fallback envelope
if hidden and isinstance(details.get("litellm_params"), dict):
_seed(_seed(details, "litellm_params"), "metadata")["hidden_params"] = hidden
existing: Final = details.get("response_cost")
if "response_cost" in hidden:
details["response_cost"] = hidden["response_cost"]
elif existing is None or existing == 0:
details["response_cost"] = view._response_cost_calculator(result=response) # pyright: ignore[reportPrivateUsage] # labeled leaf: native cost pending
payload: Final = view._build_standard_logging_payload(response, start_time, end_time) # pyright: ignore[reportPrivateUsage] # labeled leaf: native payload pending
details["standard_logging_object"] = payload
if payload is not None:
emit_standard_logging_payload(
cast(StandardLoggingPayload, payload) # cast-ok: legacy builder returns the payload TypedDict
) # cast-ok: legacy builder returns the payload TypedDict
return _redact_result(details, response)
def prepare_failure_logging(
logger: Logging, exception: BaseException, start_time: Timestamp, end_time: Timestamp
) -> str:
from litellm.litellm_core_utils import litellm_logging
formatted: Final = "".join(traceback.format_exception(exception))
view: Final = _logger(logger)
details: Final = view.model_call_details
if details.get("exception") is exception and details.get("standard_logging_object") is not None:
return formatted
details["log_event_type"] = "failed_api_call"
details["exception"] = exception
details["traceback_exception"] = _redact_string(formatted)
details["end_time"] = end_time
details.setdefault("original_response", None)
if details.get("combined_usage_object") is None:
details["response_cost"] = 0
headers: Final = getattr(exception, "headers", None)
if isinstance(headers, Mapping):
_seed(_seed(details, "litellm_params"), "metadata").update(
_details_of(headers) # pyright: ignore[reportUnknownArgumentType] # exception headers carry no element types
)
build: Final = cast( # cast-ok: labeled leaf: native payload pending
LegacyCall, litellm_logging.get_standard_logging_object_payload
)
details["standard_logging_object"] = build(
kwargs=details,
init_response_obj=_EMPTY,
start_time=start_time,
end_time=end_time,
logging_obj=logger,
status="failure",
error_str=_redact_string(str(exception)),
original_exception=exception,
standard_built_in_tools_params=view.standard_built_in_tools_params,
)
return formatted
_EVENT_HOOKS: Final[Mapping[TerminalFamily, str]] = MappingProxyType(
{
"sync_success": "success_handler",
"async_success": "async_success_handler",
"sync_failure": "failure_handler",
"async_failure": "async_failure_handler",
}
)
def should_run_callback(logger: Logging, callback: object, family: TerminalFamily) -> bool:
view: Final = _logger(logger)
params: Final = _details_of(view.model_call_details.get("litellm_params") or _EMPTY)
return view.should_run_callback(callback=callback, litellm_params=params, event_hook=_EVENT_HOOKS[family])
def should_run_guardrail_hook(logger: Logging, callback: object) -> bool:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import GuardrailEventHooks
if not isinstance(callback, CustomGuardrail):
return True
decide: Final = cast(LegacyCall, callback.should_run_guardrail) # cast-ok: legacy guardrail method is untyped
return decide(data=_logger(logger).model_call_details, event_type=GuardrailEventHooks.logging_only) is True
def logging_hook(logger: Logging, callback: CustomLogger, result: object) -> object:
view: Final = _logger(logger)
details, replaced = _integration(callback).logging_hook(
kwargs=view.model_call_details, result=result, call_type=view.call_type
)
view.model_call_details = details
return replaced
async def async_logging_hook(logger: Logging, callback: CustomLogger, result: object) -> object:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils import redact_messages
view: Final = _logger(logger)
redact: Final = cast( # cast-ok: legacy helper
LegacyCall, redact_messages.redact_message_input_output_from_custom_logger
)
redacted: Final = (
result
if isinstance(callback, CustomGuardrail)
else redact(result=result, litellm_logging_obj=logger, custom_logger=callback)
)
details, replaced = await _integration(callback).async_logging_hook(
kwargs=view.model_call_details, result=redacted, call_type=view.call_type
)
view.model_call_details = details
return replaced
def is_sync_request(logger: Logging) -> bool:
from litellm.litellm_core_utils.litellm_logging import Logging as LoggingClass
params: Final = _details_of(_logger(logger).model_call_details.get("litellm_params") or _EMPTY)
decide: Final = _call_of(LoggingClass._is_sync_litellm_request) # pyright: ignore[reportPrivateUsage] # legacy predicate
return decide(params) is True
def mark_logged(logger: Logging, marker: str) -> None:
_logger(logger).model_call_details[marker] = True
def already_logged(logger: Logging, marker: str) -> bool:
return _logger(logger).model_call_details.get(marker, False) is True
def log_success_event(
logger: Logging, callback: CustomLogger, response: object, start_time: Timestamp, end_time: Timestamp
) -> None:
_integration(callback).log_success_event(
kwargs=_logger(logger).model_call_details, response_obj=response, start_time=start_time, end_time=end_time
)
def async_log_success_event(
logger: Logging, callback: CustomLogger, response: object, start_time: Timestamp, end_time: Timestamp
) -> Awaitable[None]:
from litellm.litellm_core_utils import redact_messages
integration: Final = _integration(callback)
details: Final = integration.redact_standard_logging_payload_from_model_call_details(
model_call_details=_logger(logger).model_call_details
)
redact: Final = cast( # cast-ok: legacy helper
Callable[..., Details], redact_messages.redact_streaming_responses_for_custom_logger
)
view: Final = redact(model_call_details=details, custom_logger=callback)
return integration.async_log_success_event(
kwargs=view, response_obj=response, start_time=start_time, end_time=end_time
)
def log_failure_event(logger: Logging, callback: CustomLogger, start_time: Timestamp, end_time: Timestamp) -> None:
_integration(callback).log_failure_event(
kwargs=_logger(logger).model_call_details, response_obj=None, start_time=start_time, end_time=end_time
)
def async_log_failure_event(
logger: Logging, callback: CustomLogger, start_time: Timestamp, end_time: Timestamp
) -> Awaitable[None]:
return _integration(callback).async_log_failure_event(
kwargs=_logger(logger).model_call_details, response_obj=None, start_time=start_time, end_time=end_time
)
def _custom_logger_singleton() -> IntegrationView:
from litellm.litellm_core_utils import litellm_logging
existing: Final = _singleton("customLogger")
if isinstance(existing, CustomLogger):
return _integration(existing)
created: Final = CustomLogger()
litellm_logging.customLogger = created # pyright: ignore[reportAttributeAccessIssue] # legacy module global
return _integration(created)
def dispatch_callable(
logger: Logging,
callback: LegacyCall,
family: TerminalFamily,
response: object,
start_time: Timestamp,
end_time: Timestamp,
) -> Awaitable[None] | None:
custom: Final = _custom_logger_singleton()
details: Final = _logger(logger).model_call_details
match family:
case "sync_success" | "sync_failure":
custom.log_event(
kwargs=details,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=_print_verbose(),
callback_func=callback,
)
return None
case "async_success" | "async_failure":
return custom.async_log_event(
kwargs=details,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=_print_verbose(),
callback_func=callback,
)
def dispatch_named_success(
logger: Logging, name: str, response: object, start_time: Timestamp, end_time: Timestamp
) -> Awaitable[None] | None:
from litellm.integrations.athina import AthinaLogger
from litellm.integrations.greenscale import GreenscaleLogger
from litellm.integrations.helicone import HeliconeLogger
from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger
from litellm.integrations.lunary import LunaryLogger
from litellm.integrations.openmeter import OpenMeterLogger
from litellm.integrations.prompt_layer import PromptLayerLogger
from litellm.integrations.s3 import S3Logger
from litellm.integrations.supabase import Supabase
from litellm.integrations.traceloop import TraceloopLogger
from litellm.integrations.weights_biases import WeightsBiasesLogger
view: Final = _logger(logger)
details: Final = view.model_call_details
print_verbose: Final = _print_verbose()
without_response: Final = { # mutable-ok: legacy integrations receive a private mutable copy
key: value for key, value in details.items() if key != "original_response"
}
match name:
case "promptlayer" if isinstance(promptlayer := _singleton("promptLayerLogger"), PromptLayerLogger):
promptlayer.log_event(
kwargs=details,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
)
case "wandb" if isinstance(wandb := _singleton("weightsBiasesLogger"), WeightsBiasesLogger):
wandb.log_event(
kwargs=details,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
)
case "athina" if isinstance(athina := _singleton("athinaLogger"), AthinaLogger):
athina.log_event(
kwargs=details,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
)
case "logfire" if isinstance(logfire := _singleton("logfireLogger"), LogfireLogger):
logfire.log_event(
kwargs=without_response,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
level=LogfireLevel.INFO,
)
case "greenscale" if isinstance(greenscale := _singleton("greenscaleLogger"), GreenscaleLogger):
greenscale.log_event(
kwargs=without_response,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
)
case "supabase" if isinstance(supabase := _singleton("supabaseClient"), Supabase):
supabase.log_event(
model=view.model,
messages=view.messages,
end_user=details.get("user", "default"),
response_obj=response,
start_time=start_time,
end_time=end_time,
litellm_call_id=details["litellm_call_id"],
print_verbose=print_verbose,
)
case "lunary" if isinstance(lunary := _singleton("lunaryLogger"), LunaryLogger):
lunary.log_event(
kwargs=details,
type="llm",
event="end",
model=view.model,
input=details["input"],
user_id=details.get("user", "default"),
response_obj=response,
start_time=start_time,
end_time=end_time,
run_id=view.litellm_call_id,
print_verbose=print_verbose,
)
case "helicone" if isinstance(helicone := _singleton("heliconeLogger"), HeliconeLogger):
helicone.log_success(
model=view.model,
messages=view.messages,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
kwargs=details,
)
case "langfuse":
_langfuse(
logger, response=response, start_time=start_time, end_time=end_time, level=None, status_message=None
)
case "traceloop" if isinstance(traceloop := _singleton("traceloopLogger"), TraceloopLogger):
traceloop.log_event(
kwargs=details,
response_obj=response,
start_time=start_time,
end_time=end_time,
user_id=details.get("user", None),
print_verbose=print_verbose,
)
case "s3" if isinstance(s3 := _singleton("s3Logger"), S3Logger):
s3.log_event(
kwargs=details,
response_obj=response,
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
)
case "openmeter" if isinstance(openmeter := _singleton("openMeterLogger"), OpenMeterLogger):
return openmeter.async_log_success_event(
kwargs=details, response_obj=response, start_time=start_time, end_time=end_time
)
case "dynamodb":
return _dynamodb(details, response, start_time, end_time, print_verbose)
case _:
return None
return None
def _dynamodb(
details: Details, response: object, start_time: Timestamp, end_time: Timestamp, print_verbose: LegacyCall
) -> Awaitable[None]:
from litellm.integrations.dynamodb import DyanmoDBLogger
from litellm.litellm_core_utils import litellm_logging
existing: Final = _singleton("dynamoLogger")
dynamo: Final = existing if isinstance(existing, DyanmoDBLogger) else DyanmoDBLogger()
litellm_logging.dynamoLogger = dynamo # pyright: ignore[reportAttributeAccessIssue] # legacy module global
return dynamo._async_log_event( # pyright: ignore[reportPrivateUsage] # legacy async entry point
kwargs=details, response_obj=response, start_time=start_time, end_time=end_time, print_verbose=print_verbose
)
def _langfuse(
logger: Logging,
*,
response: object,
start_time: Timestamp,
end_time: Timestamp,
level: str | None,
status_message: str | None,
) -> None:
from litellm.integrations.langfuse.langfuse import LangFuseLogger
from litellm.integrations.langfuse.langfuse_handler import LangFuseHandler
from litellm.litellm_core_utils import litellm_logging
from litellm.types.utils import ModelResponse
view: Final = _logger(logger)
kwargs: Final = { # mutable-ok: langfuse receives a private mutable copy
key: value for key, value in view.model_call_details.items() if key != "original_response"
}
global_logger: Final = _singleton("langFuseLogger")
handler: Final = LangFuseHandler.get_langfuse_logger_for_request(
globalLangfuseLogger=global_logger if isinstance(global_logger, LangFuseLogger) else None,
standard_callback_dynamic_params=view.standard_callback_dynamic_params,
in_memory_dynamic_logger_cache=litellm_logging.in_memory_dynamic_logger_cache,
)
user: Final = kwargs.get("user")
result: Final = handler.log_event_on_langfuse(
kwargs=kwargs,
response_obj=cast(ModelResponse, response), # cast-ok: OCR responses sit outside the legacy union
start_time=start_time,
end_time=end_time,
user_id=user if isinstance(user, str) else None,
level="DEFAULT" if level is None else level,
status_message=status_message,
)
trace_id: Final = result.get("trace_id")
if isinstance(trace_id, str):
litellm_logging.in_memory_trace_id_cache.set_cache(
litellm_call_id=view.litellm_call_id, service_name="langfuse", trace_id=trace_id
)
def dispatch_named_failure(
logger: Logging,
name: str,
exception: BaseException,
formatted: str,
start_time: Timestamp,
end_time: Timestamp,
) -> None:
from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger
from litellm.integrations.lunary import LunaryLogger
from litellm.integrations.supabase import Supabase
from litellm.integrations.traceloop import TraceloopLogger
view: Final = _logger(logger)
details: Final = view.model_call_details
print_verbose: Final = _print_verbose()
match name:
case "lunary" if isinstance(lunary := _singleton("lunaryLogger"), LunaryLogger):
lunary.log_event(
kwargs=details,
type="llm",
event="error",
user_id=details.get("user", "default"),
model=view.model,
input=details["input"],
error=formatted,
run_id=view.litellm_call_id,
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
)
case "sentry" if callable(capture := _singleton("capture_exception")):
capture(exception)
case "supabase" if isinstance(supabase := _singleton("supabaseClient"), Supabase):
supabase.log_event(
model=view.model,
messages=view.messages,
end_user=details.get("user", "default"),
response_obj=None,
start_time=start_time,
end_time=end_time,
litellm_call_id=details["litellm_call_id"],
print_verbose=print_verbose,
)
case "langfuse":
_langfuse(
logger,
response=None,
start_time=start_time,
end_time=end_time,
level="ERROR",
status_message=str(exception),
)
case "traceloop" if isinstance(traceloop := _singleton("traceloopLogger"), TraceloopLogger):
traceloop.log_event(
start_time=start_time,
end_time=end_time,
response_obj=None,
user_id=details.get("user", None),
print_verbose=print_verbose,
status_message=str(exception),
level="ERROR",
kwargs=details,
)
case "logfire" if isinstance(logfire := _singleton("logfireLogger"), LogfireLogger):
logfire.log_event(
kwargs={ # mutable-ok: logfire receives a private mutable copy
key: value for key, value in details.items() if key != "original_response"
}
| {"exception": exception}, # mutable-ok: merged into the private copy above
response_obj=None,
start_time=start_time,
end_time=end_time,
level=LogfireLevel.ERROR,
print_verbose=print_verbose,
)
case _:
return
def restore_correlation_context(logger: object) -> None:
from litellm import utils
utils._restore_correlation_context_if_supported(logger) # pyright: ignore[reportPrivateUsage] # legacy interop helper
def submit_worker(job: Callable[[], None]) -> None:
import contextvars
from litellm.litellm_core_utils.thread_pool_executor import executor
context: Final = contextvars.copy_context()
_ = executor.submit(context.run, job)
def enqueue_background(coroutine: Awaitable[None]) -> None:
import contextvars
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
enqueue: Final = _call_of(_attribute(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue"))
contextvars.copy_context().run(enqueue, coroutine)

View file

@ -1,19 +1,8 @@
from __future__ import annotations
import datetime
import os
import uuid
from collections.abc import Awaitable, Mapping
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Final,
Protocol,
cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from typing import Final, Protocol
@dataclass(frozen=True, slots=True)
@ -53,50 +42,6 @@ async def drive(execution: Execution) -> object:
execution.close()
class MetadataUpdater(Protocol):
def __call__(
self,
result: object,
logging_obj: Logging,
model: str | None,
kwargs: dict[str, object],
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
@dataclass(frozen=True, slots=True)
class CallSetup:
logger: Logging
kwargs: dict[str, object]
def setup(
call_type: str,
args: tuple[object, ...],
kwargs: Mapping[str, object],
start_time: datetime.datetime,
asynchronous: bool,
) -> CallSetup:
from litellm import utils
from litellm.litellm_core_utils.litellm_logging import Logging
arguments: Final = { # mutable-ok: function_setup consumes an owned kwargs dict
"litellm_call_id": str(uuid.uuid4()),
**kwargs,
}
supplied: Final = arguments.get("litellm_logging_obj")
if isinstance(supplied, Logging):
supplied._native_callback_fast_path = False # pyright: ignore[reportPrivateUsage] # supplied loggers retain all dispatch contracts
return CallSetup(supplied, arguments)
logger, prepared = utils.function_setup(
call_type, utils.Rules(), start_time, *args, is_async_call=asynchronous, **arguments
)
if type(logger) is Logging and call_type in ("ocr", "aocr"):
logger._native_callback_fast_path = True # pyright: ignore[reportPrivateUsage] # only bridge-created OCR loggers opt into callback elision
return CallSetup(logger, prepared)
def check_limits(kwargs: Mapping[str, object]) -> None:
import litellm
from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit
@ -106,100 +51,3 @@ def check_limits(kwargs: Mapping[str, object]) -> None:
raise litellm.BudgetExceededError(current_cost=current_cost, max_budget=litellm.max_budget)
if max_retries_per_request_hit(kwargs, litellm.num_retries_per_request):
raise RuntimeError("Max retries per request hit!")
def finalize(
response: object,
logger: Logging,
kwargs: dict[str, object],
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None:
from litellm.litellm_core_utils.llm_response_utils import response_metadata
model: Final = kwargs.get("model")
update: Final = cast( # cast-ok: legacy metadata function accepts concrete kwargs
MetadataUpdater, response_metadata.update_response_metadata
)
update(response, logger, model if isinstance(model, str) else None, kwargs, start_time, end_time)
def deployment_callbacks_needed() -> bool:
import litellm
from litellm.integrations.custom_logger import CustomLogger
return any(isinstance(callback, CustomLogger) for callback in litellm.callbacks)
def callbacks_needed(logger: Logging, phase: str) -> bool:
import litellm
from litellm._logging import (
_is_debugging_on, # pyright: ignore[reportPrivateUsage] # use the same debug gate as Logging
)
if (
_is_debugging_on()
or getattr(logger, "litellm_request_debug", False)
or os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD")
):
return True
input_needed: Final = bool(
litellm.input_callback
or litellm._async_input_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
or logger.dynamic_input_callbacks
or callable(getattr(logger, "logger_fn", None))
or logger.log_raw_request_response
or litellm.log_raw_request_response
)
match phase:
case "input":
return input_needed
case "sync_success":
return bool(litellm.success_callback or logger.dynamic_success_callbacks)
case "sync_success_async":
return bool(
(litellm.success_callback or logger.dynamic_success_callbacks)
and logger._should_run_sync_callbacks_for_async_calls() # pyright: ignore[reportPrivateUsage] # preserve async call filtering of sync callbacks
)
case "async_success":
return bool(litellm._async_success_callback or logger.dynamic_async_success_callbacks) # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
case "sync_failure":
return bool(litellm.failure_callback or logger.dynamic_failure_callbacks)
case "async_failure":
return bool(litellm._async_failure_callback or logger.dynamic_async_failure_callbacks) # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
case "payload":
return bool(
input_needed
or litellm.success_callback
or litellm.failure_callback
or litellm._async_success_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
or litellm._async_failure_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
or logger.dynamic_success_callbacks
or logger.dynamic_async_success_callbacks
or logger.dynamic_failure_callbacks
or logger.dynamic_async_failure_callbacks
)
case _:
return True
def success_bookkeeping(
logger: Logging, response: object, start: datetime.datetime, end: datetime.datetime, asynchronous: bool
) -> None:
phase: Final = "async_success" if asynchronous else "sync_success"
if logger.should_run_logging(phase):
logger._success_handler_helper_fn( # pyright: ignore[reportPrivateUsage] # retain success bookkeeping without constructing a callback payload
result=response, start_time=start, end_time=end, build_logging_payload=False
)
logger.has_run_logging(phase)
def failure_bookkeeping(
logger: Logging, error: BaseException, start: datetime.datetime, end: datetime.datetime, asynchronous: bool
) -> None:
phase: Final = "async_failure" if asynchronous else "sync_failure"
if logger.should_run_logging(phase):
logger._failure_handler_helper_fn( # pyright: ignore[reportPrivateUsage] # retain failure accounting without formatting an unused traceback or payload
error, "", start, end, build_logging_payload=False
)
logger.has_run_logging(phase)

View file

@ -1,6 +1,6 @@
from __future__ import annotations
from collections.abc import Callable, Coroutine, Mapping
from collections.abc import Callable, Coroutine, Mapping, Sequence
from types import MappingProxyType
from typing import Final, Protocol, cast # noqa: TID251 # validates dynamically loaded native callables
@ -8,6 +8,56 @@ from pydantic import TypeAdapter
from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse
from litellm.rust_bridge.bindings import NativeBinding
from litellm.types.utils import CustomPricingLiteLLMParams
class OcrLoggingProtocol(Protocol):
def update_from_kwargs(
self,
*,
kwargs: dict[str, object],
model: str,
optional_params: dict[str, object],
litellm_params: dict[str, object],
custom_llm_provider: str,
) -> object: ...
def _redact(params: Mapping[str, object], secret_fields: Sequence[str]) -> dict[str, object]:
return { # mutable-ok: update_from_kwargs takes dict
name: "****" if name in secret_fields else value
for name, value in params.items()
if name != "proxy_server_request"
}
def update_logging(
logger: OcrLoggingProtocol,
kwargs: Mapping[str, object],
model: str,
custom_llm_provider: str,
optional_params: Mapping[str, object],
secret_fields: Sequence[str],
url: str,
) -> None:
logger.update_from_kwargs(
kwargs=_redact(kwargs, secret_fields),
model=model,
optional_params=_redact(optional_params, secret_fields),
litellm_params={ # mutable-ok: update_from_kwargs takes dict
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": url,
**{
name: kwargs[name] for name in ("logger_fn", "litellm_request_debug") if name in kwargs
}, # mutable-ok: splat into the dict above
**{ # mutable-ok: splat into the dict above
name: kwargs[name]
for name in CustomPricingLiteLLMParams.model_fields
if name in kwargs and kwargs[name] is not None
},
},
custom_llm_provider=custom_llm_provider,
)
class RustOcr(Protocol):

View file

@ -0,0 +1,288 @@
from __future__ import annotations
import datetime
from collections.abc import Callable, Mapping, MutableSequence, Sequence
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Final,
Literal,
Protocol,
TypeAlias,
cast, # noqa: TID251 # narrows legacy untyped registries at the boundary
)
from litellm.integrations.custom_logger import CustomLogger
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
CallbackTarget: TypeAlias = str | Callable[..., object] | CustomLogger
RegistryName: TypeAlias = Literal[
"input", "async_input", "success", "async_success", "failure", "async_failure", "callbacks"
]
NamedEvent: TypeAlias = Literal["success", "failure"]
Kwargs: TypeAlias = dict[str, object] # mutable-ok: the retained call kwargs are the dict callbacks receive and mutate
_REGISTRY_ATTRIBUTES: Final[Mapping[RegistryName, str]] = MappingProxyType(
{
"input": "input_callback",
"async_input": "_async_input_callback",
"success": "success_callback",
"async_success": "_async_success_callback",
"failure": "failure_callback",
"async_failure": "_async_failure_callback",
"callbacks": "callbacks",
}
)
class _CallbackManager(Protocol):
def add_litellm_success_callback(self, callback: CallbackTarget) -> None: ...
def add_litellm_failure_callback(self, callback: CallbackTarget) -> None: ...
def add_litellm_async_success_callback(self, callback: CallbackTarget) -> None: ...
def add_litellm_async_failure_callback(self, callback: CallbackTarget) -> None: ...
class _LoggingFactory(Protocol):
def __call__(
self,
*,
model: str | None,
messages: object,
stream: bool,
litellm_call_id: str,
litellm_trace_id: str | None,
function_id: str,
call_type: str,
start_time: datetime.datetime,
dynamic_success_callbacks: Sequence[CallbackTarget] | None,
dynamic_failure_callbacks: Sequence[CallbackTarget] | None,
dynamic_async_success_callbacks: Sequence[CallbackTarget] | None,
dynamic_async_failure_callbacks: Sequence[CallbackTarget] | None,
kwargs: Kwargs,
applied_guardrails: Sequence[str],
supports_correlation_logging: bool,
) -> Logging: ...
class _EnvironmentUpdater(Protocol):
def __call__(
self,
*,
model: str | None,
user: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream_options: object,
) -> None: ...
def _legacy(value: object) -> Callable[..., object]:
return cast(Callable[..., object], value) # cast-ok: legacy module-level helpers are untyped
def _mapping(value: object) -> Mapping[str, object] | None:
if not isinstance(value, Mapping):
return None
return cast(Mapping[str, object], value) # cast-ok: isinstance narrows only to Mapping[Unknown, Unknown]
def registry(
name: RegistryName,
) -> MutableSequence[CallbackTarget]: # mutable-ok: the public litellm registries are mutated by contract
import litellm
return cast( # cast-ok: legacy module-level lists are untyped
MutableSequence[CallbackTarget], getattr(litellm, _REGISTRY_ATTRIBUTES[name])
)
def is_async_callable(callback: object) -> bool:
from litellm.litellm_core_utils.cached_imports import get_coroutine_checker
return get_coroutine_checker().is_async_callable(callback)
def is_known_name(callback: str) -> bool:
import litellm
known: Final = cast(Sequence[str], litellm._known_custom_logger_compatible_callbacks) # pyright: ignore[reportPrivateUsage] # cast-ok: registry list has no public typed accessor
return callback in known
def resolve_named_integration(callback: str) -> CustomLogger | None:
from litellm.litellm_core_utils import litellm_logging
factory: Final = _legacy(litellm_logging._init_custom_logger_compatible_class) # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # legacy factory
resolved: Final = factory(callback, internal_usage_cache=None, llm_router=None)
return resolved if isinstance(resolved, CustomLogger) else None
def async_success_registry_has_type(callback: object) -> bool:
return any(type(existing) is type(callback) for existing in registry("async_success"))
def bootstrap_pending() -> bool:
from litellm import utils
return not utils.callback_list
def bootstrap(function_id: str | None) -> None:
from litellm import utils
from litellm.litellm_core_utils import cached_imports
combined: Final = _bootstrap_list()
utils.callback_list = combined # rebind-ok: legacy module global consumed by set_callbacks
set_callbacks: Final = _legacy(cached_imports.get_set_callbacks()) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # cached import is untyped
set_callbacks(callback_list=combined, function_id=function_id)
def _bootstrap_list() -> list[str]: # mutable-ok: set_callbacks and the legacy module global expect a list
names: Final = frozenset({*registry("input"), *registry("success"), *registry("failure")})
consumed: Final = cast(Sequence[str], names) # cast-ok: legacy list annotation is narrower than its contents
return list(consumed) # mutable-ok: consumed by set_callbacks
def expand_named(callback: str, event: NamedEvent) -> None:
from litellm import utils
utils._add_custom_logger_callback_to_specific_event(callback, event) # pyright: ignore[reportPrivateUsage] # legacy expansion helper
def append_registry(name: RegistryName, callback: CallbackTarget) -> None:
import litellm
manager: Final = cast(_CallbackManager, litellm.logging_callback_manager) # cast-ok: legacy manager is untyped
match name:
case "input" | "async_input":
registry(name).append(callback)
case "success":
manager.add_litellm_success_callback(callback)
case "async_success":
manager.add_litellm_async_success_callback(callback)
case "failure":
manager.add_litellm_failure_callback(callback)
case "async_failure":
manager.add_litellm_async_failure_callback(callback)
case "callbacks":
raise KeyError(name)
def remove_registry(name: RegistryName, callback: CallbackTarget) -> None:
target: Final = registry(name)
for index in range(len(target) - 1, -1, -1):
if target[index] is callback or target[index] == callback:
del target[index]
return
def logger_fn(callback: object) -> None:
from litellm import utils
utils.user_logger_fn = callback # rebind-ok: legacy module global read by pre_call
def breadcrumb(kwargs: Mapping[str, object]) -> None:
from litellm import utils
if utils.add_breadcrumb is None:
return
import litellm
from litellm.litellm_core_utils import core_helpers
deep_copy: Final = _legacy(core_helpers.safe_deep_copy) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # legacy helper
copied: Final = _copied(deep_copy, kwargs)
hidden: Final = frozenset(("messages", "input", "prompt")) if litellm.turn_off_message_logging else frozenset[str]()
details: Final = MappingProxyType({key: value for key, value in copied.items() if key not in hidden})
_legacy(utils.add_breadcrumb)(category="litellm.llm_call", message=f"Keyword Args: {details}", level="info")
def _copied(deep_copy: Callable[..., object], kwargs: Mapping[str, object]) -> Mapping[str, object]:
try:
copied: Final = deep_copy(dict(kwargs)) # mutable-ok: safe_deep_copy expects a dict
except Exception: # noqa: BLE001 # legacy breadcrumb falls back to the live mapping
return kwargs
return _mapping(copied) or kwargs
def prepare_environment() -> None:
from litellm import utils
utils.custom_llm_setup()
def applied_guardrails(kwargs: Mapping[str, object]) -> Sequence[str]:
from litellm.utils import get_applied_guardrails
return tuple(get_applied_guardrails(dict(kwargs))) # mutable-ok: legacy helper expects a dict
def _litellm_params(kwargs: Mapping[str, object]) -> Mapping[str, object]:
metadata: Final = kwargs.get("metadata")
litellm_metadata: Final = kwargs.get("litellm_metadata")
base: Final = MappingProxyType({"api_base": ""})
with_metadata: Final = MappingProxyType({**base, "metadata": metadata}) if "metadata" in kwargs else base
typed_metadata: Final = _mapping(litellm_metadata)
if typed_metadata is None:
return with_metadata
with_litellm_metadata: Final = MappingProxyType({**with_metadata, "litellm_metadata": typed_metadata})
if metadata:
return with_litellm_metadata
copied_metadata: Final = dict(typed_metadata) # mutable-ok: callbacks may mutate this copy
return MappingProxyType({**with_litellm_metadata, "metadata": copied_metadata})
def _owned(
callbacks: Sequence[CallbackTarget] | None,
) -> list[CallbackTarget] | None: # mutable-ok: Logging appends to these lists
return list(callbacks) if callbacks is not None else None # mutable-ok: Logging appends to these lists
def build_logging(
*,
call_type: str,
model: str | None,
kwargs: Kwargs,
start_time: datetime.datetime,
asynchronous: bool,
dynamic_success: Sequence[CallbackTarget] | None,
dynamic_async_success: Sequence[CallbackTarget] | None,
dynamic_failure: Sequence[CallbackTarget] | None,
guardrails: Sequence[str],
) -> Logging:
from litellm.litellm_core_utils.cached_imports import get_litellm_logging_class
function_id: Final = kwargs.get("id")
trace_id: Final = kwargs.get("litellm_trace_id")
factory: Final = cast(_LoggingFactory, get_litellm_logging_class()) # cast-ok: legacy constructor is untyped
logger: Final = factory(
model=model,
messages="default-message-value",
stream=False,
litellm_call_id=str(kwargs["litellm_call_id"]),
litellm_trace_id=trace_id if isinstance(trace_id, str) else None,
function_id=function_id if isinstance(function_id, str) else "",
call_type=call_type,
start_time=start_time,
dynamic_success_callbacks=_owned(dynamic_success),
dynamic_failure_callbacks=_owned(dynamic_failure),
dynamic_async_success_callbacks=_owned(dynamic_async_success),
dynamic_async_failure_callbacks=None,
kwargs=kwargs,
applied_guardrails=list(guardrails), # mutable-ok: legacy constructor annotation is list
supports_correlation_logging=asynchronous,
)
update: Final = cast(_EnvironmentUpdater, logger.update_environment_variables) # cast-ok: legacy method is untyped
update(
model=model,
user="",
optional_params=MappingProxyType({}),
litellm_params=_litellm_params(kwargs),
stream_options=kwargs.get("stream_options"),
)
return logger

View file

@ -461,10 +461,16 @@ def test_native_pending_logging_is_released_only_for_ocr(call_type: str, excepti
)
if call_type in ("ocr", "aocr"):
pending.release.assert_called_once_with(not exception_raised)
if exception_raised:
pending.close.assert_called_once_with()
pending.assert_not_called()
else:
pending.assert_called_once_with()
pending.close.assert_not_called()
assert logger._native_pending_logging is None
else:
pending.release.assert_not_called()
pending.assert_not_called()
pending.close.assert_not_called()
assert logger._native_pending_logging is pending
if exception_raised:
enqueue.assert_not_called()

View file

@ -33,3 +33,17 @@ def test_binding_validates_native_attribute(
binding: Final = bindings.NativeBinding("route", validate=lambda item: item if isinstance(item, int) else None)
assert binding.load() == expected
@pytest.mark.parametrize("available", [False, True])
def test_decline_accessor_only_catches_admission_declines(monkeypatch: pytest.MonkeyPatch, available: bool) -> None:
class Declined(Exception):
pass
class Upstream(Exception):
pass
native: Final = SimpleNamespace(RustBridgeDeclined=Declined, RustUpstreamError=Upstream) if available else None
monkeypatch.setattr(bindings, "get_native_bridge", lambda: native)
assert bindings.native_decline_types() == ((Declined,) if available else ())
assert not isinstance(Upstream("already dispatched"), bindings.native_decline_types())

View file

@ -0,0 +1,282 @@
import asyncio
import datetime
import inspect
from collections.abc import Iterator
from typing import Final
from unittest.mock import Mock
import pytest
from litellm.integrations.athina import AthinaLogger
from litellm.integrations.dynamodb import DyanmoDBLogger
from litellm.integrations.greenscale import GreenscaleLogger
from litellm.integrations.helicone import HeliconeLogger
from litellm.integrations.langfuse.langfuse import LangFuseLogger
from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger
from litellm.integrations.lunary import LunaryLogger
from litellm.integrations.openmeter import OpenMeterLogger
from litellm.integrations.prompt_layer import PromptLayerLogger
from litellm.integrations.s3 import S3Logger
from litellm.integrations.supabase import Supabase
from litellm.integrations.traceloop import TraceloopLogger
from litellm.integrations.weights_biases import WeightsBiasesLogger
from litellm.litellm_core_utils import litellm_logging
from litellm.rust_bridge import leaves
START: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
END: Final = START + datetime.timedelta(seconds=1)
DETAILS: Final[dict[str, object]] = {
"litellm_call_id": "call-id",
"input": "document",
"user": "user-1",
"original_response": "raw",
"litellm_params": {},
}
def _logger() -> Mock:
return Mock(
model="mistral/mistral-ocr-latest",
messages="document",
litellm_call_id="call-id",
model_call_details=dict(DETAILS),
standard_callback_dynamic_params={},
)
def _stub(cls: type, method: str) -> tuple[object, Mock]:
instance: Final[object] = object.__new__(cls)
recorder: Final = Mock(name=f"{cls.__name__}.{method}")
setattr(instance, method, recorder)
return instance, recorder
def _assert_binds(cls: type, method: str, recorder: Mock, instance: object) -> None:
recorder.assert_called_once()
target: Final[object] = getattr(cls, method)
assert callable(target)
signature: Final = inspect.signature(target)
bound: Final = signature.bind(instance, *recorder.call_args.args, **recorder.call_args.kwargs)
required: Final = frozenset(
name for name, parameter in signature.parameters.items() if parameter.default is inspect.Parameter.empty
)
assert required <= set(bound.arguments)
SUCCESS_CASES: Final[tuple[tuple[str, str, type, str], ...]] = (
("promptlayer", "promptLayerLogger", PromptLayerLogger, "log_event"),
("wandb", "weightsBiasesLogger", WeightsBiasesLogger, "log_event"),
("athina", "athinaLogger", AthinaLogger, "log_event"),
("logfire", "logfireLogger", LogfireLogger, "log_event"),
("greenscale", "greenscaleLogger", GreenscaleLogger, "log_event"),
("supabase", "supabaseClient", Supabase, "log_event"),
("lunary", "lunaryLogger", LunaryLogger, "log_event"),
("helicone", "heliconeLogger", HeliconeLogger, "log_success"),
("traceloop", "traceloopLogger", TraceloopLogger, "log_event"),
("s3", "s3Logger", S3Logger, "log_event"),
)
FAILURE_CASES: Final[tuple[tuple[str, str, type, str], ...]] = (
("lunary", "lunaryLogger", LunaryLogger, "log_event"),
("supabase", "supabaseClient", Supabase, "log_event"),
("traceloop", "traceloopLogger", TraceloopLogger, "log_event"),
("logfire", "logfireLogger", LogfireLogger, "log_event"),
)
def _install(monkeypatch: pytest.MonkeyPatch, global_name: str, value: object) -> None:
monkeypatch.setattr(litellm_logging, global_name, value, raising=False)
@pytest.mark.parametrize(("name", "global_name", "cls", "method"), SUCCESS_CASES)
def test_named_success_calls_integration_with_its_real_signature(
monkeypatch: pytest.MonkeyPatch, name: str, global_name: str, cls: type, method: str
) -> None:
instance, recorder = _stub(cls, method)
_install(monkeypatch, global_name, instance)
response: Final = object()
assert leaves.dispatch_named_success(_logger(), name, response, START, END) is None
_assert_binds(cls, method, recorder, instance)
kwargs: Final = recorder.call_args.kwargs
assert kwargs["start_time"] is START and kwargs["end_time"] is END
assert kwargs["response_obj"] is response
assert callable(kwargs["print_verbose"])
@pytest.mark.parametrize(("name", "global_name", "cls", "method"), FAILURE_CASES)
def test_named_failure_calls_integration_with_its_real_signature(
monkeypatch: pytest.MonkeyPatch, name: str, global_name: str, cls: type, method: str
) -> None:
instance, recorder = _stub(cls, method)
_install(monkeypatch, global_name, instance)
error: Final = RuntimeError("boom")
leaves.dispatch_named_failure(_logger(), name, error, "formatted", START, END)
_assert_binds(cls, method, recorder, instance)
kwargs: Final = recorder.call_args.kwargs
assert kwargs["start_time"] is START and kwargs["end_time"] is END
assert kwargs.get("response_obj") is None
def test_logfire_receives_enum_levels_and_exception_on_failure(monkeypatch: pytest.MonkeyPatch) -> None:
instance, recorder = _stub(LogfireLogger, "log_event")
_install(monkeypatch, "logfireLogger", instance)
error: Final = RuntimeError("boom")
leaves.dispatch_named_success(_logger(), "logfire", object(), START, END)
assert recorder.call_args.kwargs["level"] is LogfireLevel.INFO
assert "original_response" not in recorder.call_args.kwargs["kwargs"]
leaves.dispatch_named_failure(_logger(), "logfire", error, "formatted", START, END)
assert recorder.call_args.kwargs["level"] is LogfireLevel.ERROR
assert recorder.call_args.kwargs["kwargs"]["exception"] is error
assert "original_response" not in recorder.call_args.kwargs["kwargs"]
def test_lunary_marks_success_and_error_events(monkeypatch: pytest.MonkeyPatch) -> None:
instance, recorder = _stub(LunaryLogger, "log_event")
_install(monkeypatch, "lunaryLogger", instance)
leaves.dispatch_named_success(_logger(), "lunary", object(), START, END)
assert recorder.call_args.kwargs["event"] == "end"
assert recorder.call_args.kwargs["run_id"] == "call-id"
leaves.dispatch_named_failure(_logger(), "lunary", RuntimeError("boom"), "formatted", START, END)
assert recorder.call_args.kwargs["event"] == "error"
assert recorder.call_args.kwargs["error"] == "formatted"
def test_supabase_request_hook_only_fires_pre_call(monkeypatch: pytest.MonkeyPatch) -> None:
instance, recorder = _stub(Supabase, "input_log_event")
_install(monkeypatch, "supabaseClient", instance)
leaves.dispatch_named_request(_logger(), "supabase", "post_api_call")
recorder.assert_not_called()
leaves.dispatch_named_request(_logger(), "supabase", "pre_api_call")
_assert_binds(Supabase, "input_log_event", recorder, instance)
assert recorder.call_args.kwargs["end_user"] == "user-1"
def test_sentry_hooks_are_called_when_configured(monkeypatch: pytest.MonkeyPatch) -> None:
breadcrumb: Final = Mock()
capture: Final = Mock()
_install(monkeypatch, "add_breadcrumb", breadcrumb)
_install(monkeypatch, "capture_exception", capture)
error: Final = RuntimeError("boom")
leaves.dispatch_named_request(_logger(), "sentry", "pre_api_call")
assert breadcrumb.call_args.kwargs["category"] == "litellm.llm_call"
leaves.dispatch_named_failure(_logger(), "sentry", error, "formatted", START, END)
capture.assert_called_once_with(error)
@pytest.mark.parametrize(("name", "global_name"), [(case[0], case[1]) for case in SUCCESS_CASES])
def test_unconfigured_or_foreign_singleton_is_skipped(
monkeypatch: pytest.MonkeyPatch, name: str, global_name: str
) -> None:
_install(monkeypatch, global_name, None)
assert leaves.dispatch_named_success(_logger(), name, object(), START, END) is None
foreign: Final = Mock()
_install(monkeypatch, global_name, foreign)
assert leaves.dispatch_named_success(_logger(), name, object(), START, END) is None
assert not foreign.method_calls
def _run(awaitable: object) -> None:
async def consume() -> None:
assert inspect.isawaitable(awaitable)
_ = await awaitable
asyncio.run(consume())
def test_openmeter_returns_the_async_coroutine(monkeypatch: pytest.MonkeyPatch) -> None:
instance: Final = object.__new__(OpenMeterLogger)
recorder: Final = Mock()
async def async_log_success_event(**kwargs: object) -> None:
recorder(**kwargs)
monkeypatch.setattr(instance, "async_log_success_event", async_log_success_event)
_install(monkeypatch, "openMeterLogger", instance)
response: Final = object()
_run(leaves.dispatch_named_success(_logger(), "openmeter", response, START, END))
_assert_binds(OpenMeterLogger, "async_log_success_event", recorder, instance)
assert recorder.call_args.kwargs["response_obj"] is response
def test_dynamodb_reuses_the_existing_singleton(monkeypatch: pytest.MonkeyPatch) -> None:
instance: Final = object.__new__(DyanmoDBLogger)
recorder: Final = Mock()
async def _async_log_event(**kwargs: object) -> None:
recorder(**kwargs)
monkeypatch.setattr(instance, "_async_log_event", _async_log_event)
_install(monkeypatch, "dynamoLogger", instance)
_run(leaves.dispatch_named_success(_logger(), "dynamodb", object(), START, END))
_assert_binds(DyanmoDBLogger, "_async_log_event", recorder, instance)
assert litellm_logging.dynamoLogger is instance
@pytest.fixture
def langfuse_handler(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[LangFuseLogger, Mock, Mock]]:
handler: Final = object.__new__(LangFuseLogger)
recorder: Final = Mock(return_value={"trace_id": "trace-1"})
monkeypatch.setattr(handler, "log_event_on_langfuse", recorder)
_install(monkeypatch, "langFuseLogger", handler)
cache: Final = Mock()
_install(monkeypatch, "in_memory_trace_id_cache", cache)
yield handler, recorder, cache
def test_langfuse_success_logs_and_caches_trace_id(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None:
handler, recorder, cache = langfuse_handler
response: Final = object()
leaves.dispatch_named_success(_logger(), "langfuse", response, START, END)
_assert_binds(LangFuseLogger, "log_event_on_langfuse", recorder, handler)
kwargs: Final = recorder.call_args.kwargs
assert kwargs["response_obj"] is response
assert kwargs["user_id"] == "user-1"
assert kwargs["level"] == "DEFAULT" and kwargs["status_message"] is None
assert "original_response" not in kwargs["kwargs"]
cache.set_cache.assert_called_once_with(litellm_call_id="call-id", service_name="langfuse", trace_id="trace-1")
def test_langfuse_failure_logs_error_level(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None:
handler, recorder, _ = langfuse_handler
leaves.dispatch_named_failure(_logger(), "langfuse", RuntimeError("boom"), "formatted", START, END)
_assert_binds(LangFuseLogger, "log_event_on_langfuse", recorder, handler)
kwargs: Final = recorder.call_args.kwargs
assert kwargs["response_obj"] is None
assert kwargs["level"] == "ERROR" and kwargs["status_message"] == "boom"
def test_langfuse_skips_cache_without_trace_id(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None:
_, recorder, cache = langfuse_handler
recorder.return_value = {}
leaves.dispatch_named_success(_logger(), "langfuse", object(), START, END)
cache.set_cache.assert_not_called()
def test_signature_binding_rejects_drift() -> None:
instance, recorder = _stub(Supabase, "log_event")
recorder(model="m", end_userr="typo", response_obj=None, start_time=START, end_time=END)
with pytest.raises(TypeError):
_assert_binds(Supabase, "log_event", recorder, instance)

View file

@ -8,7 +8,8 @@ import litellm
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.ocr import legacy
from litellm.rust_bridge import bindings, configuration
from litellm.rust_bridge.ocr import NATIVE_AOCR, NATIVE_OCR
from litellm.rust_bridge.leaves import record_post_call, record_pre_call
from litellm.rust_bridge.ocr import NATIVE_AOCR, NATIVE_OCR, update_logging
@pytest.fixture(autouse=True)
@ -21,6 +22,73 @@ def isolated_ocr_configuration(monkeypatch: pytest.MonkeyPatch) -> Generator[Non
configuration.reset_rust_configuration()
def test_logging_redacts_views_and_preserves_opaque_arguments_and_pricing() -> None:
logger: Final = Mock()
opaque: Final = object()
logger_fn: Final = object()
kwargs: Final = {
"vertex_credentials": "secret",
"proxy_server_request": opaque,
"metadata": opaque,
"logger_fn": logger_fn,
"litellm_request_debug": False,
"litellm_call_id": "call-id",
"input_cost_per_token": 0,
"output_cost_per_token": None,
}
optional: Final = {"vertex_credentials": "secret", "pages": [1], "proxy_server_request": opaque}
update_logging(logger, kwargs, "model", "vertex_ai", optional, ("vertex_credentials",), "https://provider")
logger.update_from_kwargs.assert_called_once_with(
kwargs={
"vertex_credentials": "****",
"metadata": opaque,
"logger_fn": logger_fn,
"litellm_request_debug": False,
"litellm_call_id": "call-id",
"input_cost_per_token": 0,
"output_cost_per_token": None,
},
model="model",
custom_llm_provider="vertex_ai",
optional_params={"vertex_credentials": "****", "pages": [1]},
litellm_params={
"litellm_call_id": "call-id",
"api_base": "https://provider",
"logger_fn": logger_fn,
"litellm_request_debug": False,
"input_cost_per_token": 0,
},
)
assert kwargs["vertex_credentials"] == optional["vertex_credentials"] == "secret"
assert kwargs["proxy_server_request"] is optional["proxy_server_request"] is opaque
assert logger.update_from_kwargs.call_args.kwargs["kwargs"]["metadata"] is opaque
assert logger.update_from_kwargs.call_args.kwargs["optional_params"]["pages"] is optional["pages"]
def test_logging_callbacks_receive_captured_payload_roots_and_propagate_errors() -> None:
logger: Final = Mock(log_raw_request_response=False, logger_fn=None, model_call_details={})
body: Final[dict[str, object]] = {"document": "original"}
headers: Final = {"authorization": "key"}
response: Final = object()
record_pre_call(logger, api_key="key", body=body, headers=headers, url="https://provider")
record_post_call(logger, original_response=response, body=body, headers=headers)
logger._pre_call.assert_called_once_with(
input="OCR document processing",
api_key="key",
model=None,
additional_args={"complete_input_dict": body, "headers": headers, "api_base": "https://provider"},
)
for callback in (logger._pre_call, logger.record_post_call):
assert callback.call_args.kwargs["additional_args"]["complete_input_dict"] is body
assert callback.call_args.kwargs["additional_args"]["headers"] is headers
assert logger.record_post_call.call_args.kwargs["original_response"] is response
failure: Final = RuntimeError("callback failed")
failing_logger: Final = Mock(_pre_call=Mock(side_effect=failure))
with pytest.raises(RuntimeError) as caught:
record_pre_call(failing_logger, api_key=None, body=body, headers=headers, url="https://provider")
assert caught.value is failure
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_unavailable_native_uses_legacy(monkeypatch: pytest.MonkeyPatch, asynchronous: bool) -> None:

View file

@ -0,0 +1,169 @@
import datetime
from collections.abc import Iterator
from typing import Final
import pytest
import litellm
from litellm import utils
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.rust_bridge.loader import native_bridge_available
pytestmark = pytest.mark.skipif(not native_bridge_available(), reason="requires the Rust extension")
REGISTRIES: Final = (
"input_callback",
"_async_input_callback",
"success_callback",
"_async_success_callback",
"failure_callback",
"_async_failure_callback",
"callbacks",
)
@pytest.fixture
def clean_registries(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
for name in REGISTRIES:
monkeypatch.setattr(litellm, name, [])
monkeypatch.setattr(utils, "callback_list", [])
monkeypatch.setattr(utils, "user_logger_fn", None)
monkeypatch.setenv("OPENMETER_API_KEY", "test")
monkeypatch.setenv("OPENMETER_API_ENDPOINT", "http://127.0.0.1:9")
yield
class SyncLogger(CustomLogger):
pass
class AsyncOnly(CustomLogger):
pass
def sync_fn(*args: object, **kwargs: object) -> None:
del args, kwargs
async def async_fn(*args: object, **kwargs: object) -> None:
del args, kwargs
def snapshot() -> dict[str, list[object]]:
return {name: list(getattr(litellm, name)) for name in REGISTRIES}
def run_legacy(kwargs: dict[str, object]) -> tuple[Logging, dict[str, object]]:
logger, prepared = utils.function_setup(
"ocr",
utils.Rules(),
datetime.datetime.now(),
is_async_call=False,
**{"litellm_call_id": "legacy", **kwargs},
)
assert isinstance(logger, Logging)
return logger, prepared
def run_native(kwargs: dict[str, object]) -> tuple[Logging, dict[str, object]]:
from litellm.rust_bridge import _native
logger, prepared = _native._debug_setup(
"ocr", (), {"litellm_call_id": "native", **kwargs}, datetime.datetime.now(), False
)
assert isinstance(logger, Logging)
return logger, prepared
@pytest.mark.parametrize(
"globals_before, kwargs",
[
({}, {}),
({"callbacks": [SyncLogger()]}, {}),
({"callbacks": [async_fn]}, {}),
({}, {"callbacks": [SyncLogger(), sync_fn]}),
({"success_callback": [async_fn, "openmeter", sync_fn]}, {}),
({"failure_callback": [async_fn, sync_fn]}, {}),
({"input_callback": [async_fn, sync_fn]}, {}),
({}, {"success_callback": [sync_fn, async_fn, "s3", "dynamodb"], "failure_callback": [sync_fn]}),
({"callbacks": [SyncLogger()]}, {"callbacks": [SyncLogger()], "success_callback": [async_fn]}),
],
ids=[
"empty",
"global-custom-logger",
"global-async-callable",
"dynamic-callbacks",
"success-safety-net",
"failure-safety-net",
"input-safety-net",
"per-call-success-failure-split",
"mixed",
],
)
def test_native_setup_registry_side_effects_match_function_setup(
clean_registries: None, globals_before: dict[str, list[object]], kwargs: dict[str, object]
) -> None:
for name, values in globals_before.items():
getattr(litellm, name).extend(values)
legacy_logger, legacy_kwargs = run_legacy(
{key: list(value) if isinstance(value, list) else value for key, value in kwargs.items()}
)
legacy_snapshot: Final = snapshot()
legacy_bootstrap: Final = list(utils.callback_list or [])
for name in REGISTRIES:
getattr(litellm, name).clear()
utils.callback_list = []
for name, values in globals_before.items():
getattr(litellm, name).extend(values)
native_logger, native_kwargs = run_native(
{key: list(value) if isinstance(value, list) else value for key, value in kwargs.items()}
)
assert snapshot() == legacy_snapshot
assert sorted(map(repr, utils.callback_list or [])) == sorted(map(repr, legacy_bootstrap))
assert set(native_kwargs) - {"litellm_call_id"} == set(legacy_kwargs) - {"litellm_call_id"}
for attribute in (
"dynamic_success_callbacks",
"dynamic_async_success_callbacks",
"dynamic_failure_callbacks",
"dynamic_async_failure_callbacks",
"call_type",
"stream",
"model",
):
assert getattr(native_logger, attribute) == getattr(legacy_logger, attribute), attribute
def test_native_setup_honours_caller_supplied_logging_object(clean_registries: None) -> None:
class Supplied(Logging):
pass
supplied: Final = Supplied(
model="mistral/mistral-ocr-latest",
messages=[],
stream=False,
call_type="ocr",
start_time=datetime.datetime.now(),
litellm_call_id="supplied",
function_id="",
)
litellm.callbacks.append(SyncLogger())
logger, kwargs = run_native({"litellm_logging_obj": supplied, "callbacks": [SyncLogger()]})
assert logger is supplied
assert kwargs["callbacks"] is not None
assert litellm.success_callback == []
def test_native_setup_records_logger_fn_and_metadata(clean_registries: None) -> None:
def logger_fn(details: object) -> None:
del details
logger, kwargs = run_native(
{"model": "mistral/mistral-ocr-latest", "logger_fn": logger_fn, "metadata": {"source": "test"}}
)
assert utils.user_logger_fn is logger_fn
assert logger.litellm_params["metadata"] == {"source": "test"}
assert logger.model == "mistral/mistral-ocr-latest"
assert kwargs["metadata"] == {"source": "test"}

View file

@ -10,6 +10,7 @@ import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from tests.test_litellm_rust.support.callback_recorder import RecordingLogger
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
from tests.test_litellm_rust.support.requests import (
OCR_DOCUMENT,
OCR_RESPONSE,
@ -18,7 +19,6 @@ from tests.test_litellm_rust.support.requests import (
request_body,
request_headers,
)
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
pytestmark = pytest.mark.requires_rust_extension
@ -119,69 +119,6 @@ def test_native_ocr_pre_call_header_rebinding_does_not_replace_execution_root(oc
assert "x-rebound" not in ocr_server.requests[0].headers
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_native_ocr_pre_call_nested_document_edit_updates_caller_callback_and_provider_references(
ocr_server: RecordingServer, asynchronous: bool
) -> None:
original: Final = dict(OCR_DOCUMENT)
replacement_url: Final = "data:application/pdf;base64,ZGVm"
retained: Final = []
aliases: Final = []
class Retain(CustomLogger):
def log_pre_api_call(self, model, messages, kwargs):
aliases.append(request_body(kwargs)["document"] is original)
retained.append(request_body(kwargs)["document"])
class Edit(CustomLogger):
def log_pre_api_call(self, model, messages, kwargs):
original["document_url"] = replacement_url
arguments: Final = {
"model": "mistral/mistral-ocr-latest",
"document": original,
"api_key": "test-key",
"api_base": ocr_server.base_url,
"callbacks": [Retain(), Edit()],
}
response: Final = (
await call_native_aocr(ocr_server, **arguments)
if asynchronous
else call_native_ocr(ocr_server, **arguments)
)
assert aliases == [True]
assert retained[0]["document_url"] == replacement_url
assert original["document_url"] == replacement_url
assert ocr_server.requests[0].body["document"]["document_url"] == replacement_url
assert response.pages[0].markdown == "native OCR response"
def test_native_ocr_pre_call_document_replacement_does_not_mutate_original_document(
ocr_server: RecordingServer,
) -> None:
original: Final = dict(OCR_DOCUMENT)
replacement: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,ZGVm"}
retained: Final = []
class RetainAndReplace(CustomLogger):
def log_pre_api_call(self, model, messages, kwargs):
body = request_body(kwargs)
retained.append(body["document"])
body["document"] = replacement
call_native_ocr(
ocr_server,
document=original,
callbacks=[RetainAndReplace()],
)
assert retained[0] is original
assert original["document_url"] == OCR_DOCUMENT["document_url"]
assert ocr_server.requests[0].body["document"] == replacement
def test_native_ocr_pre_call_body_rebinding_is_visible_to_callbacks_but_not_provider(
ocr_server: RecordingServer,
) -> None:
@ -372,6 +309,7 @@ async def test_native_azure_ocr_resolves_token_before_pre_call_on_caller_context
asynchronous: bool,
) -> None:
from contextvars import ContextVar
context: Final = ContextVar("azure-token-context", default="missing")
context.set("caller")
caller_thread: Final = threading.current_thread()
@ -400,9 +338,7 @@ async def test_native_azure_ocr_resolves_token_before_pre_call_on_caller_context
"callbacks": [Edit()],
}
response: Final = (
await call_native_aocr(ocr_server, **arguments)
if asynchronous
else call_native_ocr(ocr_server, **arguments)
await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments)
)
assert response.pages[0].markdown == "native OCR response"
assert observations == ["token", "pre_call"]
@ -431,9 +367,7 @@ async def test_native_azure_ocr_token_provider_can_make_nested_native_ocr_call(
"azure_ad_token_provider": provider,
}
response: Final = (
await call_native_aocr(ocr_server, **arguments)
if asynchronous
else call_native_ocr(ocr_server, **arguments)
await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments)
)
assert response.pages[0].markdown == "native OCR response"
assert calls == ["token"]
@ -488,7 +422,9 @@ async def test_native_azure_ocr_releases_token_provider_after_terminal_outcome(
) -> None:
import gc
import weakref
from tests.test_litellm_rust.support.callback_recorder import drain_logging
class Provider:
def __call__(self) -> str:
if outcome == "failure":
@ -534,3 +470,274 @@ async def test_native_azure_ocr_releases_token_provider_after_terminal_outcome(
await asyncio.sleep(0)
gc.collect()
assert reference() is None
FORBIDDEN_ORCHESTRATION: Final = (
"pre_call",
"post_call",
"success_handler",
"async_success_handler",
"failure_handler",
"async_failure_handler",
"_success_handler_body",
"_async_success_handler_body",
"_failure_handler_body",
"_async_failure_handler_body",
"dispatch_success_handlers",
"dispatch_failure_handlers",
"handle_sync_success_callbacks_for_async_calls",
)
@pytest.fixture
def legacy_orchestration_disabled(monkeypatch: pytest.MonkeyPatch) -> list[str]:
from litellm import utils
from litellm.litellm_core_utils.litellm_logging import Logging
reached: Final[list[str]] = []
def forbid(name: str):
def method(self, *args, **kwargs):
reached.append(name)
raise AssertionError(f"legacy orchestration reached: {name}")
return method
for name in FORBIDDEN_ORCHESTRATION:
monkeypatch.setattr(Logging, name, forbid(name))
def forbid_utils(name: str):
async def hook(*args, **kwargs):
reached.append(name)
raise AssertionError(f"legacy orchestration reached: {name}")
def setup(*args, **kwargs):
reached.append(name)
raise AssertionError(f"legacy orchestration reached: {name}")
return setup if name == "function_setup" else hook
for name in (
"function_setup",
"async_pre_call_deployment_hook",
"async_post_call_success_deployment_hook",
"async_post_call_failure_deployment_hook",
):
monkeypatch.setattr(utils, name, forbid_utils(name))
return reached
class DeploymentRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.hooks: list[str] = []
async def async_pre_call_deployment_hook(self, kwargs, call_type):
self.hooks.append(f"pre:{call_type.value}")
return {**kwargs, "metadata": {**(kwargs.get("metadata") or {}), "deployment": "seen"}}
async def async_post_call_success_deployment_hook(self, request_data, response, call_type):
self.hooks.append(f"post:{request_data['metadata']['deployment']}")
return response
async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
self.hooks.append(f"failure:{type(exception).__name__}:{fallback_depth}")
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_native_ocr_success_runs_integrations_without_legacy_orchestration(
ocr_server: RecordingServer, legacy_orchestration_disabled: list[str], asynchronous: bool
) -> None:
recorder: Final = RecordingLogger()
deployment: Final = DeploymentRecorder()
litellm.callbacks.append(deployment)
arguments: Final = {"callbacks": [recorder]}
response: Final = (
await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments)
)
assert response.pages[0].markdown == "native OCR response"
success_event: Final = "async_log_success_event" if asynchronous else "log_success_event"
events: Final = await recorder.wait_for_async(success_event)
assert legacy_orchestration_disabled == []
assert deployment.hooks == (["pre:aocr", "post:seen"] if asynchronous else [])
assert recorder.names.count("log_pre_api_call") == 1
assert events[0].kwargs["standard_logging_object"]["status"] == "success"
assert events[0].kwargs["response_cost"] is not None
assert events[0].kwargs["litellm_params"]["api_base"] == f"{ocr_server.base_url}/v1/ocr"
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_native_ocr_failure_runs_integrations_without_legacy_orchestration(
ocr_server: RecordingServer, legacy_orchestration_disabled: list[str], asynchronous: bool
) -> None:
ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500))
recorder: Final = RecordingLogger()
deployment: Final = DeploymentRecorder()
litellm.callbacks.append(deployment)
arguments: Final = {"callbacks": [recorder]}
with pytest.raises(litellm.InternalServerError) as caught:
await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments)
assert legacy_orchestration_disabled == []
assert deployment.hooks == (["pre:aocr", "failure:InternalServerError:None"] if asynchronous else [])
failures: Final = tuple(event for event in recorder.events if event.name.endswith("log_failure_event"))
assert [event.name for event in failures] == (
["log_failure_event", "async_log_failure_event"] if asynchronous else ["log_failure_event"]
)
assert all(event.kwargs["exception"] is caught.value for event in failures)
assert all(event.kwargs["standard_logging_object"]["status"] == "failure" for event in failures)
assert "log_success_event" not in recorder.names
def test_native_ocr_success_hooks_all_run_before_any_success_dispatch(ocr_server: RecordingServer) -> None:
order: Final = []
finished: Final = threading.Event()
class Hooked(CustomLogger):
def __init__(self, name: str) -> None:
super().__init__()
self.name = name
def logging_hook(self, kwargs, result, call_type):
order.append(("hook", self.name))
kwargs[f"seen-by-{self.name}"] = True
return kwargs, result
def log_success_event(self, kwargs, response_obj, start_time, end_time):
order.append(("log", self.name, kwargs.get("seen-by-a"), kwargs.get("seen-by-b")))
if self.name == "b":
finished.set()
call_native_ocr_with_callbacks(ocr_server, [Hooked("a"), Hooked("b")])
assert finished.wait(10)
assert order == [("hook", "a"), ("hook", "b"), ("log", "a", True, True), ("log", "b", True, True)]
def test_native_ocr_hook_failure_is_contained_and_target_still_dispatches(ocr_server: RecordingServer) -> None:
order: Final = []
finished: Final = threading.Event()
class Broken(CustomLogger):
def logging_hook(self, kwargs, result, call_type):
raise RuntimeError("hook failed")
def log_success_event(self, kwargs, response_obj, start_time, end_time):
order.append("broken-log")
class Healthy(CustomLogger):
def logging_hook(self, kwargs, result, call_type):
order.append("healthy-hook")
return kwargs, result
def log_success_event(self, kwargs, response_obj, start_time, end_time):
order.append("healthy-log")
finished.set()
call_native_ocr_with_callbacks(ocr_server, [Broken(), Healthy()])
assert finished.wait(10)
assert order == ["healthy-hook", "broken-log", "healthy-log"]
@pytest.mark.asyncio
async def test_native_aocr_hook_replacement_of_result_reaches_success_dispatch(ocr_server: RecordingServer) -> None:
replacement: Final = object()
observed: Final = []
class Replace(CustomLogger):
async def async_logging_hook(self, kwargs, result, call_type):
return kwargs, replacement
class Observe(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
observed.append(response_obj)
recorder: Final = RecordingLogger()
await call_native_aocr_with_callbacks(ocr_server, [Replace(), Observe(), recorder])
await recorder.wait_for_async("async_log_success_event")
assert observed == [replacement]
@pytest.mark.asyncio
async def test_native_aocr_shared_logging_object_dispatches_success_once(ocr_server: RecordingServer) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
ocr_server.expected_requests = 2
recorder: Final = RecordingLogger()
litellm.callbacks.append(recorder)
logger: Final = Logging(
model="mistral-ocr-latest",
messages=[],
stream=False,
call_type="aocr",
start_time=__import__("datetime").datetime.now(),
litellm_call_id="shared",
function_id="shared",
)
logger.dynamic_async_success_callbacks = [recorder]
await call_native_aocr(ocr_server, litellm_logging_obj=logger)
await call_native_aocr(ocr_server, litellm_logging_obj=logger)
await recorder.wait_for_async("async_log_success_event")
from tests.test_litellm_rust.support.callback_recorder import drain_logging
await drain_logging()
assert logger.model_call_details["has_logged_async_success"] is True
assert recorder.names.count("async_log_success_event") == 1
def test_native_ocr_logging_preparation_failure_does_not_fail_request(
ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.litellm_core_utils import litellm_logging
def broken_payload(*args, **kwargs):
raise RuntimeError("payload unavailable")
monkeypatch.setattr(litellm_logging, "get_standard_logging_object_payload", broken_payload)
unraisable: Final = []
monkeypatch.setattr(__import__("sys"), "unraisablehook", lambda event: unraisable.append(event.exc_value))
recorder: Final = RecordingLogger()
response: Final = call_native_ocr_with_callbacks(ocr_server, [recorder])
events: Final = recorder.wait_for("log_success_event")
assert response.pages[0].markdown == "native OCR response"
assert len(events) == 1
assert events[0].thread is not threading.current_thread()
assert any(str(error) == "payload unavailable" for error in unraisable)
def test_native_ocr_writes_success_marker_and_honours_existing_marker(
ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.rust_bridge import setup as native_setup
ocr_server.expected_requests = 2
loggers: Final = []
original_build: Final = native_setup.build_logging
def build_logging(**kwargs):
logger = original_build(**kwargs)
if loggers:
logger.model_call_details["has_logged_sync_success"] = True
loggers.append(logger)
return logger
monkeypatch.setattr(native_setup, "build_logging", build_logging)
recorder: Final = RecordingLogger()
call_native_ocr_with_callbacks(ocr_server, [recorder])
recorder.wait_for("log_success_event")
assert loggers[0].model_call_details["has_logged_sync_success"] is True
call_native_ocr_with_callbacks(ocr_server, [recorder])
from litellm.litellm_core_utils.thread_pool_executor import executor
executor.submit(lambda: None).result(10)
assert recorder.names.count("log_success_event") == 1
assert recorder.names.count("logging_hook") == 1

View file

@ -79,7 +79,7 @@ async def test_native_bindings_run_callbacks_and_send_their_mutations(
class Observe(Logging):
def pre_call(self, input, api_key, additional_args):
body: Final = additional_args["complete_input_dict"]
assert body["pages"] is pages
assert body["pages"] == pages and body["pages"] is not pages
observed.append(self.model_call_details["litellm_params"]["metadata"]["marker"])
body["pages"].append(2)
additional_args["headers"]["x-callback"] = "native"
@ -98,6 +98,7 @@ async def test_native_bindings_run_callbacks_and_send_their_mutations(
assert response.pages[0].markdown == "native OCR response"
assert observed == [marker] and observed[0] is marker
assert pages == [0]
assert len(ocr_server.requests) == 1
assert ocr_server.requests[0].body["pages"] == [0, 2]
assert ocr_server.requests[0].headers["x-callback"] == "native"
@ -263,7 +264,6 @@ async def test_deployment_hook_replaces_complete_routing_request(ocr_server: Rec
assert response.pages[0].markdown == "native OCR response"
assert observed == [(replacement, "replacement-key")]
assert observed[0][0] is replacement
assert replacement == original
assert replacement is not original
assert original == {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}
@ -621,7 +621,7 @@ def test_sync_pre_call_can_make_nested_native_request(ocr_server: RecordingServe
@pytest.mark.asyncio
async def test_retained_argument_aliases_and_body_roots_survive_envelope_replacement(
async def test_callback_body_roots_survive_envelope_replacement(
ocr_server: RecordingServer,
) -> None:
pages: Final = [0]
@ -633,8 +633,8 @@ async def test_retained_argument_aliases_and_body_roots_survive_envelope_replace
def pre_call(self, input, api_key, additional_args):
body: Final = additional_args["complete_input_dict"]
headers: Final = additional_args["headers"]
observed.append((body["document"] is document, body["pages"] is pages))
pages.append(2)
observed.append((body["document"] == document, body["pages"] == pages))
body["pages"].append(2)
headers["x-retained"] = "yes"
additional_args["complete_input_dict"] = {"discarded": True}
additional_args["headers"] = {}
@ -672,6 +672,7 @@ async def test_retained_argument_aliases_and_body_roots_survive_envelope_replace
assert observed[0] == (False, False, True)
assert observed[1] == (True, True)
assert observed[3] == (True, True)
assert pages == [0]
assert ocr_server.requests[0].body["pages"] == [0, 2]
assert ocr_server.requests[0].headers["x-retained"] == "yes"
@ -933,72 +934,6 @@ async def test_response_limit_is_enforced_at_the_public_boundary(
assert "max_response_bytes" not in body
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("failure", [False, True])
async def test_empty_callbacks_keep_bookkeeping_without_optional_dispatch(
ocr_server: RecordingServer,
monkeypatch: pytest.MonkeyPatch,
asynchronous: bool,
failure: bool,
created_loggers: list[Logging],
) -> None:
from litellm import utils
from litellm.litellm_core_utils import litellm_logging, logging_worker
class DispatchProbe:
deployments = 0
submissions = 0
enqueues = 0
def deployment(self, *args: object, **kwargs: object) -> None:
self.deployments += 1
def submit(self, *args: object, **kwargs: object) -> None:
self.submissions += 1
def ensure_initialized_and_enqueue(self, coroutine: Coroutine[object, object, object]) -> None:
self.enqueues += 1
coroutine.close()
probe: Final = DispatchProbe()
for name in (
"async_pre_call_deployment_hook",
"async_post_call_success_deployment_hook",
"async_post_call_failure_deployment_hook",
):
monkeypatch.setattr(utils, name, probe.deployment)
monkeypatch.setattr(litellm_logging, "executor", probe)
monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", probe)
if failure:
ocr_server.enqueue(ResponseSpec(body={"message": "provider failed"}, status=500))
trace_id_var.set("callback-free-parent")
arguments: Final = {"litellm_trace_id": "callback-free-call", "litellm_call_id": "callback-free-id"}
if failure:
with pytest.raises(litellm.InternalServerError):
await call_aocr(ocr_server, **arguments) if asynchronous else call_ocr(ocr_server, **arguments)
else:
response: Final = (
await call_aocr(ocr_server, **arguments) if asynchronous else call_ocr(ocr_server, **arguments)
)
assert response.pages[0].markdown == "native OCR response"
assert response._hidden_params["litellm_call_id"] == "callback-free-id"
assert response._hidden_params["response_cost"] is not None
assert response._hidden_params["_response_ms"] > 0
assert trace_id_var.get() == "callback-free-parent"
assert probe.deployments == probe.submissions == probe.enqueues == 0
assert len(created_loggers) == 1
logger: Final = created_loggers[0]
assert not hasattr(logger, "_native_pending_logging")
assert logger.model_call_details["first_api_call_start_time"] <= logger.model_call_details["end_time"]
assert "standard_logging_object" not in logger.model_call_details
assert (
"original_response" not in logger.model_call_details or logger.model_call_details["original_response"] is None
)
assert "complete_input_dict" not in logger.model_call_details.get("additional_args", {})
assert logger.model_call_details["response_cost"] == (0 if failure else response._hidden_params["response_cost"])
@pytest.mark.asyncio
@pytest.mark.parametrize(
"registration", ["success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"]
@ -1032,26 +967,18 @@ async def test_terminal_registration_added_during_http_is_observed(
@pytest.fixture
def created_loggers(monkeypatch: pytest.MonkeyPatch) -> list[Logging]:
from litellm import utils
from litellm.rust_bridge import setup as native_setup
original_setup: Final = utils.function_setup
original_build: Final = native_setup.build_logging
loggers: Final[list[Logging]] = []
def setup(
call_type: str,
rules: utils.Rules,
start: datetime.datetime,
*args: object,
is_async_call: bool = True,
**kwargs: object,
) -> tuple[Logging, dict[str, object]]:
logger, prepared = original_setup(call_type, rules, start, *args, is_async_call=is_async_call, **kwargs)
assert isinstance(logger, Logging)
def build_logging(**kwargs: object) -> Logging:
logger: Final = original_build(**kwargs) # pyright: ignore[reportArgumentType] # passthrough of the factory signature
setattr(logger, "_defer_async_logging", True)
loggers.append(logger)
return logger, prepared
return logger
monkeypatch.setattr(utils, "function_setup", setup)
monkeypatch.setattr(native_setup, "build_logging", build_logging)
return loggers
@ -1076,30 +1003,3 @@ async def test_explicit_logging_consumers_keep_request_and_response_payloads(
assert details["raw_request_typed_dict"]["raw_request_body"]["model"] == "mistral-ocr-latest"
if consumer == "logger_fn":
assert [item["log_event_type"] for item in snapshots] == ["pre_api_call", "post_api_call"]
@pytest.mark.asyncio
async def test_registration_removed_before_deferred_release_skips_queue(
ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch, created_loggers: list[Logging]
) -> None:
from litellm.litellm_core_utils import logging_worker
class QueueProbe:
enqueues = 0
def ensure_initialized_and_enqueue(self, coroutine: Coroutine[object, object, object]) -> None:
self.enqueues += 1
coroutine.close()
observer: Final = RecordingLogger()
litellm._async_success_callback.append(observer)
await call_aocr(ocr_server)
logger: Final = created_loggers[0]
assert hasattr(logger, "_native_pending_logging")
litellm._async_success_callback.clear()
probe: Final = QueueProbe()
monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", probe)
ProxyBaseLLMRequestProcessing._flush_deferred_async_logging(logger, False)
assert probe.enqueues == 0
assert not observer.names
assert logger.model_call_details["response_cost"] is not None

View file

@ -12,6 +12,7 @@ from tests.test_litellm_rust.support.requests import (
OCR_RESPONSE,
call_native_aocr,
call_native_ocr,
request_body,
)
pytestmark = pytest.mark.requires_rust_extension
@ -535,7 +536,7 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen
class Observer(RecordingLogger):
def log_pre_api_call(self, model, messages, kwargs):
super().log_pre_api_call(model, messages, kwargs)
pages.append(2)
request_body(kwargs)["pages"].append(2)
arguments: Final = {
"model": "mistral/mistral-ocr-latest",
@ -548,3 +549,4 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen
assert response.pages[0].markdown == "native OCR response"
assert ocr_server.requests[0].headers["authorization"] == f"Bearer {expected_key}"
assert ocr_server.requests[0].body["pages"] == [0, 2]
assert pages == [0]

View file

@ -1,14 +1,15 @@
import json
import asyncio
import base64
import gc
import json
import os
import threading
import weakref
import gc
from pathlib import Path
from collections.abc import Generator
from collections.abc import Coroutine, Generator
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from io import BytesIO
from pathlib import Path
from typing import Final, Protocol
import httpx
@ -340,7 +341,12 @@ def test_native_lifecycle_core_encodes_python_file_input(
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("source", ["path", "pathlike", "reader", "text", "bytes"])
@pytest.mark.asyncio
async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, asynchronous, source):
async def test_native_file_sources_preserve_sdk_behavior(
ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]],
tmp_path: Path,
asynchronous: bool,
source: str,
) -> None:
server, requests = ocr_server
path: Final = tmp_path / "scan.png"
content: Final = b"document bytes"
@ -388,9 +394,12 @@ async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, a
"api_base": f"http://127.0.0.1:{server.server_port}",
}
response: Final = await litellm.aocr(**kwargs) if asynchronous else litellm.ocr(**kwargs)
assert isinstance(response, OCRResponse)
assert response.pages[0].markdown == "native OCR response"
assert len(requests) == 1
assert requests[0]["body"]["document"] == {
body: Final = requests[0]["body"]
assert isinstance(body, dict)
assert body["document"] == {
"type": "image_url",
"image_url": "data:image/png;base64," + base64.b64encode(content).decode(),
}
@ -400,7 +409,11 @@ async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, a
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.asyncio
async def test_native_file_failures_preserve_identity_and_do_not_send(ocr_server, tmp_path, asynchronous):
async def test_native_file_failures_preserve_identity_and_do_not_send(
ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]],
tmp_path: Path,
asynchronous: bool,
) -> None:
from litellm.rust_bridge import _native
server, requests = ocr_server
@ -416,37 +429,47 @@ async def test_native_file_failures_preserve_identity_and_do_not_send(ocr_server
reader: Final = Reader()
path: Final = tmp_path / "missing.pdf"
kwargs: Final = {
"model": "mistral/mistral-ocr-latest",
"api_key": "test-key",
"api_base": f"http://127.0.0.1:{server.server_port}",
}
async def invoke(source: Reader | Path) -> None:
if asynchronous:
await _native.aocr(
model="mistral/mistral-ocr-latest",
document={"type": "file", "file": source},
api_key="test-key",
api_base=f"http://127.0.0.1:{server.server_port}",
)
else:
_native.ocr(
model="mistral/mistral-ocr-latest",
document={"type": "file", "file": source},
api_key="test-key",
api_base=f"http://127.0.0.1:{server.server_port}",
)
for source, error in ((reader, KeyError), (path, FileNotFoundError)):
with pytest.raises(error) as caught:
if asynchronous:
await _native.aocr(document={"type": "file", "file": source}, **kwargs)
else:
_native.ocr(document={"type": "file", "file": source}, **kwargs)
await invoke(source)
if source is reader:
assert caught.value is failure
else:
assert caught.value.filename == str(path)
assert isinstance(caught.value, FileNotFoundError)
assert getattr(caught.value, "filename") == str(path)
assert reader.calls == 1
assert requests == []
def test_unstarted_native_file_call_does_not_read_and_releases_reader():
def test_unstarted_native_file_call_does_not_read_and_releases_reader() -> None:
from litellm.rust_bridge import _native
class Reader:
pending: Coroutine[object, object, OCRResponse] | None = None
def read(self) -> bytes:
raise AssertionError("unstarted call consumed its document")
def create_call():
def create_call() -> tuple[Coroutine[object, object, OCRResponse], weakref.ReferenceType[Reader]]:
reader: Final = Reader()
pending: Final = _native.aocr(
model="mistral/mistral-ocr-latest", document={"type": "file", "file": reader}
)
pending: Final = _native.aocr(model="mistral/mistral-ocr-latest", document={"type": "file", "file": reader})
reader.pending = pending
return pending, weakref.ref(reader)
@ -457,6 +480,49 @@ def test_unstarted_native_file_call_does_not_read_and_releases_reader():
assert reference() is None
@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="requires Unix named pipes")
@pytest.mark.asyncio
async def test_native_path_cancellation_waits_for_read_completion(
ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]], tmp_path: Path
) -> None:
from litellm.rust_bridge import _native
server, requests = ocr_server
path: Final = tmp_path / "document.pdf"
os.mkfifo(path)
entered: Final = threading.Event()
release: Final = threading.Event()
def supply_document() -> None:
with path.open("wb") as stream:
stream.write(b"document")
stream.flush()
entered.set()
release.wait(5)
writer: Final = threading.Thread(target=supply_document, daemon=True)
writer.start()
task: Final = asyncio.create_task(
_native.aocr(
model="mistral/mistral-ocr-latest",
document={"type": "file", "file": path},
api_key="test-key",
api_base=f"http://127.0.0.1:{server.server_port}",
)
)
try:
assert await asyncio.to_thread(entered.wait, 3)
task.cancel()
await asyncio.sleep(0.05)
assert not task.done()
finally:
release.set()
await asyncio.to_thread(writer.join, 3)
with pytest.raises(asyncio.CancelledError):
await task
assert requests == []
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("model", ["mistral/mistral-ocr-latest", "azure_ai/doc-intelligence/prebuilt-read"])
@pytest.mark.asyncio