diff --git a/litellm-rust/crates/core/src/call_lifecycle/callbacks.rs b/litellm-rust/crates/core/src/call_lifecycle/callbacks.rs new file mode 100644 index 00000000000..c1b523cef6d --- /dev/null +++ b/litellm-rust/crates/core/src/call_lifecycle/callbacks.rs @@ -0,0 +1,857 @@ +use super::host::HostPhase; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CallbackId(pub u64); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Delivery { + Inline, + Await, + Worker, + Background, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CallbackKind { + CustomLogger, + Callable { internal: bool }, + Named { known: bool }, + Opaque, +} + +impl CallbackKind { + fn runs_sync_handler_for_async_call(self) -> bool { + match self { + Self::Callable { internal } => !internal, + Self::Named { known } => !known, + Self::CustomLogger | Self::Opaque => false, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CallbackMethod { + LogPreApiCall, + LogPostApiCall, + PreCallDeploymentHook, + PostCallSuccessDeploymentHook, + PostCallFailureDeploymentHook, + LoggingHook, + AsyncLoggingHook, + LogSuccessEvent, + AsyncLogSuccessEvent, + LogFailureEvent, + AsyncLogFailureEvent, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CallbackInvocation { + pub target: CallbackId, + pub method: CallbackMethod, + pub delivery: Delivery, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TargetErrorPolicy { + Contain, + Propagate, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LoggedMarker { + SyncSuccess, + AsyncSuccess, + SyncFailure, + AsyncFailure, +} + +impl LoggedMarker { + pub const fn key(self) -> &'static str { + match self { + Self::SyncSuccess => "has_logged_sync_success", + Self::AsyncSuccess => "has_logged_async_success", + Self::SyncFailure => "has_logged_sync_failure", + Self::AsyncFailure => "has_logged_async_failure", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CallbackFamily { + RequestPreCall, + RequestPostCall, + DeploymentPreCall, + DeploymentPostCall, + DeploymentFailure, + SyncSuccess, + AsyncSuccess, + SyncFailure, + AsyncFailure, +} + +impl CallbackFamily { + pub const fn dispatch_method(self) -> CallbackMethod { + match self { + Self::RequestPreCall => CallbackMethod::LogPreApiCall, + Self::RequestPostCall => CallbackMethod::LogPostApiCall, + Self::DeploymentPreCall => CallbackMethod::PreCallDeploymentHook, + Self::DeploymentPostCall => CallbackMethod::PostCallSuccessDeploymentHook, + Self::DeploymentFailure => CallbackMethod::PostCallFailureDeploymentHook, + Self::SyncSuccess => CallbackMethod::LogSuccessEvent, + Self::AsyncSuccess => CallbackMethod::AsyncLogSuccessEvent, + Self::SyncFailure => CallbackMethod::LogFailureEvent, + Self::AsyncFailure => CallbackMethod::AsyncLogFailureEvent, + } + } + + pub const fn hook_method(self) -> Option { + match self { + Self::SyncSuccess => Some(CallbackMethod::LoggingHook), + Self::AsyncSuccess => Some(CallbackMethod::AsyncLoggingHook), + _ => None, + } + } + + pub const fn marker(self) -> Option { + match self { + Self::SyncSuccess => Some(LoggedMarker::SyncSuccess), + Self::AsyncSuccess => Some(LoggedMarker::AsyncSuccess), + Self::SyncFailure => Some(LoggedMarker::SyncFailure), + Self::AsyncFailure => Some(LoggedMarker::AsyncFailure), + _ => None, + } + } + + pub const fn prepares_logging(self) -> bool { + self.marker().is_some() + } + + pub const fn error_policy(self) -> TargetErrorPolicy { + match self { + Self::DeploymentPreCall | Self::DeploymentPostCall => TargetErrorPolicy::Propagate, + _ => TargetErrorPolicy::Contain, + } + } + + pub fn targets(self, global: &[CallbackId], dynamic: Option<&[CallbackId]>) -> Vec { + match self { + Self::RequestPreCall | Self::RequestPostCall => global + .iter() + .chain(dynamic.unwrap_or_default()) + .copied() + .collect(), + Self::DeploymentPreCall | Self::DeploymentPostCall | Self::DeploymentFailure => { + global.to_vec() + } + Self::SyncSuccess | Self::AsyncSuccess | Self::SyncFailure | Self::AsyncFailure => { + let Some(dynamic) = dynamic else { + return global.to_vec(); + }; + let mut seen = std::collections::HashSet::new(); + dynamic + .iter() + .chain(global) + .copied() + .filter(|id| seen.insert(*id)) + .collect() + } + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReleaseGate { + Immediate, + Deferred, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Dispatch { + pub family: CallbackFamily, + pub delivery: Delivery, + pub gate: ReleaseGate, +} + +impl Dispatch { + pub const fn immediate(family: CallbackFamily, delivery: Delivery) -> Self { + Self { + family, + delivery, + gate: ReleaseGate::Immediate, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SuccessFacts { + pub asynchronous: bool, + pub internal: bool, + pub fallbacks: bool, + pub deferred: bool, + pub sync_target_kinds: Vec, +} + +pub fn plan_success(facts: &SuccessFacts) -> Vec { + if !facts.asynchronous { + return vec![Dispatch { + family: CallbackFamily::SyncSuccess, + delivery: Delivery::Worker, + gate: ReleaseGate::Immediate, + }]; + } + let background = (!facts.internal && !facts.fallbacks).then_some(Dispatch { + family: CallbackFamily::AsyncSuccess, + delivery: Delivery::Background, + gate: if facts.deferred { + ReleaseGate::Deferred + } else { + ReleaseGate::Immediate + }, + }); + let worker = facts + .sync_target_kinds + .iter() + .any(|kind| kind.runs_sync_handler_for_async_call()) + .then_some(Dispatch { + family: CallbackFamily::SyncSuccess, + delivery: Delivery::Worker, + gate: ReleaseGate::Immediate, + }); + background.into_iter().chain(worker).collect() +} + +pub fn plan_failure(phase: HostPhase, asynchronous: bool, internal: bool) -> Option { + if asynchronous && internal { + return None; + } + match phase { + HostPhase::Failure => Some(Dispatch::immediate( + CallbackFamily::SyncFailure, + Delivery::Inline, + )), + HostPhase::AsyncFailure => Some(Dispatch::immediate( + CallbackFamily::AsyncFailure, + Delivery::Await, + )), + _ => None, + } +} + +pub fn plan_request(family: CallbackFamily) -> Dispatch { + let delivery = match family { + CallbackFamily::RequestPreCall | CallbackFamily::RequestPostCall => Delivery::Inline, + _ => Delivery::Await, + }; + Dispatch::immediate(family, delivery) +} + +pub fn object_target_eligible( + sync_request: bool, + method: CallbackMethod, + kind: CallbackKind, +) -> bool { + match (method, kind) { + ( + CallbackMethod::LogSuccessEvent | CallbackMethod::LogFailureEvent, + CallbackKind::CustomLogger | CallbackKind::Callable { .. }, + ) => sync_request, + _ => true, + } +} + +pub trait DispatchFacts { + fn eligible(&mut self, target: CallbackId, method: CallbackMethod) -> bool; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DispatchStep { + PrepareLogging, + Invoke(CallbackInvocation), + MarkLogged(LoggedMarker), + Complete { aborted: bool }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InvocationOutcome { + Completed, + Failed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Position { + Prepare, + Hook(usize), + Mark, + Dispatch(usize), + Complete { aborted: bool }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CursorFacts { + pub already_logged: bool, + pub stream: bool, + pub sync_request: bool, +} + +pub struct DispatchCursor { + dispatch: Dispatch, + targets: Vec, + facts: CursorFacts, + position: Position, +} + +impl DispatchCursor { + pub fn start(dispatch: Dispatch, targets: Vec, facts: CursorFacts) -> Self { + let family = dispatch.family; + let position = if family.marker().is_some() && facts.already_logged { + Position::Complete { aborted: false } + } else if family.prepares_logging() { + Position::Prepare + } else { + Position::Dispatch(0) + }; + Self { + dispatch, + targets, + facts, + position, + } + } + + pub fn object_target_eligible(&self, method: CallbackMethod, kind: CallbackKind) -> bool { + object_target_eligible(self.facts.sync_request, method, kind) + } + + pub const fn family(&self) -> CallbackFamily { + self.dispatch.family + } + + pub const fn delivery(&self) -> Delivery { + self.dispatch.delivery + } + + pub fn targets(&self) -> &[CallbackId] { + &self.targets + } + + pub fn accept(&mut self, outcome: InvocationOutcome) { + if outcome == InvocationOutcome::Failed + && self.dispatch.family.error_policy() == TargetErrorPolicy::Propagate + { + self.position = Position::Complete { aborted: true }; + } + } + + pub fn next(&mut self, facts: &mut dyn DispatchFacts) -> DispatchStep { + loop { + match self.position { + Position::Prepare => { + self.position = self.after_prepare(); + return DispatchStep::PrepareLogging; + } + Position::Hook(index) => { + let Some(method) = self.dispatch.family.hook_method() else { + self.position = Position::Mark; + continue; + }; + let Some(target) = self.targets.get(index).copied() else { + self.position = Position::Mark; + continue; + }; + self.position = Position::Hook(index + 1); + if facts.eligible(target, method) { + return DispatchStep::Invoke(self.invocation(target, method)); + } + } + Position::Mark => { + self.position = Position::Dispatch(0); + match self.dispatch.family.marker() { + Some(marker) if !self.facts.stream => { + return DispatchStep::MarkLogged(marker); + } + _ => continue, + } + } + Position::Dispatch(index) => { + let Some(target) = self.targets.get(index).copied() else { + self.position = Position::Complete { aborted: false }; + continue; + }; + self.position = Position::Dispatch(index + 1); + let method = self.dispatch.family.dispatch_method(); + if facts.eligible(target, method) { + return DispatchStep::Invoke(self.invocation(target, method)); + } + } + Position::Complete { aborted } => return DispatchStep::Complete { aborted }, + } + } + } + + fn after_prepare(&self) -> Position { + if self.dispatch.family.hook_method().is_some() { + Position::Hook(0) + } else { + Position::Mark + } + } + + const fn invocation(&self, target: CallbackId, method: CallbackMethod) -> CallbackInvocation { + CallbackInvocation { + target, + method, + delivery: self.dispatch.delivery, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct AllEligible; + + impl DispatchFacts for AllEligible { + fn eligible(&mut self, _: CallbackId, _: CallbackMethod) -> bool { + true + } + } + + struct Gate bool>(F); + + impl bool> DispatchFacts for Gate { + fn eligible(&mut self, target: CallbackId, method: CallbackMethod) -> bool { + (self.0)(target, method) + } + } + + fn drain(cursor: &mut DispatchCursor, facts: &mut dyn DispatchFacts) -> Vec { + let mut steps = Vec::new(); + loop { + let step = cursor.next(facts); + steps.push(step); + match step { + DispatchStep::Complete { .. } => return steps, + DispatchStep::Invoke(_) => cursor.accept(InvocationOutcome::Completed), + _ => {} + } + } + } + + fn ids(values: &[u64]) -> Vec { + values.iter().copied().map(CallbackId).collect() + } + + fn delivery_for(family: CallbackFamily) -> Delivery { + match family { + CallbackFamily::SyncSuccess => Delivery::Worker, + CallbackFamily::AsyncSuccess => Delivery::Background, + CallbackFamily::SyncFailure + | CallbackFamily::RequestPreCall + | CallbackFamily::RequestPostCall => Delivery::Inline, + _ => Delivery::Await, + } + } + + fn start( + family: CallbackFamily, + targets: Vec, + already_logged: bool, + stream: bool, + ) -> DispatchCursor { + DispatchCursor::start( + Dispatch::immediate(family, delivery_for(family)), + targets, + CursorFacts { + already_logged, + stream, + sync_request: true, + }, + ) + } + + #[test] + fn terminal_families_order_dynamic_before_global_and_keep_first_duplicate() { + let combined = CallbackFamily::SyncSuccess.targets(&ids(&[3, 1, 4]), Some(&ids(&[1, 2]))); + assert_eq!(combined, ids(&[1, 2, 3, 4])); + } + + #[test] + fn terminal_families_copy_global_without_dedup_when_dynamic_is_absent() { + let combined = CallbackFamily::AsyncFailure.targets(&ids(&[3, 3, 1]), None); + assert_eq!(combined, ids(&[3, 3, 1])); + } + + #[test] + fn request_families_order_global_before_dynamic_without_dedup() { + let combined = CallbackFamily::RequestPreCall.targets(&ids(&[1, 2]), Some(&ids(&[2, 3]))); + assert_eq!(combined, ids(&[1, 2, 2, 3])); + } + + #[test] + fn success_runs_every_hook_before_any_dispatch_and_marks_between_passes() { + let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1, 2]), false, false); + let steps = drain(&mut cursor, &mut AllEligible); + let invocation = |target, method| { + DispatchStep::Invoke(CallbackInvocation { + target: CallbackId(target), + method, + delivery: Delivery::Worker, + }) + }; + assert_eq!( + steps, + vec![ + DispatchStep::PrepareLogging, + invocation(1, CallbackMethod::LoggingHook), + invocation(2, CallbackMethod::LoggingHook), + DispatchStep::MarkLogged(LoggedMarker::SyncSuccess), + invocation(1, CallbackMethod::LogSuccessEvent), + invocation(2, CallbackMethod::LogSuccessEvent), + DispatchStep::Complete { aborted: false }, + ] + ); + } + + #[test] + fn async_success_uses_async_leaf_methods_and_background_delivery() { + let mut cursor = start(CallbackFamily::AsyncSuccess, ids(&[7]), false, false); + let steps = drain(&mut cursor, &mut AllEligible); + let methods: Vec<_> = steps + .iter() + .filter_map(|step| match step { + DispatchStep::Invoke(invocation) => { + assert_eq!(invocation.delivery, Delivery::Background); + Some(invocation.method) + } + _ => None, + }) + .collect(); + assert_eq!( + methods, + [ + CallbackMethod::AsyncLoggingHook, + CallbackMethod::AsyncLogSuccessEvent + ] + ); + } + + #[test] + fn failure_and_request_families_have_no_hook_pass() { + for (family, delivery, method) in [ + ( + CallbackFamily::SyncFailure, + Delivery::Inline, + CallbackMethod::LogFailureEvent, + ), + ( + CallbackFamily::AsyncFailure, + Delivery::Await, + CallbackMethod::AsyncLogFailureEvent, + ), + ] { + let mut cursor = start(family, ids(&[1, 2]), false, false); + let steps = drain(&mut cursor, &mut AllEligible); + assert_eq!(steps[0], DispatchStep::PrepareLogging); + assert!(matches!(steps[1], DispatchStep::MarkLogged(_))); + assert_eq!( + &steps[2..], + &[ + DispatchStep::Invoke(CallbackInvocation { + target: CallbackId(1), + method, + delivery + }), + DispatchStep::Invoke(CallbackInvocation { + target: CallbackId(2), + method, + delivery + }), + DispatchStep::Complete { aborted: false }, + ] + ); + } + let mut cursor = start(CallbackFamily::RequestPreCall, ids(&[1]), false, false); + let steps = drain(&mut cursor, &mut AllEligible); + assert_eq!( + steps, + vec![ + DispatchStep::Invoke(CallbackInvocation { + target: CallbackId(1), + method: CallbackMethod::LogPreApiCall, + delivery: Delivery::Inline, + }), + DispatchStep::Complete { aborted: false }, + ] + ); + } + + #[test] + fn already_logged_marker_skips_the_whole_terminal_family_but_not_request_families() { + let mut cursor = start(CallbackFamily::AsyncSuccess, ids(&[1]), true, false); + assert_eq!( + cursor.next(&mut AllEligible), + DispatchStep::Complete { aborted: false } + ); + let mut cursor = start(CallbackFamily::RequestPostCall, ids(&[1]), true, false); + assert!(matches!( + cursor.next(&mut AllEligible), + DispatchStep::Invoke(_) + )); + } + + #[test] + fn streaming_skips_the_marker_write_but_still_dispatches() { + let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1]), false, true); + let steps = drain(&mut cursor, &mut AllEligible); + assert!( + !steps + .iter() + .any(|step| matches!(step, DispatchStep::MarkLogged(_))) + ); + assert_eq!( + steps + .iter() + .filter(|step| matches!(step, DispatchStep::Invoke(_))) + .count(), + 2 + ); + } + + #[test] + fn ineligible_targets_are_skipped_per_method_without_affecting_others() { + let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1, 2]), false, false); + let mut facts = Gate(|target, method| { + !(target == CallbackId(1) && method == CallbackMethod::LoggingHook) + && !(target == CallbackId(2) && method == CallbackMethod::LogSuccessEvent) + }); + let invoked: Vec<_> = drain(&mut cursor, &mut facts) + .into_iter() + .filter_map(|step| match step { + DispatchStep::Invoke(invocation) => Some((invocation.target.0, invocation.method)), + _ => None, + }) + .collect(); + assert_eq!( + invoked, + [ + (2, CallbackMethod::LoggingHook), + (1, CallbackMethod::LogSuccessEvent) + ] + ); + } + + #[test] + fn contained_failures_continue_and_propagating_failures_abort() { + let mut cursor = start(CallbackFamily::SyncFailure, ids(&[1, 2]), false, false); + assert_eq!(cursor.next(&mut AllEligible), DispatchStep::PrepareLogging); + assert!(matches!( + cursor.next(&mut AllEligible), + DispatchStep::MarkLogged(_) + )); + assert!(matches!( + cursor.next(&mut AllEligible), + DispatchStep::Invoke(_) + )); + cursor.accept(InvocationOutcome::Failed); + assert!(matches!( + cursor.next(&mut AllEligible), + DispatchStep::Invoke(CallbackInvocation { + target: CallbackId(2), + .. + }) + )); + + let mut cursor = start( + CallbackFamily::DeploymentPreCall, + ids(&[1, 2]), + false, + false, + ); + assert!(matches!( + cursor.next(&mut AllEligible), + DispatchStep::Invoke(_) + )); + cursor.accept(InvocationOutcome::Failed); + assert_eq!( + cursor.next(&mut AllEligible), + DispatchStep::Complete { aborted: true } + ); + } + + #[test] + fn sync_sdk_success_selects_one_worker_dispatch() { + let plan = plan_success(&SuccessFacts { + asynchronous: false, + internal: true, + fallbacks: true, + deferred: true, + sync_target_kinds: vec![], + }); + assert_eq!( + plan, + [Dispatch { + family: CallbackFamily::SyncSuccess, + delivery: Delivery::Worker, + gate: ReleaseGate::Immediate, + }] + ); + } + + #[test] + fn async_sdk_success_enqueues_background_then_worker_only_for_external_sync_targets() { + let base = SuccessFacts { + asynchronous: true, + internal: false, + fallbacks: false, + deferred: false, + sync_target_kinds: vec![ + CallbackKind::CustomLogger, + CallbackKind::Named { known: true }, + ], + }; + assert_eq!( + plan_success(&base), + [Dispatch { + family: CallbackFamily::AsyncSuccess, + delivery: Delivery::Background, + gate: ReleaseGate::Immediate, + }] + ); + let with_external = SuccessFacts { + sync_target_kinds: vec![ + CallbackKind::CustomLogger, + CallbackKind::Callable { internal: false }, + ], + deferred: true, + ..base.clone() + }; + assert_eq!( + plan_success(&with_external), + [ + Dispatch { + family: CallbackFamily::AsyncSuccess, + delivery: Delivery::Background, + gate: ReleaseGate::Deferred, + }, + Dispatch { + family: CallbackFamily::SyncSuccess, + delivery: Delivery::Worker, + gate: ReleaseGate::Immediate, + }, + ] + ); + let internal_or_fallback = SuccessFacts { + internal: true, + sync_target_kinds: vec![CallbackKind::Named { known: false }], + ..base + }; + assert_eq!( + plan_success(&internal_or_fallback), + [Dispatch { + family: CallbackFamily::SyncSuccess, + delivery: Delivery::Worker, + gate: ReleaseGate::Immediate, + }] + ); + } + + #[test] + fn opaque_and_internal_targets_never_trigger_the_worker_for_async_calls() { + let plan = plan_success(&SuccessFacts { + asynchronous: true, + internal: false, + fallbacks: true, + deferred: false, + sync_target_kinds: vec![ + CallbackKind::Opaque, + CallbackKind::Callable { internal: true }, + ], + }); + assert!(plan.is_empty()); + } + + #[test] + fn failure_families_follow_the_phase_and_skip_internal_async_calls() { + assert_eq!( + plan_failure(HostPhase::Failure, false, true), + Some(Dispatch::immediate( + CallbackFamily::SyncFailure, + Delivery::Inline + )) + ); + assert_eq!( + plan_failure(HostPhase::AsyncFailure, true, false), + Some(Dispatch::immediate( + CallbackFamily::AsyncFailure, + Delivery::Await + )) + ); + assert_eq!(plan_failure(HostPhase::Failure, true, true), None); + assert_eq!(plan_failure(HostPhase::Success, false, false), None); + } + + #[test] + fn delivery_is_selected_by_the_plan_and_carried_on_every_invocation() { + let mut cursor = DispatchCursor::start( + Dispatch::immediate(CallbackFamily::SyncFailure, Delivery::Worker), + ids(&[1]), + CursorFacts { + already_logged: false, + stream: false, + sync_request: true, + }, + ); + let deliveries: Vec<_> = drain(&mut cursor, &mut AllEligible) + .into_iter() + .filter_map(|step| match step { + DispatchStep::Invoke(invocation) => Some(invocation.delivery), + _ => None, + }) + .collect(); + assert_eq!(deliveries, [Delivery::Worker]); + assert_eq!( + plan_request(CallbackFamily::RequestPreCall).delivery, + Delivery::Inline + ); + assert_eq!( + plan_request(CallbackFamily::DeploymentPreCall).delivery, + Delivery::Await + ); + } + + #[test] + fn sync_leaf_methods_skip_object_targets_on_async_requests() { + let asynchronous = DispatchCursor::start( + Dispatch::immediate(CallbackFamily::SyncSuccess, Delivery::Worker), + ids(&[1]), + CursorFacts { + already_logged: false, + stream: false, + sync_request: false, + }, + ); + for kind in [ + CallbackKind::CustomLogger, + CallbackKind::Callable { internal: false }, + ] { + assert!(!asynchronous.object_target_eligible(CallbackMethod::LogSuccessEvent, kind)); + assert!(!asynchronous.object_target_eligible(CallbackMethod::LogFailureEvent, kind)); + assert!(asynchronous.object_target_eligible(CallbackMethod::LoggingHook, kind)); + assert!( + asynchronous.object_target_eligible(CallbackMethod::AsyncLogSuccessEvent, kind) + ); + } + assert!(asynchronous.object_target_eligible( + CallbackMethod::LogSuccessEvent, + CallbackKind::Named { known: false } + )); + let synchronous = start(CallbackFamily::SyncSuccess, ids(&[1]), false, false); + assert!( + synchronous.object_target_eligible( + CallbackMethod::LogSuccessEvent, + CallbackKind::CustomLogger + ) + ); + } +} diff --git a/litellm-rust/crates/core/src/call_lifecycle/host.rs b/litellm-rust/crates/core/src/call_lifecycle/host.rs index f9e18b9f116..0e93b95b947 100644 --- a/litellm-rust/crates/core/src/call_lifecycle/host.rs +++ b/litellm-rust/crates/core/src/call_lifecycle/host.rs @@ -223,9 +223,11 @@ mod tests { ] { assert_eq!(lifecycle.phase(), phase); assert!( - lifecycle.accept(Err(HostFailure::Error(crate::ocr::Error::InvalidRequest( - "callback".into() - )))).is_none() + lifecycle + .accept(Err(HostFailure::Error(crate::ocr::Error::InvalidRequest( + "callback".into() + )))) + .is_none() ); } assert_eq!(lifecycle.phase(), HostPhase::Complete); diff --git a/litellm-rust/crates/core/src/call_lifecycle/mod.rs b/litellm-rust/crates/core/src/call_lifecycle/mod.rs index fcd9cbd2ab8..8788e0926b0 100644 --- a/litellm-rust/crates/core/src/call_lifecycle/mod.rs +++ b/litellm-rust/crates/core/src/call_lifecycle/mod.rs @@ -1,9 +1,17 @@ use std::future::Future; use std::time::{Instant, SystemTime, UNIX_EPOCH}; +pub mod callbacks; pub mod host; +pub mod registration; pub mod types; +pub use callbacks::{ + CallbackFamily, CallbackId, CallbackInvocation, CallbackKind, CallbackMethod, CursorFacts, + Delivery, Dispatch, DispatchCursor, DispatchFacts, DispatchStep, InvocationOutcome, + LoggedMarker, ReleaseGate, SuccessFacts, TargetErrorPolicy, object_target_eligible, + plan_failure, plan_request, plan_success, +}; pub use types::{ CallLifecycleContext, CallLifecyclePhase, CallLifecyclePhaseTiming, CallLifecycleRequest, CallLifecycleTiming, diff --git a/litellm-rust/crates/core/src/call_lifecycle/registration.rs b/litellm-rust/crates/core/src/call_lifecycle/registration.rs new file mode 100644 index 00000000000..eac0526c9fa --- /dev/null +++ b/litellm-rust/crates/core/src/call_lifecycle/registration.rs @@ -0,0 +1,374 @@ +use std::collections::HashSet; + +use super::callbacks::CallbackId; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Registry { + Input, + AsyncInput, + Success, + AsyncSuccess, + Failure, + AsyncFailure, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Registration { + Named { known: bool, async_only: bool }, + Object { asynchronous: bool }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Entry { + pub id: CallbackId, + pub registration: Registration, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Candidate { + pub resolved: Option, + pub duplicate_type: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct RegistrationFacts { + pub candidates: Vec, + pub input: Vec, + pub success: Vec, + pub failure: Vec, + pub async_success: Vec, + pub async_failure: Vec, + pub bootstrap_pending: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NamedEvent { + Success, + Failure, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RegistryMutation { + Append(Registry, CallbackId), + Remove(Registry, CallbackId), + ExpandNamed(NamedEvent, CallbackId), + Bootstrap, +} + +struct Planner { + input: Vec, + success: Vec, + failure: Vec, + async_success: HashSet, + async_failure: HashSet, + mutations: Vec, +} + +impl Planner { + fn contains(&self, registry: Registry, id: CallbackId) -> bool { + match registry { + Registry::Input => self.input.iter().any(|entry| entry.id == id), + Registry::Success => self.success.iter().any(|entry| entry.id == id), + Registry::Failure => self.failure.iter().any(|entry| entry.id == id), + Registry::AsyncSuccess => self.async_success.contains(&id), + Registry::AsyncFailure => self.async_failure.contains(&id), + Registry::AsyncInput => false, + } + } + + fn append(&mut self, registry: Registry, entry: Entry) { + if self.contains(registry, entry.id) { + return; + } + match registry { + Registry::Input => self.input.push(entry), + Registry::Success => self.success.push(entry), + Registry::Failure => self.failure.push(entry), + Registry::AsyncSuccess => { + self.async_success.insert(entry.id); + } + Registry::AsyncFailure => { + self.async_failure.insert(entry.id); + } + Registry::AsyncInput => {} + } + self.mutations + .push(RegistryMutation::Append(registry, entry.id)); + } + + fn record(&mut self, mutation: RegistryMutation) { + self.mutations.push(mutation); + } +} + +fn is_asynchronous(registration: Registration) -> bool { + matches!(registration, Registration::Object { asynchronous: true }) +} + +pub fn plan_registration(facts: &RegistrationFacts) -> Vec { + let mut planner = Planner { + input: facts.input.clone(), + success: facts.success.clone(), + failure: facts.failure.clone(), + async_success: facts.async_success.iter().copied().collect(), + async_failure: facts.async_failure.iter().copied().collect(), + mutations: Vec::new(), + }; + + for candidate in &facts.candidates { + let Some(entry) = candidate.resolved else { + continue; + }; + if candidate.duplicate_type { + continue; + } + planner.append(Registry::Input, entry); + if !is_asynchronous(entry.registration) { + planner.append(Registry::Success, entry); + planner.append(Registry::Failure, entry); + } + planner.append(Registry::AsyncSuccess, entry); + planner.append(Registry::AsyncFailure, entry); + } + + if facts.bootstrap_pending + && !(planner.input.is_empty() && planner.success.is_empty() && planner.failure.is_empty()) + { + planner.record(RegistryMutation::Bootstrap); + } + + let input = planner.input.clone(); + for entry in input + .iter() + .filter(|entry| is_asynchronous(entry.registration)) + { + planner.record(RegistryMutation::Append(Registry::AsyncInput, entry.id)); + planner.record(RegistryMutation::Remove(Registry::Input, entry.id)); + } + + let success = planner.success.clone(); + for entry in &success { + match entry.registration { + Registration::Object { asynchronous: true } + | Registration::Named { + async_only: true, .. + } => { + planner.append(Registry::AsyncSuccess, *entry); + planner.record(RegistryMutation::Remove(Registry::Success, entry.id)); + } + Registration::Named { known: true, .. } => { + planner.record(RegistryMutation::ExpandNamed(NamedEvent::Success, entry.id)); + } + _ => {} + } + } + + let failure = planner.failure.clone(); + for entry in &failure { + match entry.registration { + Registration::Object { asynchronous: true } => { + planner.append(Registry::AsyncFailure, *entry); + planner.record(RegistryMutation::Remove(Registry::Failure, entry.id)); + } + Registration::Named { known: true, .. } => { + planner.record(RegistryMutation::ExpandNamed(NamedEvent::Failure, entry.id)); + } + _ => {} + } + } + + planner.mutations +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DynamicSuccessSlot { + Sync, + Async, +} + +pub fn classify_dynamic_success(entry: Entry, named_async: bool) -> DynamicSuccessSlot { + match entry.registration { + Registration::Object { asynchronous: true } => DynamicSuccessSlot::Async, + Registration::Named { .. } if named_async => DynamicSuccessSlot::Async, + _ => DynamicSuccessSlot::Sync, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn object(id: u64, asynchronous: bool) -> Entry { + Entry { + id: CallbackId(id), + registration: Registration::Object { asynchronous }, + } + } + + fn named(id: u64, known: bool, async_only: bool) -> Entry { + Entry { + id: CallbackId(id), + registration: Registration::Named { known, async_only }, + } + } + + fn candidate(entry: Entry) -> Candidate { + Candidate { + resolved: Some(entry), + duplicate_type: false, + } + } + + #[test] + fn sync_callback_in_callbacks_registers_in_every_list_once() { + let facts = RegistrationFacts { + candidates: vec![candidate(object(1, false)), candidate(object(1, false))], + ..RegistrationFacts::default() + }; + assert_eq!( + plan_registration(&facts), + [ + RegistryMutation::Append(Registry::Input, CallbackId(1)), + RegistryMutation::Append(Registry::Success, CallbackId(1)), + RegistryMutation::Append(Registry::Failure, CallbackId(1)), + RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(1)), + RegistryMutation::Append(Registry::AsyncFailure, CallbackId(1)), + ] + ); + } + + #[test] + fn async_callable_in_callbacks_skips_sync_lists_and_moves_out_of_input() { + let facts = RegistrationFacts { + candidates: vec![candidate(object(2, true))], + ..RegistrationFacts::default() + }; + assert_eq!( + plan_registration(&facts), + [ + RegistryMutation::Append(Registry::Input, CallbackId(2)), + RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(2)), + RegistryMutation::Append(Registry::AsyncFailure, CallbackId(2)), + RegistryMutation::Append(Registry::AsyncInput, CallbackId(2)), + RegistryMutation::Remove(Registry::Input, CallbackId(2)), + ] + ); + } + + #[test] + fn unresolved_and_duplicate_type_named_candidates_are_skipped() { + let facts = RegistrationFacts { + candidates: vec![ + Candidate { + resolved: None, + duplicate_type: false, + }, + Candidate { + resolved: Some(object(3, false)), + duplicate_type: true, + }, + ], + ..RegistrationFacts::default() + }; + assert!(plan_registration(&facts).is_empty()); + } + + #[test] + fn already_registered_callbacks_are_not_appended_again() { + let facts = RegistrationFacts { + candidates: vec![candidate(object(1, false))], + input: vec![object(1, false)], + success: vec![object(1, false)], + failure: vec![object(1, false)], + async_success: vec![CallbackId(1)], + async_failure: vec![CallbackId(1)], + ..RegistrationFacts::default() + }; + assert!(plan_registration(&facts).is_empty()); + } + + #[test] + fn bootstrap_runs_once_when_any_public_list_is_populated() { + let empty = RegistrationFacts { + bootstrap_pending: true, + ..RegistrationFacts::default() + }; + assert!(plan_registration(&empty).is_empty()); + let populated = RegistrationFacts { + bootstrap_pending: true, + candidates: vec![candidate(object(1, false))], + ..RegistrationFacts::default() + }; + assert!(plan_registration(&populated).contains(&RegistryMutation::Bootstrap)); + let already = RegistrationFacts { + bootstrap_pending: false, + success: vec![object(1, false)], + ..RegistrationFacts::default() + }; + assert!(!plan_registration(&already).contains(&RegistryMutation::Bootstrap)); + } + + #[test] + fn success_safety_net_moves_async_and_async_only_names_and_expands_known_names() { + let facts = RegistrationFacts { + success: vec![ + object(1, true), + named(2, false, true), + named(3, true, false), + named(4, false, false), + object(5, false), + ], + ..RegistrationFacts::default() + }; + assert_eq!( + plan_registration(&facts), + [ + RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(1)), + RegistryMutation::Remove(Registry::Success, CallbackId(1)), + RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(2)), + RegistryMutation::Remove(Registry::Success, CallbackId(2)), + RegistryMutation::ExpandNamed(NamedEvent::Success, CallbackId(3)), + ] + ); + } + + #[test] + fn failure_safety_net_ignores_async_only_names() { + let facts = RegistrationFacts { + failure: vec![ + object(1, true), + named(2, false, true), + named(3, true, false), + ], + async_failure: vec![CallbackId(1)], + ..RegistrationFacts::default() + }; + assert_eq!( + plan_registration(&facts), + [ + RegistryMutation::Remove(Registry::Failure, CallbackId(1)), + RegistryMutation::ExpandNamed(NamedEvent::Failure, CallbackId(3)), + ] + ); + } + + #[test] + fn dynamic_success_split_follows_async_callables_and_selected_names() { + assert_eq!( + classify_dynamic_success(object(1, true), false), + DynamicSuccessSlot::Async + ); + assert_eq!( + classify_dynamic_success(object(1, false), false), + DynamicSuccessSlot::Sync + ); + assert_eq!( + classify_dynamic_success(named(2, true, false), true), + DynamicSuccessSlot::Async + ); + assert_eq!( + classify_dynamic_success(named(2, true, true), false), + DynamicSuccessSlot::Sync + ); + } +} diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs index 50d5b703152..9529a726ac3 100644 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs @@ -816,7 +816,8 @@ mod tests { "type":"document_url", "document_url":"https://example.com/document.pdf" })) - .unwrap().into(); + .unwrap() + .into(); perform_ocr(request).await.unwrap(); server.await.unwrap(); diff --git a/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs index fb16445e11d..fc11f62833c 100644 --- a/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs @@ -361,10 +361,12 @@ mod tests { } }), ); - let request = request.with_document(serde_json::from_value(json!({ - "type":"image_url","image_url":"https://example.com/original.png" - })) - .unwrap()); + let request = request.with_document( + serde_json::from_value(json!({ + "type":"image_url","image_url":"https://example.com/original.png" + })) + .unwrap(), + ); let request = crate::ocr::prepare::prepare_request(request); let http = CohereParseConfig .prepare_request(&request, &crate::ocr::test_support::ocr_client()) @@ -503,10 +505,12 @@ mod tests { "https://example.com", json!({"output_format":null,"req_format":null}), ); - let request = request.with_document(serde_json::from_value( + let request = request.with_document( + serde_json::from_value( json!({"type":"image_url","image_url":"https://example.com/a.png"}), ) - .unwrap()); + .unwrap(), + ); assert_eq!( request.response_format().unwrap(), crate::ocr::types::OcrResponseFormat::Litellm diff --git a/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs index 7492a2ef6ec..4b05f1ee168 100644 --- a/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs @@ -738,7 +738,8 @@ mod tests { "result":{"chunks":[]} }))]) .await; - let request = crate::ocr::test_support::with_source(wire_request(model, &base, options), source); + let request = + crate::ocr::test_support::with_source(wire_request(model, &base, options), source); perform_ocr(request).await.unwrap(); server.await.unwrap(); @@ -857,7 +858,9 @@ mod tests { #[tokio::test] async fn rejects_invalid_document_sources_before_network(#[case] source: &str) { let request = crate::ocr::test_support::with_source( - wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})), source); + wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})), + source, + ); assert!(perform_ocr(request).await.is_err()); } @@ -913,7 +916,9 @@ mod tests { let raw = json!({"job_id":"job-1","result":{"chunks":[]}}); let (base, seen, server) = mock_server(vec![MockResponse::json(raw)]).await; let mut request = crate::ocr::test_support::with_source( - wire_request("reducto/parse-v3", &base, json!({})), "reducto://ready.pdf"); + wire_request("reducto/parse-v3", &base, json!({})), + "reducto://ready.pdf", + ); request.transport.extra_headers = vec![("authorization".into(), "Bearer existing".into())]; let response = perform_ocr(request).await.unwrap(); @@ -984,6 +989,7 @@ mod tests { request: OcrDuringCallRequest, ) -> OcrHookFuture<'_, OcrDuringCallRequest> { Box::pin(async move { + assert_eq!(request.optional_params["use_cache"], json!(true)); assert_eq!( request.body["document_url"], "data:application/pdf;base64,YWJj" @@ -1000,7 +1006,7 @@ mod tests { async fn guardrail_rewrites_document_before_upload() { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"result":{"chunks":[]}}))]).await; - let mut request = wire_request("reducto/parse-v3", &base, json!({})); + let mut request = wire_request("reducto/parse-v3", &base, json!({"use_cache":true})); request.hooks = Arc::new(RewriteDocument); perform_ocr(request).await.unwrap(); diff --git a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs index beca4dc141d..a3eb35f16f2 100644 --- a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs @@ -338,8 +338,12 @@ mod tests { options.clone(), ); let vertex = wire_request("vertex_ai/mistral-ocr-maas", "https://vertex.test", options); - let direct = crate::ocr::prepare::prepare_request(crate::ocr::test_support::resolved_request(direct)); - let vertex = crate::ocr::prepare::prepare_request(crate::ocr::test_support::resolved_request(vertex)); + let direct = crate::ocr::prepare::prepare_request( + crate::ocr::test_support::resolved_request(direct), + ); + let vertex = crate::ocr::prepare::prepare_request( + crate::ocr::test_support::resolved_request(vertex), + ); let direct_http = MistralOCRConfig .prepare_request(&direct, &client) .await diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index 5881519855c..6ceb4eedbd2 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -39,9 +39,10 @@ impl OcrClient { ) -> Result { use super::{ NativeOutcome, OcrAdmission, OcrCall, OcrCallStep, OcrHookHost, OcrHost, - OcrHostOperation, OcrHostResult, + OcrHostOperation, OcrHostResult, OcrProjectedRequest, }; + let intercepts_requests = request.hooks.intercepts_requests(); let host = OcrHookHost::new(request.hooks.clone()); let mut request = Some(request); let NativeOutcome::Completed(mut call) = OcrCall::admit(self.clone(), OcrAdmission::all()) @@ -54,14 +55,15 @@ impl OcrClient { loop { match call.resume(result.take()).await? { OcrCallStep::Host(OcrHostOperation::ProjectRequest) => { - result = Some(OcrHostResult::Request(Ok(( - Box::new(request.take().ok_or_else(|| { + result = Some(OcrHostResult::Request(Ok(OcrProjectedRequest { + request: Box::new(request.take().ok_or_else(|| { crate::ocr::Error::InvalidRequest( "OCR request was already projected".into(), ) })?), - false, - )))) + intercepts_requests, + host_token_provider: false, + }))) } OcrCallStep::Host(operation) => result = Some(host.invoke(operation).await), OcrCallStep::Complete(response) => return Ok(response), diff --git a/litellm-rust/crates/core/src/ocr/document.rs b/litellm-rust/crates/core/src/ocr/document.rs index 0f737e381ef..e9a13f0fd11 100644 --- a/litellm-rust/crates/core/src/ocr/document.rs +++ b/litellm-rust/crates/core/src/ocr/document.rs @@ -15,7 +15,10 @@ pub(crate) fn read_path_document( ) -> Result { let mut bytes = Vec::new(); std::fs::File::open(path) - .and_then(|file| file.take(OCR_INLINE_MAX_BYTES as u64 + 1).read_to_end(&mut bytes)) + .and_then(|file| { + file.take(OCR_INLINE_MAX_BYTES as u64 + 1) + .read_to_end(&mut bytes) + }) .map_err(|source| super::Error::FileRead { path: path.to_owned(), source: std::sync::Arc::new(source), @@ -201,9 +204,14 @@ mod tests { #[test] fn path_preparation_preserves_io_causes_and_enforces_the_inline_limit() { - let path = std::env::temp_dir().join(format!("ocr-document-{:032x}.png", rand::random::())); + let path = + std::env::temp_dir().join(format!("ocr-document-{:032x}.png", rand::random::())); let error = read_path_document(&path, None).unwrap_err(); - let super::super::Error::FileRead { path: failed_path, source } = error else { + let super::super::Error::FileRead { + path: failed_path, + source, + } = error + else { panic!("missing path must produce a typed file error"); }; assert_eq!(failed_path, path); @@ -215,9 +223,16 @@ mod tests { let image = read_path_document(&path, None).unwrap(); let overridden = read_path_document(&path, Some("application/pdf")).unwrap(); std::fs::remove_file(&path).unwrap(); - assert!(matches!(oversized, Err(super::super::Error::InlineDocumentTooLarge))); - assert!(matches!(image, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2UgYnl0ZXM=")); - assert!(matches!(overridden, OcrDocument::DocumentUrl { document_url, .. } if document_url == "data:application/pdf;base64,aW1hZ2UgYnl0ZXM=")); + assert!(matches!( + oversized, + Err(super::super::Error::InlineDocumentTooLarge) + )); + assert!( + matches!(image, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2UgYnl0ZXM=") + ); + assert!( + matches!(overridden, OcrDocument::DocumentUrl { document_url, .. } if document_url == "data:application/pdf;base64,aW1hZ2UgYnl0ZXM=") + ); } fn document(source: &str) -> OcrDocument { diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/core/src/ocr/error.rs index 0c70c478305..a8ac969577f 100644 --- a/litellm-rust/crates/core/src/ocr/error.rs +++ b/litellm-rust/crates/core/src/ocr/error.rs @@ -8,6 +8,8 @@ pub enum Error { }, #[error("File is empty or could not be read")] EmptyFile, + #[error("Host OCR document read failed")] + HostDocumentRead, #[error("Failed to read OCR file {}: {source}", path.display())] FileRead { path: std::path::PathBuf, diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index d66da5c81d5..7e42111da0a 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use super::OcrClient; use super::hooks::{OcrHooks, OcrLifecycleHooks, OcrPostCallRequest}; -use super::types::{ResolvedOcrRequest, LiteLLMOcrResponse, PreparedOcrRequest}; +use super::types::{LiteLLMOcrResponse, PreparedOcrRequest, ResolvedOcrRequest}; use crate::call_lifecycle::{CallLifecycle, CallLifecycleContext}; use crate::llms::base_llm::ocr::transformation::OcrResponseContext; diff --git a/litellm-rust/crates/core/src/ocr/hooks.rs b/litellm-rust/crates/core/src/ocr/hooks.rs index 20cb3841c87..ee9b0f776d9 100644 --- a/litellm-rust/crates/core/src/ocr/hooks.rs +++ b/litellm-rust/crates/core/src/ocr/hooks.rs @@ -22,6 +22,7 @@ pub struct OcrPreCallRequest { pub struct OcrDuringCallRequest { pub model: String, pub custom_llm_provider: String, + pub optional_params: Value, pub api_key: Option, pub url: String, pub headers: Vec<(String, String)>, diff --git a/litellm-rust/crates/core/src/ocr/lifecycle.rs b/litellm-rust/crates/core/src/ocr/lifecycle.rs index c616685e101..dd88e11dca6 100644 --- a/litellm-rust/crates/core/src/ocr/lifecycle.rs +++ b/litellm-rust/crates/core/src/ocr/lifecycle.rs @@ -1,6 +1,7 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::{mpsc, oneshot}; @@ -9,8 +10,8 @@ use super::hooks::{ OcrDuringCallRequest, OcrHookFuture, OcrHooks, OcrLogFuture, OcrPostCallRequest, OcrPreCallRequest, }; -use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient, OcrDocumentInput, OcrFileContent}; use super::types::ResolvedOcrRequest; +use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient, OcrDocumentInput, OcrFileContent}; use crate::call_lifecycle::host::{ HostCall, HostCallFuture, HostCallStep, HostFailure, HostLifecycle, HostPhase, }; @@ -82,8 +83,14 @@ impl OcrHostOperation { } } +pub struct OcrProjectedRequest { + pub request: Box, + pub intercepts_requests: bool, + pub host_token_provider: bool, +} + pub enum OcrHostResult { - Request(Result<(Box, bool), super::Error>), + Request(Result), Document(Result), Lifecycle(Result<(), HostFailure>), AzureAdToken(Result), @@ -160,9 +167,10 @@ impl OcrCall { Some(OcrHostResult::Request(result)) if self.projecting => { self.projecting = false; match result { - Ok((request, azure_ad_token_provider)) => { - self.execution.request = Some(*request); - self.execution.azure_ad_token_provider = azure_ad_token_provider; + Ok(projected) => { + self.execution.set_request(*projected.request); + self.execution.intercepts_requests = projected.intercepts_requests; + self.execution.host_token_provider = projected.host_token_provider; } Err(error) => self.accept(Err(HostFailure::Error(error))), } @@ -268,6 +276,15 @@ impl OcrCall { fn accept(&mut self, result: Result<(), HostFailure>) { let cancelled = matches!(&result, Err(HostFailure::Cancelled(_))); if let Some(error) = self.lifecycle.accept(result) { + if let Some((_, timing)) = self + .execution + .terminal + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_mut() + { + timing.end_time = epoch_seconds(); + } if cancelled { self.error = Some(error); } else { @@ -333,11 +350,33 @@ struct OcrExecution { pending_result: Option>, execution: Option>>, completed: bool, - azure_ad_token_provider: bool, + intercepts_requests: bool, + host_token_provider: bool, terminal: Arc>>, } impl OcrExecution { + fn set_request(&mut self, mut request: LiteLLMOcrRequest) { + let call_id = request + .litellm_call_id + .clone() + .unwrap_or_else(|| format!("ocr-{:032x}", rand::random::())); + let context = CallLifecycleContext::new( + "ocr", + request.model.clone(), + request.provider_name(), + call_id.clone(), + ); + request.litellm_call_id = Some(call_id); + let start = epoch_seconds(); + *self + .terminal + .lock() + .unwrap_or_else(|error| error.into_inner()) = + Some((context, CallLifecycleTiming::new(start, start, Vec::new()))); + self.request = Some(request); + } + fn new(client: OcrClient) -> Self { let (operations_tx, operations_rx) = mpsc::unbounded_channel(); Self { @@ -350,7 +389,8 @@ impl OcrExecution { pending_result: None, execution: None, completed: false, - azure_ad_token_provider: false, + intercepts_requests: false, + host_token_provider: false, terminal: Arc::default(), } } @@ -366,11 +406,16 @@ impl OcrExecution { } let result = if self.reading { let Some(OcrHostResult::Document(content)) = result else { - return Err(super::Error::InvalidRequest("OCR document read result is required".into())); + return Err(super::Error::InvalidRequest( + "OCR document read result is required".into(), + )); }; self.reading = false; let content = content?; - let request = self.request.take().expect("pending document read has a request"); + let request = self + .request + .take() + .expect("pending document read has a request"); let OcrDocumentInput::HostReader { mime_type } = &request.document else { unreachable!("only host readers request document reads"); }; @@ -428,7 +473,10 @@ impl OcrExecution { async fn prepare(&mut self) -> Result, super::Error> { if self.preparation.is_none() { - let request = self.request.take().expect("admitted OCR call has a request"); + let request = self + .request + .take() + .expect("admitted OCR call has a request"); if matches!(request.document, OcrDocumentInput::HostReader { .. }) { self.request = Some(request); self.reading = true; @@ -444,15 +492,25 @@ impl OcrExecution { OcrDocumentInput::Path { path, mime_type } => { super::document::read_path_document(path, mime_type.as_deref())? } - OcrDocumentInput::Bytes { bytes, file_name, mime_type } => { - super::document::encode_file_document(bytes, file_name.as_deref(), mime_type.as_deref())? - } + OcrDocumentInput::Bytes { + bytes, + file_name, + mime_type, + } => super::document::encode_file_document( + bytes, + file_name.as_deref(), + mime_type.as_deref(), + )?, _ => unreachable!("only native file inputs require preparation"), }; Ok(request.with_document(document)) })); } - let result = self.preparation.as_mut().expect("document preparation started").await; + let result = self + .preparation + .as_mut() + .expect("document preparation started") + .await; self.preparation = None; let request = result.map_err(|error| super::Error::DocumentTask(Arc::new(error)))??; self.start(request); @@ -461,8 +519,7 @@ impl OcrExecution { fn start(&mut self, mut request: ResolvedOcrRequest) { let client = self.client.take().expect("admitted OCR call has a client"); - let intercepts_requests = request.hooks.intercepts_requests(); - if self.azure_ad_token_provider { + if self.host_token_provider { request.azure_ad_token_provider = Some(TokenProviderHandle::new(Arc::new( OcrAzureAdTokenProvider { operations: self.operations_tx.clone(), @@ -471,7 +528,7 @@ impl OcrExecution { } request.hooks = Arc::new(ProtocolHooks { operations: self.operations_tx.clone(), - intercepts_requests, + intercepts_requests: self.intercepts_requests, terminal: self.terminal.clone(), }); self.execution = Some(tokio::spawn(async move { @@ -517,6 +574,13 @@ struct ProtocolHooks { terminal: Arc>>, } +fn epoch_seconds() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} + #[derive(Debug)] struct OcrAzureAdTokenProvider { operations: mpsc::UnboundedSender, @@ -743,36 +807,60 @@ mod tests { use crate::ocr::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; use crate::ocr::{ LiteLLMOcrRequest, NativeOutcome, NoopOcrHost, OcrAdmission, OcrCall, OcrCallStep, - OcrDecline, OcrDocument, OcrHost, OcrHostOperation, OcrHostResult, + OcrDecline, OcrDocument, OcrHost, OcrHostOperation, OcrHostResult, OcrProjectedRequest, }; + fn projected_request(request: LiteLLMOcrRequest) -> OcrHostResult { + OcrHostResult::Request(Ok(OcrProjectedRequest { + intercepts_requests: request.hooks.intercepts_requests(), + request: Box::new(request), + host_token_provider: false, + })) + } + #[tokio::test] async fn sdk_paths_are_read_during_execution_and_hooks_observe_normalized_documents() { struct CaptureDocument(Arc>>); impl OcrHooks for CaptureDocument { - fn intercepts_requests(&self) -> bool { true } + fn intercepts_requests(&self) -> bool { + true + } fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { self.0.lock().unwrap().push(request.document.clone()); Box::pin(async { Ok(request) }) } } let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - let path = std::env::temp_dir().join(format!("ocr-sdk-{:032x}.pdf", rand::random::())); + let path = + std::env::temp_dir().join(format!("ocr-sdk-{:032x}.pdf", rand::random::())); let documents = Arc::new(Mutex::new(Vec::new())); let request = LiteLLMOcrRequest::from_inputs( - "mistral/model".into(), path.clone(), None, Default::default(), - crate::ocr::OcrConnectionInputs { api_base: Some(base), api_key: Some("test-key".into()), ..Default::default() }, - ).unwrap().with_host_hooks(Arc::new(CaptureDocument(documents.clone())), None); + "mistral/model".into(), + path.clone(), + None, + Default::default(), + crate::ocr::OcrConnectionInputs { + api_base: Some(base), + api_key: Some("test-key".into()), + ..Default::default() + }, + ) + .unwrap() + .with_host_hooks(Arc::new(CaptureDocument(documents.clone())), None); std::fs::write(&path, b"sdk document").unwrap(); let result = perform_ocr(request).await; std::fs::remove_file(&path).unwrap(); result.unwrap(); server.await.unwrap(); let expected = json!({"type":"document_url","document_url":"data:application/pdf;base64,c2RrIGRvY3VtZW50"}); - assert_eq!(serde_json::to_value(&documents.lock().unwrap()[0]).unwrap(), expected); + assert_eq!( + serde_json::to_value(&documents.lock().unwrap()[0]).unwrap(), + expected + ); let requests = seen.lock().unwrap(); assert_eq!(requests.len(), 1); - let body: Value = serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap(); + let body: Value = + serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap(); assert_eq!(body["document"], expected); } @@ -781,9 +869,18 @@ mod tests { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; let request = wire_request("mistral/model", &base, json!({})) .with_document(crate::ocr::OcrDocumentInput::HostReader { mime_type: None }) - .with_host_hooks(Arc::new(AdmissionSpy { effects: Arc::new(Mutex::new(0)) }), None); + .with_host_hooks( + Arc::new(AdmissionSpy { + effects: Arc::new(Mutex::new(0)), + }), + None, + ); let mut request = Some(request); - let NativeOutcome::Completed(mut call) = OcrCall::admit(crate::ocr::test_support::ocr_client(), OcrAdmission::all()) else { panic!("admission declined"); }; + let NativeOutcome::Completed(mut call) = + OcrCall::admit(crate::ocr::test_support::ocr_client(), OcrAdmission::all()) + else { + panic!("admission declined"); + }; let mut result = None; let mut reads = 0; let mut pre_calls = 0; @@ -791,18 +888,21 @@ mod tests { result = Some(match operation { OcrHostOperation::ProjectRequest => { assert_eq!(reads, 0); - OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false))) + projected_request(request.take().unwrap()) } OcrHostOperation::ReadDocument => { reads += 1; OcrHostResult::Document(Ok(crate::ocr::OcrFileContent { - bytes: bytes::Bytes::from_static(b"image"), file_name: Some("scan.png".into()), + bytes: bytes::Bytes::from_static(b"image"), + file_name: Some("scan.png".into()), })) } OcrHostOperation::PreCall(request) => { assert_eq!(reads, 1); pre_calls += 1; - assert!(matches!(&request.document, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2U=")); + assert!( + matches!(&request.document, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2U=") + ); OcrHostResult::PreCall(Ok(request)) } operation => NoopOcrHost.invoke(operation).await, @@ -812,6 +912,37 @@ mod tests { assert_eq!((reads, pre_calls, seen.lock().unwrap().len()), (1, 1, 1)); } + #[tokio::test] + async fn sdk_file_failure_dispatches_the_typed_error_once() { + struct CaptureFailure(Arc>>); + impl OcrHooks for CaptureFailure { + fn failure<'a>( + &'a self, + _: &'a CallLifecycleContext, + error: &'a crate::ocr::Error, + _: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + self.0.lock().unwrap().push(error.clone()); + Box::pin(async {}) + } + } + let path = + std::env::temp_dir().join(format!("missing-ocr-{:032x}.pdf", rand::random::())); + let failures = Arc::new(Mutex::new(Vec::new())); + let request = wire_request("mistral/model", "http://127.0.0.1:1", json!({})) + .with_document(path.clone().into()) + .with_host_hooks(Arc::new(CaptureFailure(failures.clone())), None); + let error = perform_ocr(request).await.unwrap_err(); + assert!( + matches!(error, crate::ocr::Error::FileRead { path: actual, .. } if actual == path) + ); + let failures = failures.lock().unwrap(); + assert_eq!(failures.len(), 1); + assert!( + matches!(&failures[0], crate::ocr::Error::FileRead { source, .. } if source.kind() == std::io::ErrorKind::NotFound) + ); + } + #[tokio::test] async fn cancellation_acknowledges_blocking_preparation_completion() { use std::sync::atomic::{AtomicBool, Ordering}; @@ -831,7 +962,8 @@ mod tests { std::future::poll_fn(|cx| { assert!(stop.as_mut().poll(cx).is_pending()); std::task::Poll::Ready(()) - }).await; + }) + .await; assert!(!finished.load(Ordering::SeqCst)); release_tx.send(()).unwrap(); stop.await; @@ -1029,6 +1161,8 @@ mod tests { mut request: OcrDuringCallRequest, ) -> OcrHookFuture<'_, OcrDuringCallRequest> { Box::pin(async move { + assert_eq!(request.optional_params["pages"], json!([0])); + assert_eq!(request.optional_params["extra_body"]["pages"], json!([2])); assert_eq!(request.body["pages"], json!([2])); assert_eq!(request.body.get("future"), Some(&Value::Null)); request.body.as_object_mut().unwrap().remove("future"); @@ -1282,10 +1416,7 @@ mod tests { result = Some(OcrHostResult::Lifecycle(Ok(()))) } OcrHostOperation::ProjectRequest => { - result = Some(OcrHostResult::Request(Ok(( - Box::new(request.take().unwrap()), - false, - )))) + result = Some(projected_request(request.take().unwrap())) } OcrHostOperation::AcquireAzureAdToken => { panic!("test request has no token provider") @@ -1307,7 +1438,9 @@ mod tests { Ok(request) })); } - OcrHostOperation::PostCall(_) | OcrHostOperation::ReadDocument => panic!("transport should not be reached"), + OcrHostOperation::PostCall(_) | OcrHostOperation::ReadDocument => { + panic!("transport should not be reached") + } }, Err(error) => break error, Ok(OcrCallStep::Complete(_)) => panic!("failed call completed"), @@ -1340,10 +1473,7 @@ mod tests { let error = loop { match call.resume(result.take()).await { Ok(OcrCallStep::Host(OcrHostOperation::ProjectRequest)) => { - result = Some(OcrHostResult::Request(Ok(( - Box::new(request.take().unwrap()), - false, - )))); + result = Some(projected_request(request.take().unwrap())); } Ok(OcrCallStep::Host(operation)) => { if let OcrHostOperation::PostCall(request) = &operation { @@ -1407,7 +1537,7 @@ mod tests { }); result = Some(match operation { OcrHostOperation::ProjectRequest => { - OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false))) + projected_request(request.take().unwrap()) } operation => host.invoke(operation).await, }); @@ -1467,7 +1597,9 @@ mod tests { OcrHostResult::Lifecycle(Err(HostFailure::Error(selected.clone()))) } OcrHostOperation::Failure { error, .. } => { - assert!(matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed")); + assert!( + matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed") + ); failures.push("sync"); OcrHostResult::Lifecycle(Err(HostFailure::Error( crate::ocr::Error::InvalidRequest("failure callback failed".into()), @@ -1483,7 +1615,7 @@ mod tests { panic!("finalization failure used provider/success dispatch") } OcrHostOperation::ProjectRequest => { - OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false))) + projected_request(request.take().unwrap()) } operation => host.invoke(operation).await, }); @@ -1493,7 +1625,9 @@ mod tests { } }; server.await.unwrap(); - assert!(matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed")); + assert!( + matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed") + ); assert_eq!(failures, ["sync", "async"]); assert_eq!(seen.lock().unwrap().len(), 1); } @@ -1520,10 +1654,7 @@ mod tests { match call.resume(result.take()).await.unwrap() { OcrCallStep::Host(OcrHostOperation::PreCall(_)) => break, OcrCallStep::Host(OcrHostOperation::ProjectRequest) => { - result = Some(OcrHostResult::Request(Ok(( - Box::new(request.take().unwrap()), - false, - )))) + result = Some(projected_request(request.take().unwrap())) } OcrCallStep::Host(operation) => result = Some(host.invoke(operation).await), OcrCallStep::Complete(_) => panic!("provider executed before pre-call result"), @@ -1746,7 +1877,7 @@ mod tests { _ = entered.notified() => break, step = call.resume(result.take()) => { result = Some(match step.unwrap() { - OcrCallStep::Host(OcrHostOperation::ProjectRequest) => OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false))), + OcrCallStep::Host(OcrHostOperation::ProjectRequest) => projected_request(request.take().unwrap()), OcrCallStep::Host(operation) => NoopOcrHost.invoke(operation).await, OcrCallStep::Complete(_) => panic!("pending provider completed"), }); @@ -1773,7 +1904,9 @@ mod tests { ) .await .unwrap(); - assert!(matches!(result, Err(crate::ocr::Error::InvalidRequest(message)) if message == "cancelled")); + assert!( + matches!(result, Err(crate::ocr::Error::InvalidRequest(message)) if message == "cancelled") + ); assert!( dropped.load(Ordering::SeqCst), "cancellation returned while provider captures were still alive" diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index eb3162cc79e..95bb827c104 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -18,12 +18,13 @@ pub use client::{OcrClient, ocr}; pub use document::{encode_file_document, mime_type_for_name, upload_mime_type}; pub use lifecycle::{ NativeOutcome, NativeResult, NoopOcrHost, OcrAdmission, OcrCall, OcrCallStep, OcrDecline, - OcrHookHost, OcrHost, OcrHostOperation, OcrHostResult, + OcrHookHost, OcrHost, OcrHostOperation, OcrHostResult, OcrProjectedRequest, }; pub use provider_config::{get_api_key_env_var, get_health_check_document}; pub use types::{ LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrConnectionInputs, OcrCredentialInputs, - OcrDocument, OcrDocumentInput, OcrFileContent, OcrPage, OcrPageDimensions, OcrPageImage, OcrTransportConfig, OcrUsageInfo, + OcrDocument, OcrDocumentInput, OcrFileContent, OcrPage, OcrPageDimensions, OcrPageImage, + OcrTransportConfig, OcrUsageInfo, }; #[cfg(test)] diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 785f4a761b6..19e8912ee7d 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -3,7 +3,7 @@ use serde_json::Value; use super::OcrClient; use super::hooks::OcrDuringCallRequest; -use super::types::{ResolvedOcrRequest, OcrConnection, OcrDocument, PreparedOcrRequest}; +use super::types::{OcrConnection, OcrDocument, PreparedOcrRequest, ResolvedOcrRequest}; pub(crate) async fn transform_request_body( client: &OcrClient, @@ -28,6 +28,7 @@ where .during_call(OcrDuringCallRequest { model: request.model.clone(), custom_llm_provider: request.provider_name().into(), + optional_params: Value::Object(request.optional_params.clone().into()), api_key: request.connection.api_key.clone(), url: url.into(), headers: headers.to_vec(), @@ -78,6 +79,7 @@ pub(crate) async fn guardrail_document( .during_call(OcrDuringCallRequest { model: request.model.clone(), custom_llm_provider: request.provider_name().into(), + optional_params: Value::Object(request.optional_params.clone().into()), api_key: request.connection.api_key.clone(), url: url.into(), headers: headers.to_vec(), diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index b7483224a32..d92cd546be4 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -3,8 +3,8 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -use serde::{Deserialize, Serialize}; use bytes::Bytes; +use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use serde_with::serde_as; @@ -93,7 +93,10 @@ impl From for OcrDocumentInput { impl From for OcrDocumentInput { fn from(path: PathBuf) -> Self { - Self::Path { path, mime_type: None } + Self::Path { + path, + mime_type: None, + } } } @@ -324,7 +327,6 @@ impl LiteLLMOcrRequest { config, }) } - } impl LiteLLMOcrRequest { @@ -383,7 +385,6 @@ impl LiteLLMOcrRequest { ..self } } - } impl LiteLLMOcrRequest { diff --git a/litellm-rust/crates/python-bridge/src/diagnostics.rs b/litellm-rust/crates/python-bridge/src/diagnostics.rs index cc153a89b8f..0c2b4c9e731 100644 --- a/litellm-rust/crates/python-bridge/src/diagnostics.rs +++ b/litellm-rust/crates/python-bridge/src/diagnostics.rs @@ -1,6 +1,6 @@ use litellm_python_interop::release_count; use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyTuple}; #[pyfunction] fn gil_stats(py: Python<'_>) -> PyResult> { @@ -9,6 +9,27 @@ fn gil_stats(py: Python<'_>) -> PyResult> { Ok(stats.into_any().unbind()) } +#[pyfunction] +fn _debug_setup( + py: Python<'_>, + call_type: String, + args: Bound<'_, PyTuple>, + kwargs: Bound<'_, PyDict>, + start: Py, + asynchronous: bool, +) -> PyResult<(Py, Py)> { + let leaked: &'static str = Box::leak(call_type.into_boxed_str()); + let result = crate::lifecycle::debug_setup( + py, + leaked, + &args.unbind(), + &kwargs.unbind(), + &start, + asynchronous, + )?; + Ok(result) +} + #[cfg(feature = "panic-test")] #[pyfunction] fn _panic_for_test() { @@ -17,6 +38,7 @@ fn _panic_for_test() { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(gil_stats, module)?)?; + module.add_function(wrap_pyfunction!(_debug_setup, module)?)?; #[cfg(feature = "panic-test")] module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?; Ok(()) diff --git a/litellm-rust/crates/python-bridge/src/lifecycle/bindings.rs b/litellm-rust/crates/python-bridge/src/lifecycle/bindings.rs index 06b32b67fd5..1a1ecbce116 100644 --- a/litellm-rust/crates/python-bridge/src/lifecycle/bindings.rs +++ b/litellm-rust/crates/python-bridge/src/lifecycle/bindings.rs @@ -1,7 +1,6 @@ -use pyo3::exceptions::PyBaseException; use pyo3::gc::{PyTraverseError, PyVisit}; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyTuple}; +use pyo3::types::PyDict; #[derive(FromPyObject)] pub(crate) struct PythonLogger(Py); @@ -19,34 +18,6 @@ impl PythonLogger { visit.call(&self.0) } - pub(crate) fn callbacks_needed(&self, py: Python<'_>, phase: &str) -> PyResult { - if !self - .object(py) - .getattr("_native_callback_fast_path") - .is_ok_and(|value| value.is_truthy().unwrap_or(false)) - { - return Ok(true); - } - py.import("litellm.rust_bridge.lifecycle")? - .getattr("callbacks_needed")? - .call1((self.object(py), phase))? - .extract() - } - - pub(super) fn success_bookkeeping( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - asynchronous: bool, - ) -> PyResult<()> { - py.import("litellm.rust_bridge.lifecycle")? - .getattr("success_bookkeeping")? - .call1((self.object(py), response, start, end, asynchronous))?; - Ok(()) - } - pub(super) fn defers_async_logging(&self, py: Python<'_>) -> bool { self.object(py) .getattr("_defer_async_logging") @@ -56,150 +27,10 @@ impl PythonLogger { pub(super) fn defer_success( &self, py: Python<'_>, - pending: Py, + pending: Py, ) -> PyResult<()> { self.object(py).setattr("_native_pending_logging", pending) } - - pub(super) fn sync_success_for_async_call( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()> { - if !self.callbacks_needed(py, "sync_success_async")? { - return Ok(()); - } - self.object(py).call_method1( - "handle_sync_success_callbacks_for_async_calls", - (response, start, end), - )?; - Ok(()) - } - - pub(super) fn failure( - &self, - py: Python<'_>, - error: &Py, - start: &Py, - end: &Option>, - asynchronous: bool, - ) -> PyResult>> { - if !self.callbacks_needed( - py, - if asynchronous { - "async_failure" - } else { - "sync_failure" - }, - )? { - py.import("litellm.rust_bridge.lifecycle")? - .getattr("failure_bookkeeping")? - .call1((self.object(py), error, start, end, asynchronous))?; - return Ok(None); - } - let trace = py - .import("traceback")? - .getattr("format_exception")? - .call1((error,))?; - let trace = pyo3::types::PyString::new(py, "").call_method1("join", (trace,))?; - let value = self.object(py).call_method1( - if asynchronous { - "async_failure_handler" - } else { - "failure_handler" - }, - (error, trace, start, end), - )?; - Ok(asynchronous.then(|| value.unbind())) - } - - pub(super) fn restore_context(&self, py: Python<'_>) -> PyResult<()> { - py.import("litellm.utils")? - .getattr("_restore_correlation_context_if_supported")? - .call1((self.object(py),))?; - Ok(()) - } - - pub(super) fn submit_success( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()> { - if !self.callbacks_needed(py, "sync_success")? { - return self.success_bookkeeping(py, response, start, end, false); - } - let context = py.import("contextvars")?.call_method0("copy_context")?; - py.import("litellm.litellm_core_utils.litellm_logging")? - .getattr("executor")? - .call_method1( - "submit", - ( - context.getattr("run")?, - self.object(py).getattr("success_handler")?, - response, - start, - end, - ), - )?; - Ok(()) - } - - pub(super) fn enqueue_success( - &self, - py: Python<'_>, - response: &Option>, - start: &Py, - end: &Option>, - ) -> PyResult<()> { - if !self.callbacks_needed(py, "async_success")? { - return self.success_bookkeeping(py, response, start, end, true); - } - let context = py.import("contextvars")?.call_method0("copy_context")?; - let worker = py - .import("litellm.litellm_core_utils.logging_worker")? - .getattr("GLOBAL_LOGGING_WORKER")? - .getattr("ensure_initialized_and_enqueue")?; - let coroutine = self - .object(py) - .call_method1("async_success_handler", (response, start, end))?; - let enqueue = context.call_method1("run", (worker, &coroutine)); - if enqueue.is_err() - && let Err(error) = coroutine.call_method0("close") - { - error.write_unraisable(py, Some(&coroutine)); - } - enqueue.map(|_| ()) - } -} - -pub(super) struct SetupResult<'py>(Bound<'py, PyAny>); - -impl SetupResult<'_> { - pub(super) fn logger(&self) -> PyResult { - self.0.getattr("logger")?.extract() - } - - pub(super) fn kwargs(&self) -> PyResult> { - Ok(self.0.getattr("kwargs")?.extract()?) - } -} - -pub(super) fn setup<'py>( - py: Python<'py>, - call_type: &str, - args: &Py, - kwargs: &Py, - start: &Py, - asynchronous: bool, -) -> PyResult> { - py.import("litellm.rust_bridge.lifecycle")? - .getattr("setup")? - .call1((call_type, args, kwargs, start, asynchronous)) - .map(SetupResult) } pub(super) fn finalize( @@ -210,9 +41,11 @@ pub(super) fn finalize( start: &Py, end: &Option>, ) -> PyResult<()> { - py.import("litellm.rust_bridge.lifecycle")? - .getattr("finalize")? - .call1((response, logger.object(py), kwargs, start, end))?; + let model = kwargs.bind(py).get_item("model")?; + let model = model.filter(|value| value.is_instance_of::()); + py.import("litellm.litellm_core_utils.llm_response_utils.response_metadata")? + .getattr("update_response_metadata")? + .call1((response, logger.object(py), model, kwargs, start, end))?; Ok(()) } @@ -222,170 +55,3 @@ pub(super) fn is_internal_call(py: Python<'_>) -> PyResult { .call_method0("get")? .extract() } - -pub(super) struct DeploymentHooks; - -impl DeploymentHooks { - pub(super) fn needed(py: Python<'_>) -> PyResult { - py.import("litellm.rust_bridge.lifecycle")? - .getattr("deployment_callbacks_needed")? - .call0()? - .extract() - } - - pub(super) fn before_call( - py: Python<'_>, - kwargs: &Py, - call_type: &str, - ) -> PyResult> { - py.import("litellm.utils")? - .getattr("async_pre_call_deployment_hook")? - .call1((kwargs, call_type)) - .map(Bound::unbind) - } - - pub(super) fn after_success( - py: Python<'_>, - kwargs: &Py, - response: &Option>, - call_type: &str, - ) -> PyResult> { - py.import("litellm.utils")? - .getattr("async_post_call_success_deployment_hook")? - .call1((kwargs, response, call_type)) - .map(Bound::unbind) - } - - pub(super) fn after_failure( - py: Python<'_>, - kwargs: &Py, - error: &Py, - call_type: &str, - ) -> PyResult> { - py.import("litellm.utils")? - .getattr("async_post_call_failure_deployment_hook")? - .call1((kwargs, error, call_type)) - .map(Bound::unbind) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pyo3::exceptions::PyTypeError; - - #[test] - fn setup_fields_are_checked_in_order_without_eager_logger_method_reads() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - pyo3::ffi::c_str!( - r#" -reads = [] -class Logger: - def __getattribute__(self, name): - reads.append(name) - raise AssertionError('logger methods must remain lazy') -logger = Logger() -class Setup: - @property - def logger(self): - reads.append('logger') - return logger - @property - def kwargs(self): - reads.append('kwargs') - return [] -result = Setup() -"# - ), - Some(&locals), - Some(&locals), - ) - .unwrap(); - let result = SetupResult(locals.get_item("result").unwrap().unwrap()); - let logger = result.logger().unwrap(); - assert!( - logger - .object(py) - .is(locals.get_item("logger").unwrap().unwrap()) - ); - assert_eq!( - locals - .get_item("reads") - .unwrap() - .unwrap() - .extract::>() - .unwrap(), - ["logger"] - ); - assert!( - result - .kwargs() - .unwrap_err() - .is_instance_of::(py) - ); - assert_eq!( - locals - .get_item("reads") - .unwrap() - .unwrap() - .extract::>() - .unwrap(), - ["logger", "kwargs"] - ); - }); - } - - #[test] - fn logger_resolves_each_callback_at_invocation_and_preserves_arguments() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - pyo3::ffi::c_str!( - r#" -calls = [] -response, start, end = object(), object(), object() -class Logger: - @property - def handle_sync_success_callbacks_for_async_calls(self): - generation = len(calls) - def callback(*args): - assert args == (response, start, end) - calls.append(generation) - return callback -logger = Logger() -"# - ), - Some(&locals), - Some(&locals), - ) - .unwrap(); - let logger: PythonLogger = locals - .get_item("logger") - .unwrap() - .unwrap() - .extract() - .unwrap(); - let response = Some(locals.get_item("response").unwrap().unwrap().unbind()); - let start = locals.get_item("start").unwrap().unwrap().unbind(); - let end = Some(locals.get_item("end").unwrap().unwrap().unbind()); - for _ in 0..2 { - logger - .sync_success_for_async_call(py, &response, &start, &end) - .unwrap(); - } - assert_eq!( - locals - .get_item("calls") - .unwrap() - .unwrap() - .extract::>() - .unwrap(), - [0, 1] - ); - }); - } -} diff --git a/litellm-rust/crates/python-bridge/src/lifecycle/compat.rs b/litellm-rust/crates/python-bridge/src/lifecycle/compat.rs new file mode 100644 index 00000000000..ce8d508a552 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/lifecycle/compat.rs @@ -0,0 +1,168 @@ +use litellm_core::call_lifecycle::{ + CallbackFamily, Delivery, ReleaseGate, SuccessFacts, plan_success, +}; +use pyo3::prelude::*; + +use super::bindings::PythonLogger; +use super::{PythonCallState, dispatch, missing_state}; + +pub(super) fn dispatch_success( + py: Python<'_>, + state: &PythonCallState, + logger: &PythonLogger, +) -> PyResult<()> { + let facts = SuccessFacts { + asynchronous: state.asynchronous, + internal: state.internal, + fallbacks: !state + .kwargs + .bind(py) + .get_item("fallbacks")? + .is_none_or(|value| value.is_none()), + deferred: logger.defers_async_logging(py), + sync_target_kinds: sync_kinds(py, logger)?, + }; + let leaves = dispatch::leaves(py)?; + let object = logger.object(py); + for selected in plan_success(&facts) { + match (selected.family, selected.delivery, selected.gate) { + (CallbackFamily::SyncSuccess, Delivery::Worker, _) => { + let handler = object.getattr("success_handler")?; + let bound = py.import("functools")?.getattr("partial")?.call1(( + handler, + &state.response, + &state.start, + &state.end, + ))?; + leaves.getattr("submit_worker")?.call1((bound,))?; + } + (CallbackFamily::AsyncSuccess, Delivery::Background, ReleaseGate::Immediate) => { + let coroutine = object.call_method1( + "async_success_handler", + (&state.response, &state.start, &state.end), + )?; + let enqueue = leaves.getattr("enqueue_background")?.call1((&coroutine,)); + if enqueue.is_err() + && let Err(error) = coroutine.call_method0("close") + { + error.write_unraisable(py, Some(&coroutine)); + } + enqueue?; + } + (CallbackFamily::AsyncSuccess, Delivery::Background, ReleaseGate::Deferred) => { + let pending = Py::new( + py, + SuppliedDeferred { + logger: Some(logger.clone_ref(py)), + response: state.response.as_ref().map(|value| value.clone_ref(py)), + start: state.start.clone_ref(py), + end: state.end.as_ref().map(|value| value.clone_ref(py)), + }, + )?; + object.setattr("_native_pending_logging", pending)?; + } + _ => return Err(missing_state()), + } + } + Ok(()) +} + +fn sync_kinds( + py: Python<'_>, + logger: &PythonLogger, +) -> PyResult> { + let (targets, ids) = dispatch::family_targets(py, logger, CallbackFamily::SyncSuccess)?; + Ok(targets.kinds(&ids)) +} + +pub(super) fn dispatch_failure( + py: Python<'_>, + state: &PythonCallState, + family: CallbackFamily, +) -> PyResult>> { + let logger = state.logger()?.object(py); + let error = state.error.as_ref().ok_or_else(missing_state)?; + let trace = py + .import("traceback")? + .getattr("format_exception")? + .call1((error,))?; + let trace = pyo3::types::PyString::new(py, "").call_method1("join", (trace,))?; + match family { + CallbackFamily::SyncFailure => { + logger.call_method1("failure_handler", (error, trace, &state.start, &state.end))?; + Ok(None) + } + CallbackFamily::AsyncFailure => Ok(Some( + logger + .call_method1( + "async_failure_handler", + (error, trace, &state.start, &state.end), + )? + .unbind(), + )), + _ => Err(missing_state()), + } +} + +#[pyclass] +struct SuppliedDeferred { + logger: Option, + response: Option>, + start: Py, + end: Option>, +} + +#[pymethods] +impl SuppliedDeferred { + fn __call__(slf: &Bound<'_, Self>, py: Python<'_>) -> PyResult<()> { + let logger = slf.borrow_mut().logger.take(); + let Some(logger) = logger else { + return Ok(()); + }; + let (response, start, end) = { + let this = slf.borrow(); + ( + this.response.as_ref().map(|value| value.clone_ref(py)), + this.start.clone_ref(py), + this.end.as_ref().map(|value| value.clone_ref(py)), + ) + }; + let object = logger.object(py); + let coroutine = object.call_method1("async_success_handler", (response, start, end))?; + let enqueue = dispatch::leaves(py)? + .getattr("enqueue_background")? + .call1((&coroutine,)); + if enqueue.is_err() + && let Err(error) = coroutine.call_method0("close") + { + error.write_unraisable(py, Some(&coroutine)); + } + match enqueue { + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(object)); + Ok(()) + } + result => result.map(|_| ()), + } + } + + fn __traverse__(&self, visit: pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> { + if let Some(logger) = &self.logger { + logger.traverse(&visit)?; + } + visit.call(&self.response)?; + visit.call(&self.start)?; + visit.call(&self.end) + } + + fn close(slf: &Bound<'_, Self>) { + let mut this = slf.borrow_mut(); + this.logger = None; + this.response = None; + this.end = None; + } + + fn __clear__(slf: &Bound<'_, Self>) { + Self::close(slf); + } +} diff --git a/litellm-rust/crates/python-bridge/src/lifecycle/dispatch.rs b/litellm-rust/crates/python-bridge/src/lifecycle/dispatch.rs new file mode 100644 index 00000000000..ca204d1d01b --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/lifecycle/dispatch.rs @@ -0,0 +1,1000 @@ +use litellm_core::call_lifecycle::{ + CallbackFamily, CallbackId, CallbackInvocation, CallbackKind, CallbackMethod, CursorFacts, + Delivery, Dispatch, DispatchCursor, DispatchFacts, DispatchStep, InvocationOutcome, + LoggedMarker, object_target_eligible, plan_request, +}; +use pyo3::exceptions::{PyBaseException, PyException, PyRuntimeError}; +use pyo3::gc::{PyTraverseError, PyVisit}; +use pyo3::prelude::*; +use pyo3::types::PyString; + +use super::bindings::PythonLogger; + +const LEAVES: &str = "litellm.rust_bridge.leaves"; + +pub(super) fn leaves(py: Python<'_>) -> PyResult> { + py.import(LEAVES) +} + +#[derive(Clone, Copy)] +pub(super) enum Outcome { + Success, + Failure, +} + +pub(super) struct Targets { + objects: Vec>, + kinds: Vec, +} + +impl Targets { + pub(super) fn read( + py: Python<'_>, + lists: &[Bound<'_, PyAny>], + ) -> PyResult<(Self, Vec>)> { + let custom_logger = py + .import("litellm.integrations.custom_logger")? + .getattr("CustomLogger")?; + let known = py + .import("litellm")? + .getattr("_known_custom_logger_compatible_callbacks")?; + let mut targets = Self { + objects: Vec::new(), + kinds: Vec::new(), + }; + let mut ids = Vec::with_capacity(lists.len()); + for list in lists { + let mut family = Vec::new(); + for object in list.try_iter()? { + let object = object?; + family.push(targets.intern(py, &object, &custom_logger, &known)?); + } + ids.push(family); + } + Ok((targets, ids)) + } + + fn intern( + &mut self, + py: Python<'_>, + object: &Bound<'_, PyAny>, + custom_logger: &Bound<'_, PyAny>, + known: &Bound<'_, PyAny>, + ) -> PyResult { + for (index, existing) in self.objects.iter().enumerate() { + let existing = existing.bind(py); + if existing.is(object) || existing.eq(object)? { + return Ok(CallbackId(index as u64)); + } + } + let kind = if object.is_instance(custom_logger)? { + CallbackKind::CustomLogger + } else if let Ok(name) = object.cast::() { + CallbackKind::Named { + known: known.contains(name)?, + } + } else if object.is_callable() { + CallbackKind::Callable { + internal: internal_callable(object)?, + } + } else { + CallbackKind::Opaque + }; + self.objects.push(object.clone().unbind()); + self.kinds.push(kind); + Ok(CallbackId((self.objects.len() - 1) as u64)) + } + + pub(super) fn kinds(&self, ids: &[CallbackId]) -> Vec { + ids.iter().map(|id| self.kinds[id.0 as usize]).collect() + } + + fn object<'py>(&self, py: Python<'py>, id: CallbackId) -> &Bound<'py, PyAny> { + self.objects[id.0 as usize].bind(py) + } + + fn kind(&self, id: CallbackId) -> CallbackKind { + self.kinds[id.0 as usize] + } + + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + for object in &self.objects { + visit.call(object)?; + } + Ok(()) + } +} + +fn internal_callable(object: &Bound<'_, PyAny>) -> PyResult { + let name = if let Ok(name) = object.getattr("__name__") { + name.extract::()? + } else if let Ok(func) = object.getattr("__func__") { + func.getattr("__name__")?.extract::()? + } else { + object.get_type().name()?.to_string() + }; + Ok([ + "_PROXY", + "_service_logger.ServiceLogging", + "sync_deployment_callback_on_success", + ] + .iter() + .any(|prefix| name.contains(prefix))) +} + +pub(super) struct Job { + pub logger: PythonLogger, + pub targets: Targets, + pub ids: Vec, + pub dispatch: Dispatch, + pub response: Option>, + pub error: Option>, + pub start: Py, + pub end: Py, + pub stream: bool, + pub sync_request: bool, +} + +impl Job { + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + self.logger.traverse(visit)?; + self.targets.traverse(visit)?; + visit.call(&self.response)?; + visit.call(&self.error)?; + visit.call(&self.start)?; + visit.call(&self.end) + } + + fn family_name(&self) -> &'static str { + match self.dispatch.family { + CallbackFamily::SyncSuccess => "sync_success", + CallbackFamily::AsyncSuccess => "async_success", + CallbackFamily::SyncFailure => "sync_failure", + CallbackFamily::AsyncFailure => "async_failure", + _ => "request", + } + } + + fn outcome(&self) -> Outcome { + match self.dispatch.family { + CallbackFamily::SyncSuccess | CallbackFamily::AsyncSuccess => Outcome::Success, + _ => Outcome::Failure, + } + } +} + +struct Eligibility<'a, 'py> { + py: Python<'py>, + job: &'a Job, + leaves: &'a Bound<'py, PyModule>, +} + +impl DispatchFacts for Eligibility<'_, '_> { + fn eligible(&mut self, target: CallbackId, method: CallbackMethod) -> bool { + let object = self.job.targets.object(self.py, target); + let kind = self.job.targets.kind(target); + if !object_target_eligible(self.job.sync_request, method, kind) { + return false; + } + let result = match method { + CallbackMethod::LoggingHook | CallbackMethod::AsyncLoggingHook => { + if kind != CallbackKind::CustomLogger { + return false; + } + self.leaves + .getattr("should_run_guardrail_hook") + .and_then(|f| f.call1((self.job.logger.object(self.py), object))) + .and_then(|v| v.extract::()) + } + CallbackMethod::LogSuccessEvent + | CallbackMethod::AsyncLogSuccessEvent + | CallbackMethod::LogFailureEvent + | CallbackMethod::AsyncLogFailureEvent => { + if kind == CallbackKind::Opaque { + return false; + } + self.leaves + .getattr("should_run_callback") + .and_then(|f| { + f.call1(( + self.job.logger.object(self.py), + object, + self.job.family_name(), + )) + }) + .and_then(|v| v.extract::()) + } + _ => Ok(kind != CallbackKind::Opaque), + }; + match result { + Ok(value) => value, + Err(error) => { + error.write_unraisable(self.py, Some(object)); + false + } + } + } +} + +pub(super) enum Step { + Await(Py), + Done, +} + +pub(super) struct Runner { + job: Job, + cursor: DispatchCursor, + result: Option>, + formatted: Option>, + pending: Option, +} + +impl Runner { + pub(super) fn start(py: Python<'_>, job: Job) -> PyResult { + let leaves = leaves(py)?; + let already_logged = match job.dispatch.family.marker() { + Some(marker) => leaves + .getattr("already_logged")? + .call1((job.logger.object(py), marker.key()))? + .extract::()?, + None => false, + }; + let cursor = DispatchCursor::start( + job.dispatch, + job.ids.clone(), + CursorFacts { + already_logged, + stream: job.stream, + sync_request: job.sync_request, + }, + ); + let result = job.response.as_ref().map(|value| value.clone_ref(py)); + Ok(Self { + job, + cursor, + result, + formatted: None, + pending: None, + }) + } + + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + self.job.traverse(visit)?; + visit.call(&self.result)?; + visit.call(&self.formatted) + } + + pub(super) fn logger<'py>(&self, py: Python<'py>) -> &Bound<'py, PyAny> { + self.job.logger.object(py) + } + + pub(super) fn resume( + &mut self, + py: Python<'_>, + awaited: Option>>, + ) -> PyResult { + let leaves = leaves(py)?; + if let Some(awaited) = awaited { + let invocation = self.pending.take().ok_or_else(super::missing_state)?; + let result = awaited.map(|value| { + (invocation.method == CallbackMethod::AsyncLoggingHook).then_some(value) + }); + self.accept(py, &leaves, invocation.target, result)?; + } + loop { + let step = self.cursor.next(&mut Eligibility { + py, + job: &self.job, + leaves: &leaves, + }); + match step { + DispatchStep::PrepareLogging => self.prepare(py, &leaves)?, + DispatchStep::MarkLogged(marker) => self.mark(py, &leaves, marker)?, + DispatchStep::Invoke(invocation) => { + let outcome = self.invoke(py, &leaves, invocation); + match outcome { + Ok(Some(awaitable)) => { + self.pending = Some(invocation); + return Ok(Step::Await(awaitable)); + } + Ok(None) => self.accept(py, &leaves, invocation.target, Ok(None))?, + Err(error) => self.accept(py, &leaves, invocation.target, Err(error))?, + } + } + DispatchStep::Complete { .. } => { + leaves + .getattr("restore_correlation_context")? + .call1((self.job.logger.object(py),))?; + return Ok(Step::Done); + } + } + } + } + + fn prepare(&mut self, py: Python<'_>, leaves: &Bound<'_, PyModule>) -> PyResult<()> { + let logger = self.job.logger.object(py); + let result = match self.job.outcome() { + Outcome::Success => leaves + .getattr("prepare_success_logging")? + .call1((logger, &self.result, &self.job.start, &self.job.end)) + .map(|redacted| self.result = Some(redacted.unbind())), + Outcome::Failure => leaves + .getattr("prepare_failure_logging")? + .call1((logger, &self.job.error, &self.job.start, &self.job.end)) + .map(|formatted| self.formatted = Some(formatted.unbind())), + }; + match result { + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(logger)); + Ok(()) + } + result => result, + } + } + + fn mark( + &self, + py: Python<'_>, + leaves: &Bound<'_, PyModule>, + marker: LoggedMarker, + ) -> PyResult<()> { + leaves + .getattr("mark_logged")? + .call1((self.job.logger.object(py), marker.key()))?; + Ok(()) + } + + fn invoke( + &mut self, + py: Python<'_>, + leaves: &Bound<'_, PyModule>, + invocation: CallbackInvocation, + ) -> PyResult>> { + let logger = self.job.logger.object(py); + let target = self.job.targets.object(py, invocation.target); + let kind = self.job.targets.kind(invocation.target); + let value = match (invocation.method, kind) { + (CallbackMethod::LoggingHook, CallbackKind::CustomLogger) => { + let replaced = + leaves + .getattr("logging_hook")? + .call1((logger, target, &self.result))?; + self.result = Some(replaced.unbind()); + return Ok(None); + } + (CallbackMethod::AsyncLoggingHook, CallbackKind::CustomLogger) => leaves + .getattr("async_logging_hook")? + .call1((logger, target, &self.result))?, + (CallbackMethod::LogSuccessEvent, CallbackKind::CustomLogger) => leaves + .getattr("log_success_event")? + .call1((logger, target, &self.result, &self.job.start, &self.job.end))?, + (CallbackMethod::AsyncLogSuccessEvent, CallbackKind::CustomLogger) => leaves + .getattr("async_log_success_event")? + .call1((logger, target, &self.result, &self.job.start, &self.job.end))?, + (CallbackMethod::LogFailureEvent, CallbackKind::CustomLogger) => leaves + .getattr("log_failure_event")? + .call1((logger, target, &self.job.start, &self.job.end))?, + (CallbackMethod::AsyncLogFailureEvent, CallbackKind::CustomLogger) => leaves + .getattr("async_log_failure_event")? + .call1((logger, target, &self.job.start, &self.job.end))?, + ( + CallbackMethod::LogSuccessEvent + | CallbackMethod::AsyncLogSuccessEvent + | CallbackMethod::LogFailureEvent + | CallbackMethod::AsyncLogFailureEvent, + CallbackKind::Callable { .. }, + ) => leaves.getattr("dispatch_callable")?.call1(( + logger, + target, + self.job.family_name(), + &self.result, + &self.job.start, + &self.job.end, + ))?, + ( + CallbackMethod::LogSuccessEvent | CallbackMethod::AsyncLogSuccessEvent, + CallbackKind::Named { .. }, + ) => leaves.getattr("dispatch_named_success")?.call1(( + logger, + target, + &self.result, + &self.job.start, + &self.job.end, + ))?, + ( + CallbackMethod::LogFailureEvent | CallbackMethod::AsyncLogFailureEvent, + CallbackKind::Named { .. }, + ) => leaves.getattr("dispatch_named_failure")?.call1(( + logger, + target, + &self.job.error, + &self.formatted, + &self.job.start, + &self.job.end, + ))?, + _ => return Ok(None), + }; + match invocation.delivery { + Delivery::Inline | Delivery::Worker => Ok(None), + Delivery::Await | Delivery::Background if value.is_none() => Ok(None), + Delivery::Await | Delivery::Background if value.hasattr("__await__")? => { + Ok(Some(value.unbind())) + } + Delivery::Await | Delivery::Background => Err(PyRuntimeError::new_err(format!( + "{:?} leaf for {:?} returned a non-awaitable {}", + invocation.method, + self.job.dispatch.family, + value.get_type().name()? + ))), + } + } + + fn accept( + &mut self, + py: Python<'_>, + leaves: &Bound<'_, PyModule>, + target: CallbackId, + result: PyResult>>, + ) -> PyResult<()> { + match result { + Ok(Some(replaced)) => { + self.result = Some(replaced); + self.cursor.accept(InvocationOutcome::Completed); + } + Ok(None) => self.cursor.accept(InvocationOutcome::Completed), + Err(error) if error.is_instance_of::(py) => { + let object = self.job.targets.object(py, target); + if let Err(report) = leaves.getattr("report_target_failure").and_then(|f| { + f.call1(( + self.job.logger.object(py), + object, + self.job.family_name(), + error.value(py), + )) + }) { + report.write_unraisable(py, Some(object)); + } + self.cursor.accept(InvocationOutcome::Failed); + } + Err(error) => return Err(error), + } + Ok(()) + } +} + +#[pyclass] +pub(super) struct WorkerJob { + runner: Option, +} + +impl WorkerJob { + pub(super) fn new(runner: Runner) -> Self { + Self { + runner: Some(runner), + } + } +} + +#[pymethods] +impl WorkerJob { + fn __call__(slf: &Bound<'_, Self>, py: Python<'_>) -> PyResult<()> { + let Some(mut runner) = slf.borrow_mut().runner.take() else { + return Ok(()); + }; + match runner.resume(py, None) { + Ok(Step::Done) => Ok(()), + Ok(Step::Await(_)) => Err(PyRuntimeError::new_err( + "worker dispatch selected an awaiting delivery", + )), + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(runner.job.logger.object(py))); + Ok(()) + } + Err(error) => Err(error), + } + } + + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + match &self.runner { + Some(runner) => runner.traverse(&visit), + None => Ok(()), + } + } + + fn __clear__(&mut self) { + self.runner = None; + } +} + +pub(super) struct AwaitingBody { + runner: Option, +} + +impl AwaitingBody { + pub(super) fn new(runner: Runner) -> Self { + Self { + runner: Some(runner), + } + } +} + +impl super::handle::ExecutionBody for AwaitingBody { + fn resume( + &mut self, + result: Option>>, + ) -> PyResult { + Python::attach(|py| { + let runner = self.runner.as_mut().ok_or_else(super::missing_state)?; + match runner.resume(py, result) { + Ok(Step::Await(awaitable)) => Ok(super::handle::ExecutionStep::Await(awaitable)), + Ok(Step::Done) => { + self.runner = None; + Ok(super::handle::ExecutionStep::Return(py.None())) + } + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(runner.job.logger.object(py))); + self.runner = None; + Ok(super::handle::ExecutionStep::Return(py.None())) + } + Err(error) => Err(error), + } + }) + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + match &self.runner { + Some(runner) => runner.traverse(visit), + None => Ok(()), + } + } +} + +pub(super) fn coroutine(py: Python<'_>, runner: Runner) -> PyResult> { + let execution = Py::new(py, super::handle::Execution::new(AwaitingBody::new(runner)))?; + py.import("litellm.rust_bridge.lifecycle")? + .getattr("drive")? + .call1((execution,)) + .map(Bound::unbind) +} + +pub(super) struct RequestJob<'a> { + pub logger: &'a PythonLogger, + pub family: CallbackFamily, +} + +pub(super) fn dispatch_request(py: Python<'_>, job: RequestJob<'_>) -> PyResult<()> { + let leaves = leaves(py)?; + let (targets, ids) = family_targets(py, job.logger, job.family)?; + let mut cursor = DispatchCursor::start( + plan_request(job.family), + ids, + CursorFacts { + already_logged: false, + stream: false, + sync_request: true, + }, + ); + let logger = job.logger.object(py); + let event = match job.family { + CallbackFamily::RequestPreCall => "pre_api_call", + _ => "post_api_call", + }; + let mut facts = RequestEligibility; + loop { + match cursor.next(&mut facts) { + DispatchStep::Invoke(invocation) => { + let target = targets.object(py, invocation.target); + let result = match (invocation.method, targets.kind(invocation.target)) { + (CallbackMethod::LogPreApiCall, CallbackKind::CustomLogger) => { + leaves.getattr("log_pre_api_call")?.call1((logger, target)) + } + (CallbackMethod::LogPostApiCall, CallbackKind::CustomLogger) => { + leaves.getattr("log_post_api_call")?.call1((logger, target)) + } + (_, CallbackKind::Named { .. }) => leaves + .getattr("dispatch_named_request")? + .call1((logger, target, event)), + (CallbackMethod::LogPreApiCall, CallbackKind::Callable { .. }) => leaves + .getattr("dispatch_callable_request")? + .call1((logger, target)), + _ => Ok(py.None().into_bound(py)), + }; + match result { + Ok(_) => cursor.accept(InvocationOutcome::Completed), + Err(error) if error.is_instance_of::(py) => { + if let Err(report) = leaves + .getattr("report_target_failure") + .and_then(|f| f.call1((logger, target, "request", error.value(py)))) + { + report.write_unraisable(py, Some(target)); + } + cursor.accept(InvocationOutcome::Failed); + } + Err(error) => return Err(error), + } + } + DispatchStep::Complete { .. } => return Ok(()), + DispatchStep::PrepareLogging | DispatchStep::MarkLogged(_) => {} + } + } +} + +struct RequestEligibility; + +impl DispatchFacts for RequestEligibility { + fn eligible(&mut self, _: CallbackId, _: CallbackMethod) -> bool { + true + } +} + +pub(super) fn read_lists<'py>( + py: Python<'py>, + logger: &PythonLogger, + family: CallbackFamily, +) -> PyResult<(Bound<'py, PyAny>, Option>)> { + let litellm = py.import("litellm")?; + let logger = logger.object(py); + let (global, dynamic) = match family { + CallbackFamily::RequestPreCall | CallbackFamily::RequestPostCall => { + ("input_callback", "dynamic_input_callbacks") + } + CallbackFamily::SyncSuccess => ("success_callback", "dynamic_success_callbacks"), + CallbackFamily::AsyncSuccess => { + ("_async_success_callback", "dynamic_async_success_callbacks") + } + CallbackFamily::SyncFailure => ("failure_callback", "dynamic_failure_callbacks"), + CallbackFamily::AsyncFailure => { + ("_async_failure_callback", "dynamic_async_failure_callbacks") + } + CallbackFamily::DeploymentPreCall + | CallbackFamily::DeploymentPostCall + | CallbackFamily::DeploymentFailure => ("callbacks", ""), + }; + let global = litellm.getattr(global)?; + let dynamic = if dynamic.is_empty() { + None + } else { + let value = logger.getattr(dynamic)?; + (!value.is_none()).then_some(value) + }; + Ok((global, dynamic)) +} + +pub(super) fn family_targets( + py: Python<'_>, + logger: &PythonLogger, + family: CallbackFamily, +) -> PyResult<(Targets, Vec)> { + let (global, dynamic) = read_lists(py, logger, family)?; + let lists: Vec> = std::iter::once(global).chain(dynamic.clone()).collect(); + let (targets, ids) = Targets::read(py, &lists)?; + let global_ids = ids.first().cloned().unwrap_or_default(); + let dynamic_ids = ids.get(1).map(Vec::as_slice); + let ordered = family.targets( + &global_ids, + dynamic.is_some().then_some(dynamic_ids.unwrap_or(&[])), + ); + Ok((targets, ordered)) +} + +pub(super) enum DeploymentEvent { + PreCall, + PostCall, + Failure { + request: Py, + exception: Py, + fallback_depth: Py, + }, +} + +pub(super) struct DeploymentBody { + logger: PythonLogger, + targets: Targets, + cursor: DispatchCursor, + event: DeploymentEvent, + call_type: &'static str, + current: Py, + kwargs: Py, + pending: Option, +} + +impl DeploymentBody { + pub(super) fn start( + py: Python<'_>, + logger: &PythonLogger, + family: CallbackFamily, + call_type: &'static str, + kwargs: &Py, + current: Py, + error: Option<&Py>, + ) -> PyResult { + let (targets, ids) = family_targets(py, logger, family)?; + let event = match family { + CallbackFamily::DeploymentPreCall => DeploymentEvent::PreCall, + CallbackFamily::DeploymentPostCall => DeploymentEvent::PostCall, + CallbackFamily::DeploymentFailure => { + let exception = error.ok_or_else(super::missing_state)?; + let view = leaves(py)? + .getattr("failure_deployment_hook_view")? + .call1((kwargs, exception))?; + let (request, snapshot, fallback_depth): (Py, Py, Py) = + view.extract()?; + DeploymentEvent::Failure { + request, + exception: snapshot + .into_bound(py) + .cast_into::()? + .unbind(), + fallback_depth, + } + } + _ => return Err(super::missing_state()), + }; + Ok(Self { + logger: logger.clone_ref(py), + targets, + cursor: DispatchCursor::start( + plan_request(family), + ids, + CursorFacts { + already_logged: false, + stream: false, + sync_request: true, + }, + ), + event, + call_type, + current, + kwargs: kwargs.clone_ref(py), + pending: None, + }) + } + + fn invoke(&self, py: Python<'_>, target: CallbackId) -> PyResult> { + let leaves = leaves(py)?; + let object = self.targets.object(py, target); + let awaitable = match &self.event { + DeploymentEvent::PreCall => leaves.getattr("pre_call_deployment_hook")?.call1(( + object, + &self.current, + self.call_type, + ))?, + DeploymentEvent::PostCall => leaves + .getattr("post_call_success_deployment_hook")? + .call1((object, &self.kwargs, &self.current, self.call_type))?, + DeploymentEvent::Failure { + request, + exception, + fallback_depth, + } => leaves + .getattr("post_call_failure_deployment_hook")? + .call1((object, request, exception, self.call_type, fallback_depth))?, + }; + Ok(awaitable.unbind()) + } + + fn accept( + &mut self, + py: Python<'_>, + target: CallbackId, + result: PyResult>, + ) -> PyResult<()> { + match result { + Ok(value) => { + if !value.is_none(py) && !matches!(self.event, DeploymentEvent::Failure { .. }) { + self.current = value; + } + self.cursor.accept(InvocationOutcome::Completed); + Ok(()) + } + Err(error) + if matches!(self.event, DeploymentEvent::Failure { .. }) + && error.is_instance_of::(py) => + { + let object = self.targets.object(py, target); + leaves(py)? + .getattr("report_deployment_failure_hook_error")? + .call1((object, error.value(py)))?; + self.cursor.accept(InvocationOutcome::Failed); + Ok(()) + } + Err(error) => Err(error), + } + } +} + +impl super::handle::ExecutionBody for DeploymentBody { + fn resume( + &mut self, + result: Option>>, + ) -> PyResult { + Python::attach(|py| { + if let Some(result) = result { + let target = self.pending.take().ok_or_else(super::missing_state)?; + self.accept(py, target, result)?; + } + let mut facts = DeploymentEligibility(&self.targets); + loop { + match self.cursor.next(&mut facts) { + DispatchStep::Invoke(invocation) => { + let awaitable = self.invoke(py, invocation.target)?; + self.pending = Some(invocation.target); + return Ok(super::handle::ExecutionStep::Await(awaitable)); + } + DispatchStep::Complete { .. } => { + return Ok(super::handle::ExecutionStep::Return( + self.current.clone_ref(py), + )); + } + DispatchStep::PrepareLogging | DispatchStep::MarkLogged(_) => {} + } + } + }) + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + self.logger.traverse(visit)?; + self.targets.traverse(visit)?; + visit.call(&self.current)?; + visit.call(&self.kwargs)?; + if let DeploymentEvent::Failure { + request, + exception, + fallback_depth, + } = &self.event + { + visit.call(request)?; + visit.call(exception)?; + visit.call(fallback_depth)?; + } + Ok(()) + } +} + +struct DeploymentEligibility<'a>(&'a Targets); + +impl DispatchFacts for DeploymentEligibility<'_> { + fn eligible(&mut self, target: CallbackId, _: CallbackMethod) -> bool { + self.0.kind(target) == CallbackKind::CustomLogger + } +} + +pub(super) fn deployment_coroutine(py: Python<'_>, body: DeploymentBody) -> PyResult> { + let execution = Py::new(py, super::handle::Execution::new(body))?; + py.import("litellm.rust_bridge.lifecycle")? + .getattr("drive")? + .call1((execution,)) + .map(Bound::unbind) +} + +#[cfg(test)] +mod tests { + use super::*; + use pyo3::types::{PyDict, PyList}; + + fn fixture(py: Python<'_>) -> Bound<'_, PyDict> { + let locals = PyDict::new(py); + py.run( + pyo3::ffi::c_str!( + r#" +import sys, types +for name in ("litellm", "litellm.integrations", "litellm.integrations.custom_logger"): + sys.modules.setdefault(name, types.ModuleType(name)) +class CustomLogger: pass +sys.modules["litellm.integrations.custom_logger"].CustomLogger = CustomLogger +sys.modules["litellm"]._known_custom_logger_compatible_callbacks = [] +class Logger: pass +logger = Logger() +target = CustomLogger() +"# + ), + Some(&locals), + Some(&locals), + ) + .unwrap(); + locals + } + + fn runner(py: Python<'_>, locals: &Bound<'_, PyDict>, family: CallbackFamily) -> Runner { + let logger = locals.get_item("logger").unwrap().unwrap(); + let target = locals.get_item("target").unwrap().unwrap(); + let list = PyList::new(py, [&target]).unwrap().into_any(); + let (targets, ids) = Targets::read(py, &[list]).unwrap(); + let ids: Vec = ids.into_iter().flatten().collect(); + let dispatch = Dispatch::immediate(family, Delivery::Worker); + Runner { + cursor: DispatchCursor::start( + dispatch, + ids.clone(), + CursorFacts { + already_logged: false, + stream: false, + sync_request: true, + }, + ), + job: Job { + logger: logger.extract().unwrap(), + targets, + ids, + dispatch, + sync_request: true, + response: Some(target.clone().unbind()), + error: None, + start: py.None(), + end: py.None(), + stream: false, + }, + result: Some(target.unbind()), + formatted: None, + pending: None, + } + } + + fn assert_collectable(py: Python<'_>, locals: &Bound<'_, PyDict>, handle: Py) { + locals.set_item("handle", handle).unwrap(); + py.run( + pyo3::ffi::c_str!( + r#" +import gc +import weakref +target.handle = handle +reference = weakref.ref(target) +del logger, target, handle +gc.collect() +assert reference() is None, "cycle through the retained target was not collected" +"# + ), + Some(locals), + Some(locals), + ) + .unwrap(); + } + + #[test] + fn worker_job_collects_cycles_through_logger_targets_and_response() { + Python::initialize(); + Python::attach(|py| { + let locals = fixture(py); + let runner = runner(py, &locals, CallbackFamily::SyncSuccess); + let handle = Py::new(py, WorkerJob::new(runner)).unwrap().into_any(); + assert_collectable(py, &locals, handle); + }); + } + + #[test] + fn deferred_success_collects_cycles_and_close_is_idempotent() { + Python::initialize(); + Python::attach(|py| { + let locals = fixture(py); + let runner = runner(py, &locals, CallbackFamily::AsyncSuccess); + let deferred = Py::new( + py, + super::super::DeferredSuccess { + runner: Some(runner), + }, + ) + .unwrap(); + assert_collectable(py, &locals, deferred.into_any()); + }); + } + + #[test] + fn deferred_success_releases_at_most_once_and_close_prevents_release() { + Python::initialize(); + Python::attach(|py| { + let locals = fixture(py); + let runner = runner(py, &locals, CallbackFamily::AsyncSuccess); + let deferred = Py::new( + py, + super::super::DeferredSuccess { + runner: Some(runner), + }, + ) + .unwrap(); + deferred.call_method0(py, "close").unwrap(); + deferred.call_method0(py, "close").unwrap(); + deferred.call0(py).unwrap(); + assert!(deferred.borrow(py).runner.is_none()); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs b/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs index 8c0a8b414ef..bf6c537e240 100644 --- a/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs +++ b/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs @@ -1,41 +1,40 @@ use std::sync::Arc; use std::task::Poll; -use futures_util::future::{AbortHandle, Abortable}; -#[cfg(test)] -use litellm_core::call_lifecycle::host::HostCallFuture; -use litellm_core::call_lifecycle::host::{ - HostCall as NativeCall, HostCallStep as NativeCallStep, HostFailure, HostPhase, HostStep, -}; use pyo3::exceptions::{PyBaseException, PyException, PyRuntimeError}; use pyo3::gc::{PyTraverseError, PyVisit}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; + +use futures_util::future::{AbortHandle, Abortable}; use tokio::sync::Mutex; -use crate::execution::{poll_async_value, run_async_value, run_sync_value}; +use litellm_core::call_lifecycle::host::{ + HostCall as NativeCall, HostCallStep as NativeCallStep, HostFailure, HostPhase, HostStep, +}; +use litellm_core::call_lifecycle::{ + CallbackFamily, Delivery, Dispatch, ReleaseGate, SuccessFacts, plan_failure, plan_success, +}; mod arguments; mod bindings; +mod compat; +mod dispatch; mod handle; mod preparation; +mod setup; +use crate::execution::{poll_async_value, run_async_value, run_sync_value}; pub(crate) use arguments::{BoundArguments, Signature}; -use bindings::DeploymentHooks; pub(crate) use bindings::PythonLogger; use handle::{Execution, ExecutionBody, ExecutionStep}; -pub(crate) enum OperationClass { - Phase(HostPhase), - Route, -} - pub(crate) trait PythonRoute: Send + Sync { type Call: NativeCall + 'static; fn state(&self) -> &PythonCallState; fn state_mut(&mut self) -> &mut PythonCallState; - fn classify(operation: &::Operation) -> OperationClass; + fn phase(operation: &::Operation) -> Option; fn lifecycle_result() -> ::Result; fn map_error(error: ::Error) -> PyErr; fn host_error(message: String) -> ::Error; @@ -98,6 +97,18 @@ pub(crate) fn run_call( } } +pub(crate) fn debug_setup( + py: Python<'_>, + call_type: &'static str, + args: &Py, + kwargs: &Py, + start: &Py, + asynchronous: bool, +) -> PyResult<(Py, Py)> { + let result = setup::setup(py, call_type, args, kwargs, start, asynchronous)?; + Ok((result.logger.object(py).clone().unbind(), result.kwargs)) +} + pub(crate) fn missing_state() -> PyErr { pyo3::exceptions::PyRuntimeError::new_err("missing native call state") } @@ -167,12 +178,12 @@ impl PythonLifecycle { HostFailure::Cancelled(native) }; let state = self.route.state_mut(); - if state.error.is_none() || (cancelled && phase != Some(HostPhase::DeploymentFailure)) { - state.retain_error(py, error); - } - if state.end.is_none() { - state.end = now(py).ok(); - } + state.retain_first_error( + py, + error, + cancelled && phase != Some(HostPhase::DeploymentFailure), + ); + let _ = state.finish(py); failure } @@ -215,10 +226,7 @@ impl PythonLifecycle { } HostStep::Ready(NativeCallStep::Host(operation)) => operation, }; - let phase = match R::classify(&operation) { - OperationClass::Phase(phase) => Some(phase), - OperationClass::Route => None, - }; + let phase = R::phase(&operation); let result = match phase { Some(phase) => match self.route.state_mut().invoke(py, phase) { Ok(HostStep::Suspend(awaitable)) => { @@ -295,6 +303,7 @@ pub(crate) struct PythonCallState { pub error: Option>, pub asynchronous: bool, pub internal: bool, + pub supplied: bool, pub call_type: &'static str, } @@ -314,42 +323,34 @@ impl PythonCallState { match phase { HostPhase::Setup => self.setup(py)?, HostPhase::DeploymentPreCall => { - if !DeploymentHooks::needed(py)? { - return Ok(HostStep::Ready(self.kwargs.clone_ref(py).into_any())); - } - return Ok(HostStep::Suspend(DeploymentHooks::before_call( + let copied = self.kwargs.bind(py).copy()?.into_any().unbind(); + return Ok(HostStep::Suspend(self.deployment( py, - &self.kwargs, - self.call_type, + CallbackFamily::DeploymentPreCall, + copied, )?)); } HostPhase::Prepare => self.prepare(py)?, HostPhase::DeploymentPostCall => { - if !DeploymentHooks::needed(py)? { - return self - .response - .as_ref() - .map(|value| HostStep::Ready(value.clone_ref(py))) - .ok_or_else(missing_state); - } - return Ok(HostStep::Suspend(DeploymentHooks::after_success( + let response = self + .response + .as_ref() + .map(|value| value.clone_ref(py)) + .unwrap_or_else(|| py.None()); + return Ok(HostStep::Suspend(self.deployment( py, - &self.kwargs, - &self.response, - self.call_type, + CallbackFamily::DeploymentPostCall, + response, )?)); } HostPhase::Finalize => self.finalize(py)?, HostPhase::Success => self.dispatch_success(py)?, HostPhase::DeploymentFailure => { - if let Some(error) = &self.error - && DeploymentHooks::needed(py)? - { - return Ok(HostStep::Suspend(DeploymentHooks::after_failure( + if self.error.is_some() { + return Ok(HostStep::Suspend(self.deployment( py, - &self.kwargs, - error, - self.call_type, + CallbackFamily::DeploymentFailure, + py.None(), )?)); } } @@ -368,6 +369,24 @@ impl PythonCallState { Ok(HostStep::Ready(py.None())) } + fn deployment( + &self, + py: Python<'_>, + family: CallbackFamily, + current: Py, + ) -> PyResult> { + let body = dispatch::DeploymentBody::start( + py, + self.logger()?, + family, + self.call_type, + &self.kwargs, + current, + self.error.as_ref(), + )?; + dispatch::deployment_coroutine(py, body) + } + fn accept(&mut self, py: Python<'_>, phase: HostPhase, value: Py) -> PyResult<()> { match phase { HostPhase::DeploymentPreCall => { @@ -396,6 +415,7 @@ impl PythonCallState { error: None, asynchronous, internal: false, + supplied: false, call_type, }) } @@ -409,7 +429,7 @@ impl PythonCallState { pub fn setup(&mut self, py: Python<'_>) -> PyResult<()> { self.start = now(py)?; self.internal = bindings::is_internal_call(py)?; - let result = bindings::setup( + let result = setup::setup( py, self.call_type, &self.args, @@ -417,8 +437,9 @@ impl PythonCallState { &self.start, self.asynchronous, )?; - self.logger = Some(result.logger()?); - self.kwargs = result.kwargs()?; + self.logger = Some(result.logger); + self.kwargs = result.kwargs; + self.supplied = result.supplied; Ok(()) } @@ -438,6 +459,16 @@ impl PythonCallState { ) } + pub fn dispatch_request(&self, py: Python<'_>, family: CallbackFamily) -> PyResult<()> { + dispatch::dispatch_request( + py, + dispatch::RequestJob { + logger: self.logger()?, + family, + }, + ) + } + pub fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> { match self.try_dispatch_success(py) { Err(error) if error.is_instance_of::(py) => { @@ -448,51 +479,76 @@ impl PythonCallState { } } + fn job(&self, py: Python<'_>, selected: Dispatch) -> PyResult { + let logger = self.logger()?; + let (targets, ids) = dispatch::family_targets(py, logger, selected.family)?; + let sync_request = dispatch::leaves(py)? + .getattr("is_sync_request")? + .call1((logger.object(py),))? + .extract()?; + Ok(dispatch::Job { + logger: logger.clone_ref(py), + targets, + ids, + dispatch: selected, + sync_request, + response: self.response.as_ref().map(|value| value.clone_ref(py)), + error: self.error.as_ref().map(|value| value.clone_ref(py)), + start: self.start.clone_ref(py), + end: self + .end + .as_ref() + .map(|value| value.clone_ref(py)) + .unwrap_or_else(|| py.None()), + stream: false, + }) + } + fn try_dispatch_success(&self, py: Python<'_>) -> PyResult<()> { let logger = self.logger()?; - let pending = || PendingSuccess { - logger: logger.clone_ref(py), - response: self.response.as_ref().map(|value| value.clone_ref(py)), - start: self.start.clone_ref(py), - end: self.end.as_ref().map(|value| value.clone_ref(py)), + if self.supplied { + return compat::dispatch_success(py, self, logger); + } + let (sync_targets, sync_ids) = + dispatch::family_targets(py, logger, CallbackFamily::SyncSuccess)?; + let facts = SuccessFacts { + asynchronous: self.asynchronous, + internal: self.internal, + fallbacks: !self + .kwargs + .bind(py) + .get_item("fallbacks")? + .is_none_or(|value| value.is_none()), + deferred: logger.defers_async_logging(py), + sync_target_kinds: sync_targets.kinds(&sync_ids), }; - if !self.asynchronous { - if !logger.callbacks_needed(py, "sync_success")? { - return logger.success_bookkeeping( - py, - &self.response, - &self.start, - &self.end, - false, - ); - } - pending().sync(py) - } else { - if !self.internal - && self - .kwargs - .bind(py) - .get_item("fallbacks")? - .is_none_or(|value| value.is_none()) - { - if !logger.callbacks_needed(py, "async_success")? { - logger.success_bookkeeping(py, &self.response, &self.start, &self.end, true)?; - } else if logger.defers_async_logging(py) { + for selected in plan_success(&facts) { + let runner = dispatch::Runner::start(py, self.job(py, selected)?)?; + match (selected.delivery, selected.gate) { + (Delivery::Worker, _) => { + let job = Py::new(py, dispatch::WorkerJob::new(runner))?; + dispatch::leaves(py)? + .getattr("submit_worker")? + .call1((job,))?; + } + (Delivery::Background, ReleaseGate::Immediate) => { + DeferredSuccess::release(py, runner)?; + } + (Delivery::Background, ReleaseGate::Deferred) => { logger.defer_success( py, Py::new( py, - PendingLogging { - pending: Some(pending()), + DeferredSuccess { + runner: Some(runner), }, )?, )?; - } else { - pending().asynchronous(py)?; } + (Delivery::Inline | Delivery::Await, _) => return Err(missing_state()), } - logger.sync_success_for_async_call(py, &self.response, &self.start, &self.end) } + Ok(()) } pub fn dispatch_failure( @@ -500,19 +556,39 @@ impl PythonCallState { py: Python<'_>, asynchronous: bool, ) -> PyResult>> { - if self.logger.is_none() || (self.asynchronous && self.internal) { + if self.logger.is_none() || self.error.is_none() { return Ok(None); } - let Some(error) = &self.error else { + let phase = if asynchronous { + HostPhase::AsyncFailure + } else { + HostPhase::Failure + }; + let Some(selected) = plan_failure(phase, self.asynchronous, self.internal) else { return Ok(None); }; - self.logger()? - .failure(py, error, &self.start, &self.end, asynchronous) + if self.supplied { + return compat::dispatch_failure(py, self, selected.family); + } + let mut runner = dispatch::Runner::start(py, self.job(py, selected)?)?; + match selected.delivery { + Delivery::Inline => match runner.resume(py, None)? { + dispatch::Step::Done => Ok(None), + dispatch::Step::Await(_) => Err(missing_state()), + }, + Delivery::Await => Ok(Some(dispatch::coroutine(py, runner)?)), + Delivery::Worker | Delivery::Background => Err(missing_state()), + } } pub fn cleanup(&mut self, py: Python<'_>) { if let Some(logger) = self.logger.take() - && let Err(error) = logger.restore_context(py) + && let Err(error) = dispatch::leaves(py).and_then(|leaves| { + leaves + .getattr("restore_correlation_context")? + .call1((logger.object(py),)) + .map(|_| ()) + }) { error.write_unraisable(py, None); } @@ -522,6 +598,19 @@ impl PythonCallState { self.error = Some(error.into_value(py)); } + pub fn retain_first_error(&mut self, py: Python<'_>, error: PyErr, replace: bool) { + if self.error.is_none() || replace { + self.retain_error(py, error); + } + } + + pub fn finish(&mut self, py: Python<'_>) -> PyResult<()> { + if self.end.is_none() { + self.end = Some(now(py)?); + } + Ok(()) + } + pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.args)?; visit.call(&self.kwargs)?; @@ -535,74 +624,96 @@ impl PythonCallState { } } -struct PendingSuccess { - logger: PythonLogger, - response: Option>, - start: Py, - end: Option>, -} - -impl PendingSuccess { - fn sync(&self, py: Python<'_>) -> PyResult<()> { - self.logger - .submit_success(py, &self.response, &self.start, &self.end) - } - - fn asynchronous(&self, py: Python<'_>) -> PyResult<()> { - self.logger - .enqueue_success(py, &self.response, &self.start, &self.end) - } -} - #[pyclass] -struct PendingLogging { - pending: Option, +struct DeferredSuccess { + runner: Option, +} + +impl DeferredSuccess { + fn release(py: Python<'_>, runner: dispatch::Runner) -> PyResult<()> { + let coroutine = dispatch::coroutine(py, runner)?; + let enqueue = dispatch::leaves(py)? + .getattr("enqueue_background")? + .call1((&coroutine,)); + if enqueue.is_err() + && let Err(error) = coroutine.call_method0(py, "close") + { + error.write_unraisable(py, Some(coroutine.bind(py))); + } + enqueue.map(|_| ()) + } } #[pymethods] -impl PendingLogging { - fn release(slf: &Bound<'_, Self>, py: Python<'_>, success: bool) -> PyResult<()> { - let pending = slf.borrow_mut().pending.take(); - if let Some(pending) = pending - && success - { - match pending.asynchronous(py) { - Err(error) if error.is_instance_of::(py) => { - error.write_unraisable(py, Some(pending.logger.object(py))); - } - result => return result, +impl DeferredSuccess { + fn __call__(slf: &Bound<'_, Self>, py: Python<'_>) -> PyResult<()> { + let runner = slf.borrow_mut().runner.take(); + let Some(runner) = runner else { + return Ok(()); + }; + let logger = runner.logger(py).clone().unbind(); + match Self::release(py, runner) { + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(logger.bind(py))); + Ok(()) } + result => result, } - Ok(()) } fn __traverse__(&self, visit: pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> { - if let Some(pending) = &self.pending { - pending.logger.traverse(&visit)?; - visit.call(&pending.response)?; - visit.call(&pending.start)?; - visit.call(&pending.end)?; + match &self.runner { + Some(runner) => runner.traverse(&visit), + None => Ok(()), } - Ok(()) + } + + fn close(slf: &Bound<'_, Self>) { + let runner = slf.borrow_mut().runner.take(); + drop(runner); } fn __clear__(slf: &Bound<'_, Self>) { - let pending = slf.borrow_mut().pending.take(); - drop(pending); + Self::close(slf); } } #[cfg(test)] mod tests { use super::*; + use pyo3::types::PyDict; use std::sync::Mutex; + use litellm_core::call_lifecycle::host::HostCallFuture; + static PYTHON_GLOBALS: Mutex<()> = Mutex::new(()); - fn install_logging_worker(py: Python<'_>, worker: &Bound<'_, PyAny>) -> PyResult<()> { - py.import("litellm.litellm_core_utils.logging_worker")? - .setattr("GLOBAL_LOGGING_WORKER", worker) + fn load_lifecycle_module(py: Python<'_>) -> Bound<'_, PyModule> { + py.run( + pyo3::ffi::c_str!( + r#" +import sys +import types +for name in ("litellm", "litellm.rust_bridge"): + sys.modules.setdefault(name, types.ModuleType(name)) +"# + ), + None, + None, + ) + .unwrap(); + let source = std::ffi::CString::new(include_str!( + "../../../../../litellm/rust_bridge/lifecycle.py" + )) + .unwrap(); + PyModule::from_code( + py, + &source, + pyo3::ffi::c_str!("lifecycle.py"), + pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), + ) + .unwrap() } struct RetainingHost { @@ -715,8 +826,8 @@ mod tests { &mut self.0 } - fn classify(_: &()) -> OperationClass { - OperationClass::Route + fn phase(_: &()) -> Option { + None } fn lifecycle_result() {} @@ -774,17 +885,7 @@ mod tests { .unwrap_or_else(|error| error.into_inner()); Python::initialize(); Python::attach(|py| { - let source = std::ffi::CString::new(include_str!( - "../../../../../litellm/rust_bridge/lifecycle.py" - )) - .unwrap(); - PyModule::from_code( - py, - &source, - pyo3::ffi::c_str!("lifecycle.py"), - pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), - ) - .unwrap(); + load_lifecycle_module(py); let route = SyntheticRoute( PythonCallState::new( py, @@ -820,17 +921,7 @@ mod tests { Python::initialize(); Python::attach(|py| { py.import("asyncio").unwrap(); - let source = std::ffi::CString::new(include_str!( - "../../../../../litellm/rust_bridge/lifecycle.py" - )) - .unwrap(); - let module = PyModule::from_code( - py, - &source, - pyo3::ffi::c_str!("lifecycle.py"), - pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), - ) - .unwrap(); + let module = load_lifecycle_module(py); let locals = PyDict::new(py); locals .set_item("drive", module.getattr("drive").unwrap()) @@ -938,64 +1029,11 @@ assert reference() is None error: None, asynchronous, internal: false, + supplied: false, call_type: "test", } } - #[test] - fn success_dispatch_reports_ordinary_failures_without_replacing_response() { - let _guard = PYTHON_GLOBALS - .lock() - .unwrap_or_else(|error| error.into_inner()); - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - pyo3::ffi::c_str!( - r#" -import sys - -response = object() -failure = ValueError('terminal diagnostic') -diagnostics = [] -old_hook = sys.unraisablehook -sys.unraisablehook = lambda event: diagnostics.append(event.exc_value) - -class Logger: - def handle_sync_success_callbacks_for_async_calls(self, *args): - raise failure - -logger = Logger() -"# - ), - Some(&locals), - Some(&locals), - ) - .unwrap(); - let response = locals.get_item("response").unwrap().unwrap().unbind(); - let mut lifecycle_state = state( - py, - locals.get_item("logger").unwrap().unwrap().unbind(), - response.clone_ref(py), - true, - ); - lifecycle_state.internal = true; - lifecycle_state.dispatch_success(py).unwrap(); - assert!(lifecycle_state.response.as_ref().unwrap().is(&response)); - py.run( - pyo3::ffi::c_str!( - r#" -assert diagnostics == [failure] -sys.unraisablehook = old_hook -"# - ), - Some(&locals), - Some(&locals), - ) - .unwrap(); - }); - } - #[test] fn retained_failure_preserves_exception_identity() { Python::initialize(); @@ -1011,139 +1049,6 @@ sys.unraisablehook = old_hook }); } - #[test] - fn deferred_release_uses_release_context_and_allows_reentry_once() { - let _guard = PYTHON_GLOBALS - .lock() - .unwrap_or_else(|error| error.into_inner()); - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - pyo3::ffi::c_str!( - r#" -import sys -import types -from contextvars import ContextVar - -litellm = types.ModuleType('litellm') -core_utils = types.ModuleType('litellm.litellm_core_utils') -logging_worker = types.ModuleType('litellm.litellm_core_utils.logging_worker') -litellm.litellm_core_utils = core_utils -core_utils.logging_worker = logging_worker -sys.modules['litellm'] = litellm -sys.modules['litellm.litellm_core_utils'] = core_utils -sys.modules['litellm.litellm_core_utils.logging_worker'] = logging_worker - -marker = ContextVar('marker', default='unset') -observed = [] - -class Coroutine: - def close(self): - observed.append('closed') - -class Worker: - def ensure_initialized_and_enqueue(self, coroutine): - observed.append(marker.get()) - pending.release(True) - coroutine.close() - -class Logger: - def async_success_handler(self, *args): - observed.append('created') - return Coroutine() - -worker = Worker() -logger = Logger() -"# - ), - Some(&locals), - Some(&locals), - ) - .unwrap(); - install_logging_worker(py, &locals.get_item("worker").unwrap().unwrap()).unwrap(); - let pending = Py::new( - py, - PendingLogging { - pending: Some(PendingSuccess { - logger: locals - .get_item("logger") - .unwrap() - .unwrap() - .extract() - .unwrap(), - response: Some(py.None()), - start: py.None(), - end: Some(py.None()), - }), - }, - ) - .unwrap(); - locals.set_item("pending", &pending).unwrap(); - py.run( - pyo3::ffi::c_str!( - r#" -marker.set('release') -pending.release(True) -pending.release(True) -assert observed == ['created', 'release', 'closed'] -"# - ), - Some(&locals), - Some(&locals), - ) - .unwrap(); - }); - } - - #[test] - fn deferred_logging_collects_cycles_through_typed_logger() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - pyo3::ffi::c_str!("class Logger: pass\nlogger = Logger()"), - Some(&locals), - Some(&locals), - ) - .unwrap(); - let pending = Py::new( - py, - PendingLogging { - pending: Some(PendingSuccess { - logger: locals - .get_item("logger") - .unwrap() - .unwrap() - .extract() - .unwrap(), - response: None, - start: py.None(), - end: None, - }), - }, - ) - .unwrap(); - locals.set_item("pending", pending).unwrap(); - py.run( - pyo3::ffi::c_str!( - r#" -import gc -import weakref -logger.pending = pending -reference = weakref.ref(logger) -del logger, pending -gc.collect() -assert reference() is None -"# - ), - Some(&locals), - Some(&locals), - ) - .unwrap(); - }); - } - #[test] fn coroutine_collects_cycles_retained_by_bridge_host() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/lifecycle/setup.rs b/litellm-rust/crates/python-bridge/src/lifecycle/setup.rs new file mode 100644 index 00000000000..c7370be628f --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/lifecycle/setup.rs @@ -0,0 +1,337 @@ +use litellm_core::call_lifecycle::CallbackId; +use litellm_core::call_lifecycle::registration::{ + Candidate, DynamicSuccessSlot, Entry, NamedEvent, Registration, RegistrationFacts, Registry, + RegistryMutation, classify_dynamic_success, plan_registration, +}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList, PyString, PyTuple}; + +use super::bindings::PythonLogger; + +const SETUP_MODULE: &str = "litellm.rust_bridge.setup"; + +struct Targets<'py> { + objects: Vec>, +} + +impl<'py> Targets<'py> { + fn new() -> Self { + Self { + objects: Vec::new(), + } + } + + fn intern(&mut self, object: &Bound<'py, PyAny>) -> CallbackId { + if let Some(index) = self + .objects + .iter() + .position(|existing| existing.is(object) || existing.eq(object).unwrap_or(false)) + { + return CallbackId(index as u64); + } + self.objects.push(object.clone()); + CallbackId((self.objects.len() - 1) as u64) + } + + fn get(&self, id: CallbackId) -> PyResult<&Bound<'py, PyAny>> { + self.objects + .get(id.0 as usize) + .ok_or_else(super::missing_state) + } +} + +fn registration<'py>( + setup: &Bound<'py, PyModule>, + object: &Bound<'py, PyAny>, +) -> PyResult { + if let Ok(name) = object.cast::() { + let name = name.to_str()?; + let known = setup.getattr("is_known_name")?.call1((name,))?.extract()?; + return Ok(Registration::Named { + known, + async_only: matches!(name, "dynamodb" | "openmeter"), + }); + } + let asynchronous = setup + .getattr("is_async_callable")? + .call1((object,))? + .extract()?; + Ok(Registration::Object { asynchronous }) +} + +fn entries<'py>( + setup: &Bound<'py, PyModule>, + targets: &mut Targets<'py>, + list: &Bound<'py, PyAny>, +) -> PyResult> { + list.try_iter()? + .map(|object| { + let object = object?; + Ok(Entry { + id: targets.intern(&object), + registration: registration(setup, &object)?, + }) + }) + .collect() +} + +fn registry_name(registry: Registry) -> &'static str { + match registry { + Registry::Input => "input", + Registry::AsyncInput => "async_input", + Registry::Success => "success", + Registry::AsyncSuccess => "async_success", + Registry::Failure => "failure", + Registry::AsyncFailure => "async_failure", + } +} + +fn read_registry<'py>(setup: &Bound<'py, PyModule>, name: &str) -> PyResult> { + setup.getattr("registry")?.call1((name,)) +} + +fn read_candidates<'py>( + setup: &Bound<'py, PyModule>, + targets: &mut Targets<'py>, + dynamic: Option>, +) -> PyResult> { + let mut candidates = Vec::new(); + let global = read_registry(setup, "callbacks")?; + let sources = std::iter::once(global).chain(dynamic); + for source in sources { + for object in source.try_iter()? { + let object = object?; + let candidate = if object.is_instance_of::() { + let resolved = setup + .getattr("resolve_named_integration")? + .call1((&object,))?; + if resolved.is_none() { + Candidate { + resolved: None, + duplicate_type: false, + } + } else { + let duplicate_type = setup + .getattr("async_success_registry_has_type")? + .call1((&resolved,))? + .extract()?; + Candidate { + resolved: Some(Entry { + id: targets.intern(&resolved), + registration: registration(setup, &resolved)?, + }), + duplicate_type, + } + } + } else { + Candidate { + resolved: Some(Entry { + id: targets.intern(&object), + registration: registration(setup, &object)?, + }), + duplicate_type: false, + } + }; + candidates.push(candidate); + } + } + Ok(candidates) +} + +fn apply<'py>( + setup: &Bound<'py, PyModule>, + targets: &Targets<'py>, + mutations: &[RegistryMutation], + function_id: Option<&Bound<'py, PyAny>>, +) -> PyResult<()> { + for mutation in mutations { + match mutation { + RegistryMutation::Append(registry, id) => { + setup + .getattr("append_registry")? + .call1((registry_name(*registry), targets.get(*id)?))?; + } + RegistryMutation::Remove(registry, id) => { + setup + .getattr("remove_registry")? + .call1((registry_name(*registry), targets.get(*id)?))?; + } + RegistryMutation::ExpandNamed(event, id) => { + let event = match event { + NamedEvent::Success => "success", + NamedEvent::Failure => "failure", + }; + setup + .getattr("expand_named")? + .call1((targets.get(*id)?, event))?; + } + RegistryMutation::Bootstrap => { + setup.getattr("bootstrap")?.call1((function_id,))?; + } + } + } + Ok(()) +} + +struct DynamicLists<'py> { + success: Option>, + async_success: Option>, + failure: Option>, +} + +fn split_dynamic<'py>( + py: Python<'py>, + setup: &Bound<'py, PyModule>, + targets: &mut Targets<'py>, + kwargs: &Bound<'py, PyDict>, +) -> PyResult> { + let success = match kwargs.get_item("success_callback")? { + Some(value) if value.is_instance_of::() => { + let list = value.cast_into::()?; + let sync = PyList::empty(py); + let asynchronous = PyList::empty(py); + for object in list.iter() { + let entry = Entry { + id: targets.intern(&object), + registration: registration(setup, &object)?, + }; + let named_async = object + .cast::() + .ok() + .and_then(|name| { + name.to_str() + .ok() + .map(|name| matches!(name, "dynamodb" | "s3")) + }) + .unwrap_or(false); + match classify_dynamic_success(entry, named_async) { + DynamicSuccessSlot::Sync => sync.append(&object)?, + DynamicSuccessSlot::Async => asynchronous.append(&object)?, + } + } + kwargs.del_item("success_callback")?; + Some((sync, (!asynchronous.is_empty()).then_some(asynchronous))) + } + _ => None, + }; + let failure = match kwargs.get_item("failure_callback")? { + Some(value) if value.is_instance_of::() => { + kwargs.del_item("failure_callback")?; + Some(value.cast_into::()?) + } + _ => None, + }; + let (success, async_success) = match success { + Some((sync, asynchronous)) => (Some(sync), asynchronous), + None => (None, None), + }; + Ok(DynamicLists { + success, + async_success, + failure, + }) +} + +pub(super) struct Setup { + pub logger: PythonLogger, + pub kwargs: Py, + pub supplied: bool, +} + +pub(super) fn setup( + py: Python<'_>, + call_type: &str, + args: &Py, + kwargs: &Py, + start: &Py, + asynchronous: bool, +) -> PyResult { + let setup = py.import(SETUP_MODULE)?; + let kwargs = kwargs.bind(py).copy()?; + if !kwargs.contains("litellm_call_id")? { + let call_id = py.import("uuid")?.call_method0("uuid4")?.str()?; + kwargs.set_item("litellm_call_id", call_id)?; + } + if let Some(supplied) = kwargs.get_item("litellm_logging_obj")? { + let logging_class = py + .import("litellm.litellm_core_utils.litellm_logging")? + .getattr("Logging")?; + if supplied.is_instance(&logging_class)? { + return Ok(Setup { + logger: supplied.extract()?, + kwargs: kwargs.unbind(), + supplied: true, + }); + } + } + + setup.getattr("prepare_environment")?.call0()?; + let guardrails = setup.getattr("applied_guardrails")?.call1((&kwargs,))?; + let function_id = kwargs.get_item("id")?; + + let mut targets = Targets::new(); + let dynamic = match kwargs.get_item("callbacks")? { + Some(value) => { + kwargs.del_item("callbacks")?; + (!value.is_none()).then_some(value) + } + None => None, + }; + let candidates = read_candidates(&setup, &mut targets, dynamic)?; + let facts = RegistrationFacts { + candidates, + input: entries(&setup, &mut targets, &read_registry(&setup, "input")?)?, + success: entries(&setup, &mut targets, &read_registry(&setup, "success")?)?, + failure: entries(&setup, &mut targets, &read_registry(&setup, "failure")?)?, + async_success: entries( + &setup, + &mut targets, + &read_registry(&setup, "async_success")?, + )? + .into_iter() + .map(|entry| entry.id) + .collect(), + async_failure: entries( + &setup, + &mut targets, + &read_registry(&setup, "async_failure")?, + )? + .into_iter() + .map(|entry| entry.id) + .collect(), + bootstrap_pending: setup.getattr("bootstrap_pending")?.call0()?.extract()?, + }; + apply( + &setup, + &targets, + &plan_registration(&facts), + function_id.as_ref(), + )?; + + let dynamic = split_dynamic(py, &setup, &mut targets, &kwargs)?; + setup.getattr("breadcrumb")?.call1((&kwargs,))?; + if let Some(logger_fn) = kwargs.get_item("logger_fn")? { + setup.getattr("logger_fn")?.call1((logger_fn,))?; + } + let model = match args.bind(py).get_item(0) { + Ok(model) => Some(model), + Err(_) => kwargs.get_item("model")?, + }; + let build = setup.getattr("build_logging")?; + let build_kwargs = PyDict::new(py); + build_kwargs.set_item("call_type", call_type)?; + build_kwargs.set_item("model", model)?; + build_kwargs.set_item("kwargs", &kwargs)?; + build_kwargs.set_item("start_time", start)?; + build_kwargs.set_item("asynchronous", asynchronous)?; + build_kwargs.set_item("dynamic_success", dynamic.success)?; + build_kwargs.set_item("dynamic_async_success", dynamic.async_success)?; + build_kwargs.set_item("dynamic_failure", dynamic.failure)?; + build_kwargs.set_item("guardrails", guardrails)?; + let logger = build.call((), Some(&build_kwargs))?; + Ok(Setup { + logger: logger.extract()?, + kwargs: kwargs.unbind(), + supplied: false, + }) +} diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index c96052aed97..4dac89b1b23 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -13,6 +13,11 @@ use litellm_python_interop::from_py_preserving_errors as from_py; use crate::auth::{AZURE_AD_TOKEN_PROVIDER, PythonTokenProvider}; use crate::lifecycle::BoundArguments; +pub(crate) struct Projection { + pub native: Native, + pub retained: Retained, +} + /// Fields every lifecycle route reads from its bound `*args, **kwargs` before /// asking core to build the typed request. Route-specific inputs (for example /// the OCR `document`) are read separately by the route. diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/callbacks.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/callbacks.rs index 94d7dc5ab63..217f69f592d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/callbacks.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/callbacks.rs @@ -3,158 +3,94 @@ use pyo3::types::PyDict; use serde_json::Value; use litellm_core::ocr::LiteLLMOcrResponse; -use litellm_core::ocr::hooks::OcrPreCallRequest; +use litellm_core::ocr::hooks::OcrDuringCallRequest; use litellm_python_interop::to_py_preserving_errors as to_py; -use crate::lifecycle::PythonLogger; +use litellm_core::call_lifecycle::CallbackFamily; -pub(super) struct OcrLoggingFields { - model: String, - custom_llm_provider: String, - optional_params: Value, +use super::host::PythonPayload; +use crate::lifecycle::{PythonCallState, PythonLogger}; + +pub(super) fn update_logging( + py: Python<'_>, + logger: &PythonLogger, + kwargs: &Py, + request: &OcrDuringCallRequest, + secret_fields: &[&str], +) -> PyResult<()> { + py.import("litellm.rust_bridge.ocr")? + .getattr("update_logging")? + .call1(( + logger.object(py), + kwargs, + &request.model, + &request.custom_llm_provider, + to_py(py, &request.optional_params)?, + secret_fields, + &request.url, + ))?; + Ok(()) } -impl From<&OcrPreCallRequest> for OcrLoggingFields { - fn from(request: &OcrPreCallRequest) -> Self { - Self { - model: request.model.clone(), - custom_llm_provider: request.custom_llm_provider.clone(), - optional_params: request.optional_params.clone(), - } - } -} - -impl PythonLogger { - pub(super) fn update_ocr( - &self, - py: Python<'_>, - kwargs: &Py, - pre_call: &OcrLoggingFields, - secret_fields: &[&str], - url: &str, - ) -> PyResult<()> { - let update = PyDict::new(py); - update.set_item("kwargs", redact(py, kwargs.bind(py), secret_fields)?)?; - update.set_item("model", &pre_call.model)?; - update.set_item( - "optional_params", - redact( - py, - &to_py(py, &pre_call.optional_params)? - .into_bound(py) - .cast_into::()?, - secret_fields, - )?, - )?; - let params = PyDict::new(py); - params.set_item( - "litellm_call_id", - kwargs.bind(py).get_item("litellm_call_id")?, - )?; - params.set_item("api_base", url)?; - for name in ["logger_fn", "litellm_request_debug"] { - if let Some(value) = kwargs.bind(py).get_item(name)? { - params.set_item(name, value)?; - } - } - for name in custom_pricing_fields(py)? { - if let Some(value) = kwargs.bind(py).get_item(&name)? - && !value.is_none() - { - params.set_item(name, value)?; - } - } - update.set_item("litellm_params", params)?; - update.set_item("custom_llm_provider", &pre_call.custom_llm_provider)?; - self.object(py) - .call_method("update_from_kwargs", (), Some(&update))?; - Ok(()) - } - - pub(crate) fn pre_ocr( - &self, - py: Python<'_>, - api_key: Option<&str>, - body: &Bound<'_, PyDict>, - headers: &Bound<'_, PyDict>, - url: &str, - ) -> PyResult<()> { - let additional = PyDict::new(py); - additional.set_item("complete_input_dict", body)?; - additional.set_item("headers", headers)?; - additional.set_item("api_base", url)?; +pub(super) fn pre_call( + py: Python<'_>, + state: &PythonCallState, + request: &OcrDuringCallRequest, + payload: &PythonPayload, +) -> PyResult<()> { + let logger = state.logger()?; + if state.supplied { + let additional_args = PyDict::new(py); + additional_args.set_item("complete_input_dict", &payload.body)?; + additional_args.set_item("headers", &payload.headers)?; + additional_args.set_item("api_base", &request.url)?; let kwargs = PyDict::new(py); kwargs.set_item("input", "OCR document processing")?; - kwargs.set_item("api_key", api_key)?; - kwargs.set_item("additional_args", &additional)?; - if self.callbacks_needed(py, "input")? { - self.object(py).call_method("pre_call", (), Some(&kwargs))?; - } else { - self.object(py) - .call_method("_pre_call", (), Some(&kwargs))?; - self.object(py).call_method0("record_api_call_start_time")?; - } - Ok(()) - } - - pub(crate) fn post_ocr( - &self, - py: Python<'_>, - original_response: &Value, - body: Option<&Py>, - headers: Option<&Py>, - ) -> PyResult<()> { - let additional = PyDict::new(py); - additional.set_item("complete_input_dict", body)?; - additional.set_item("headers", headers)?; - if self.callbacks_needed(py, "input")? { - let kwargs = PyDict::new(py); - kwargs.set_item("original_response", to_py(py, original_response)?)?; - kwargs.set_item("additional_args", &additional)?; - self.object(py) - .call_method("post_call", (), Some(&kwargs))?; - } else { - let response = py - .import("json")? - .call_method1("dumps", (to_py(py, original_response)?,))?; - self.object(py).call_method1( - "record_post_call", - (response, py.None(), py.None(), additional), - )?; - } - Ok(()) + kwargs.set_item("api_key", request.api_key.as_deref())?; + kwargs.set_item("additional_args", additional_args)?; + logger + .object(py) + .call_method("pre_call", (), Some(&kwargs))?; + return Ok(()); } + let kwargs = PyDict::new(py); + kwargs.set_item("api_key", request.api_key.as_deref())?; + kwargs.set_item("body", &payload.body)?; + kwargs.set_item("headers", &payload.headers)?; + kwargs.set_item("url", &request.url)?; + py.import("litellm.rust_bridge.leaves")? + .getattr("record_pre_call")? + .call((logger.object(py),), Some(&kwargs))?; + state.dispatch_request(py, CallbackFamily::RequestPreCall) } -fn custom_pricing_fields(py: Python<'_>) -> PyResult> { - py.import("litellm.types.utils")? - .getattr("CustomPricingLiteLLMParams")? - .getattr("model_fields")? - .cast_into::()? - .keys() - .iter() - .map(|name| name.extract::()) - .collect() -} - -fn redact( +pub(super) fn post_call( py: Python<'_>, - params: &Bound<'_, PyDict>, - secret_fields: &[&str], -) -> PyResult> { - let redacted = PyDict::new(py); - for (name, value) in params { - let name = name.extract::()?; - if name == "proxy_server_request" { - continue; - } - if secret_fields.contains(&name.as_str()) { - redacted.set_item(name, "****")?; - } else { - redacted.set_item(name, value)?; - } + state: &PythonCallState, + original_response: &Value, + payload: &PythonPayload, +) -> PyResult<()> { + let logger = state.logger()?; + if state.supplied { + let additional_args = PyDict::new(py); + additional_args.set_item("complete_input_dict", &payload.body)?; + additional_args.set_item("headers", &payload.headers)?; + let kwargs = PyDict::new(py); + kwargs.set_item("original_response", to_py(py, original_response)?)?; + kwargs.set_item("additional_args", additional_args)?; + logger + .object(py) + .call_method("post_call", (), Some(&kwargs))?; + return Ok(()); } - Ok(redacted.unbind()) + let kwargs = PyDict::new(py); + kwargs.set_item("original_response", to_py(py, original_response)?)?; + kwargs.set_item("body", &payload.body)?; + kwargs.set_item("headers", &payload.headers)?; + py.import("litellm.rust_bridge.leaves")? + .getattr("record_post_call")? + .call((logger.object(py),), Some(&kwargs))?; + state.dispatch_request(py, CallbackFamily::RequestPostCall) } pub(super) fn response(py: Python<'_>, response: &LiteLLMOcrResponse) -> PyResult> { diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs index 6df86e0dc54..8d414435e9d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs @@ -36,11 +36,11 @@ fn extract_bytes(value: &Bound<'_, PyAny>) -> PyResult { return Ok(Bytes::from_owner(value.extract::()?)); } // Bytes subclasses may retain GC edges that a native Bytes owner cannot traverse. - Ok(Bytes::copy_from_slice(value.extract::()?.as_ref())) + Ok(Bytes::copy_from_slice( + value.extract::()?.as_ref(), + )) } - - pub(super) struct FileDocumentInput { pub input: OcrDocumentInput, pub reader: Option, @@ -56,9 +56,11 @@ impl FromPyObject<'_, '_> for FileDocumentInput { Err(error) if error.is_instance_of::(py) => None, Err(error) => return Err(error), }; - let missing = || PyValueError::new_err( - "document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes", - ); + let missing = || { + PyValueError::new_err( + "document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes", + ) + }; let file = document.get_item("file").map_err(|error| { if error.is_instance_of::(py) { missing() @@ -93,7 +95,9 @@ impl FromPyObject<'_, '_> for FileDocumentInput { reader: None, }); } - let reader = file.getattr_opt("read")?.filter(|value| value.is_callable()); + let reader = file + .getattr_opt("read")? + .filter(|value| value.is_callable()); let Some(reader) = reader else { return Err(PyValueError::new_err(format!( "Unsupported file input type: {}. Expected pathlib.Path, bytes, or a file-like object.", @@ -107,7 +111,10 @@ impl FromPyObject<'_, '_> for FileDocumentInput { .transpose()?; Ok(Self { input: OcrDocumentInput::HostReader { mime_type }, - reader: Some(PythonFileReader { reader: reader.unbind(), name }), + reader: Some(PythonFileReader { + reader: reader.unbind(), + name, + }), }) } } @@ -127,12 +134,21 @@ mod tests { let error = document.extract::().err().unwrap(); assert!(error.is_instance_of::(py)); } - for expression in [c"{'file': b'abc', 'mime_type': None}", c"{'file': b'abc', 'mime_type': 7}"] { - let error = py.eval(expression, None, None).unwrap().extract::().err().unwrap(); + for expression in [ + c"{'file': b'abc', 'mime_type': None}", + c"{'file': b'abc', 'mime_type': 7}", + ] { + let error = py + .eval(expression, None, None) + .unwrap() + .extract::() + .err() + .unwrap(); assert!(error.is_instance_of::(py)); } let locals = PyDict::new(py); - py.run(c"from pathlib import Path + py.run( + c"from pathlib import Path failure = KeyError('reader failed') class Reader: def __init__(self): @@ -143,12 +159,31 @@ class Reader: reader = Reader() document = {'file': reader} path_document = {'file': Path('/nonexistent/ocr-projection-test.pdf')} -", Some(&locals), Some(&locals)).unwrap(); +", + Some(&locals), + Some(&locals), + ) + .unwrap(); let document = locals.get_item("document").unwrap().unwrap(); let input: FileDocumentInput = document.extract().unwrap(); - assert_eq!(locals.get_item("reader").unwrap().unwrap().getattr("reads").unwrap().extract::().unwrap(), 0); + assert_eq!( + locals + .get_item("reader") + .unwrap() + .unwrap() + .getattr("reads") + .unwrap() + .extract::() + .unwrap(), + 0 + ); assert!(input.reader.is_some()); - let path: FileDocumentInput = locals.get_item("path_document").unwrap().unwrap().extract().unwrap(); + let path: FileDocumentInput = locals + .get_item("path_document") + .unwrap() + .unwrap() + .extract() + .unwrap(); assert!(matches!(path.input, OcrDocumentInput::Path { .. })); }); } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs index 655cd9ea1ee..c7f239054de 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs @@ -103,9 +103,13 @@ pub(super) fn public_exception( std::io::ErrorKind::PermissionDenied => 13, _ => 5, }); - return Ok(PyErr::from_value(py.import("builtins")?.getattr("OSError")?.call1(( - errno, source.to_string(), path, - ))?)); + return Ok(PyErr::from_value( + py.import("builtins")?.getattr("OSError")?.call1(( + errno, + source.to_string(), + path.into_pyobject(py)?.call_method0("__fspath__")?, + ))?, + )); } raise_public(py, classify(error), model, provider, None) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index 3cbe4e03a10..9fcd53569bc 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -1,11 +1,11 @@ -use std::sync::Arc; - +use pyo3::gc::{PyTraverseError, PyVisit}; use pyo3::prelude::*; use pyo3::types::PyDict; use litellm_auth::ResolvedCredential; +use litellm_core::call_lifecycle::host::HostPhase; use litellm_core::ocr::hooks::{OcrDuringCallRequest, OcrPostCallRequest}; -use litellm_core::ocr::{OcrCall, OcrHostOperation, OcrHostResult}; +use litellm_core::ocr::{OcrCall, OcrHostOperation, OcrHostResult, OcrProjectedRequest}; use litellm_python_interop::{ from_py_preserving_errors as from_py, to_py_preserving_errors as to_py, }; @@ -13,75 +13,100 @@ use litellm_python_interop::{ use super::document::PythonFileReader; use super::errors::to_pyerr as ocr_error_to_pyerr; use super::project::project; -use super::{ASYNC_SIGNATURE, SIGNATURE, callbacks, errors}; +use super::{callbacks, errors}; use crate::auth::PythonTokenProvider; -use crate::lifecycle::{OperationClass, PythonCallState, PythonRoute, missing_state, now}; +use crate::lifecycle::{PythonCallState, PythonRoute, Signature, missing_state}; +use crate::marshal::Projection; pub(super) struct PythonOcrHost { state: PythonCallState, - projected: Option, + signature: &'static Signature, + retained: Option, } -struct ProjectedOcrHost { - model: String, - provider: &'static str, - secret_fields: Vec<&'static str>, - azure_ad_token_provider: Option, - pre_call: Option, - reader: Option, - reader_failed: bool, - body: Option>, - headers: Option>, +pub(super) struct OcrRetained { + pub model: String, + pub provider: &'static str, + pub secret_fields: Vec<&'static str>, + pub azure_ad_token_provider: Option, + pub payload: Option, + pub reader: Option, +} + +pub(super) struct PythonPayload { + pub body: Py, + pub headers: Py, +} + +impl PythonPayload { + fn from_request(py: Python<'_>, request: &OcrDuringCallRequest) -> PyResult { + let body = to_py(py, &request.body)? + .into_bound(py) + .cast_into::()?; + let headers = PyDict::new(py); + for (name, value) in &request.headers { + headers.set_item(name, value)?; + } + Ok(Self { + body: body.unbind(), + headers: headers.unbind(), + }) + } + + fn write_back( + &self, + py: Python<'_>, + mut request: OcrDuringCallRequest, + ) -> PyResult { + request.body = from_py(self.body.bind(py))?; + request.headers = self + .headers + .bind(py) + .iter() + .map(|(name, value)| Ok((name.extract::()?, value.extract::()?))) + .collect::>>()?; + Ok(request) + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.body)?; + visit.call(&self.headers) + } } impl PythonOcrHost { - pub(super) fn new(state: PythonCallState) -> Self { + pub(super) fn new(state: PythonCallState, signature: &'static Signature) -> Self { Self { state, - projected: None, + signature, + retained: None, } } - fn projected(&self) -> PyResult<&ProjectedOcrHost> { - self.projected.as_ref().ok_or_else(missing_state) + fn retained(&self) -> PyResult<&OcrRetained> { + self.retained.as_ref().ok_or_else(missing_state) } - fn projected_mut(&mut self) -> PyResult<&mut ProjectedOcrHost> { - self.projected.as_mut().ok_or_else(missing_state) + fn retained_mut(&mut self) -> PyResult<&mut OcrRetained> { + self.retained.as_mut().ok_or_else(missing_state) } fn project(&mut self, py: Python<'_>) -> PyResult { - let signature = if self.state.asynchronous { - &ASYNC_SIGNATURE - } else { - &SIGNATURE - }; - let arguments = signature.bind(self.state.args.bind(py), self.state.kwargs.bind(py))?; - let projected = project(py, &arguments)?; - let has_token_provider = projected.azure_ad_token_provider.is_some(); - self.projected = Some(ProjectedOcrHost { - model: projected.request.model.clone(), - provider: projected.request.provider_name(), - secret_fields: projected.secret_fields, - azure_ad_token_provider: projected.azure_ad_token_provider, - pre_call: None, - reader: projected.reader, - reader_failed: false, - body: None, - headers: None, - }); - Ok(OcrHostResult::Request(Ok(( - Box::new( - projected - .request - .with_host_hooks(Arc::new(BridgeOcrHooks), None), - ), - has_token_provider, - )))) + let arguments = self + .signature + .bind(self.state.args.bind(py), self.state.kwargs.bind(py))?; + let Projection { native, retained } = project(py, &arguments)?; + let host_token_provider = retained.azure_ad_token_provider.is_some(); + self.retained = Some(retained); + Ok(OcrHostResult::Request(Ok(OcrProjectedRequest { + request: Box::new(native), + intercepts_requests: true, + host_token_provider, + }))) } fn acquire_azure_ad_token(&self, py: Python<'_>) -> PyResult { - self.projected()? + self.retained()? .azure_ad_token_provider .as_ref() .ok_or_else(missing_state)? @@ -91,34 +116,21 @@ impl PythonOcrHost { fn during_call( &mut self, py: Python<'_>, - mut request: OcrDuringCallRequest, + request: OcrDuringCallRequest, ) -> PyResult { - let projected = self.projected()?; - let pre_call = projected.pre_call.as_ref().ok_or_else(missing_state)?; + let retained = self.retained()?; let logger = self.state.logger()?; - logger.update_ocr( + callbacks::update_logging( py, + logger, &self.state.kwargs, - pre_call, - &projected.secret_fields, - &request.url, + &request, + &retained.secret_fields, )?; - let body = to_py(py, &request.body)? - .into_bound(py) - .cast_into::()?; - let headers = PyDict::new(py); - for (name, value) in &request.headers { - headers.set_item(name, value)?; - } - logger.pre_ocr(py, request.api_key.as_deref(), &body, &headers, &request.url)?; - request.body = from_py(&body)?; - request.headers = headers - .iter() - .map(|(name, value)| Ok((name.extract::()?, value.extract::()?))) - .collect::>>()?; - let projected = self.projected_mut()?; - projected.body = Some(body.unbind()); - projected.headers = Some(headers.unbind()); + let payload = PythonPayload::from_request(py, &request)?; + callbacks::pre_call(py, &self.state, &request, &payload)?; + let request = payload.write_back(py, request)?; + self.retained_mut()?.payload = Some(payload); Ok(request) } @@ -127,26 +139,23 @@ impl PythonOcrHost { py: Python<'_>, request: OcrPostCallRequest, ) -> PyResult { - let projected = self.projected()?; - self.state.logger()?.post_ocr( - py, - &request.original_response, - projected.body.as_ref(), - projected.headers.as_ref(), - )?; + let payload = self + .retained()? + .payload + .as_ref() + .ok_or_else(missing_state)?; + callbacks::post_call(py, &self.state, &request.original_response, payload)?; Ok(request) } fn map_failure(&mut self, py: Python<'_>, error: litellm_core::ocr::Error) -> PyResult<()> { - if self.state.end.is_none() { - self.state.end = Some(now(py)?); - } - let (model, provider) = match &self.projected { - Some(projected) => (projected.model.as_str(), projected.provider), + self.state.finish(py)?; + let (model, provider) = match &self.retained { + Some(retained) => (retained.model.as_str(), retained.provider), None => ("", ""), }; let mapped = match self.state.error.take() { - Some(host_error) if self.projected.as_ref().is_some_and(|host| host.reader_failed) => { + Some(host_error) if matches!(error, litellm_core::ocr::Error::HostDocumentRead) => { PyErr::from_value(host_error.into_bound(py).into_any()) } Some(host_error) => errors::public_host_exception(py, &host_error, model, provider)?, @@ -168,10 +177,8 @@ impl PythonRoute for PythonOcrHost { &mut self.state } - fn classify(operation: &OcrHostOperation) -> OperationClass { - operation - .phase() - .map_or(OperationClass::Route, OperationClass::Phase) + fn phase(operation: &OcrHostOperation) -> Option { + operation.phase() } fn lifecycle_result() -> OcrHostResult { @@ -190,18 +197,24 @@ impl PythonRoute for PythonOcrHost { Ok(match operation { OcrHostOperation::ProjectRequest => self.project(py)?, OcrHostOperation::ReadDocument => { - let reader = self.projected_mut()?.reader.take().ok_or_else(missing_state)?; - let result = reader.read(py); - self.projected_mut()?.reader_failed = result.is_err(); - OcrHostResult::Document(Ok(result?)) + let reader = self + .retained_mut()? + .reader + .take() + .ok_or_else(missing_state)?; + match reader.read(py) { + Ok(content) => OcrHostResult::Document(Ok(content)), + Err(error) if error.is_instance_of::(py) => { + self.state.retain_first_error(py, error, false); + OcrHostResult::Document(Err(litellm_core::ocr::Error::HostDocumentRead)) + } + Err(error) => return Err(error), + } } OcrHostOperation::AcquireAzureAdToken => { OcrHostResult::AzureAdToken(Ok(self.acquire_azure_ad_token(py)?)) } - OcrHostOperation::PreCall(request) => { - self.projected_mut()?.pre_call = Some((&request).into()); - OcrHostResult::PreCall(Ok(request)) - } + OcrHostOperation::PreCall(request) => OcrHostResult::PreCall(Ok(request)), OcrHostOperation::DuringCall(request) => { OcrHostResult::DuringCall(Ok(self.during_call(py, request)?)) } @@ -209,7 +222,7 @@ impl PythonRoute for PythonOcrHost { OcrHostResult::PostCall(Ok(self.post_call(py, request)?)) } OcrHostOperation::ConstructResponse(response) => { - self.state.end = Some(now(py)?); + self.state.finish(py)?; self.state.response = Some(callbacks::response(py, response.as_ref())?); OcrHostResult::Lifecycle(Ok(())) } @@ -224,28 +237,22 @@ impl PythonRoute for PythonOcrHost { } fn cleanup(&mut self) { - self.projected = None; + self.retained = None; } - fn traverse(&self, visit: &pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> { - let Some(projected) = &self.projected else { + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + let Some(retained) = &self.retained else { return Ok(()); }; - if let Some(provider) = &projected.azure_ad_token_provider { + if let Some(provider) = &retained.azure_ad_token_provider { provider.traverse(visit)?; } - if let Some(reader) = &projected.reader { + if let Some(reader) = &retained.reader { reader.traverse(visit)?; } - visit.call(&projected.body)?; - visit.call(&projected.headers) - } -} - -struct BridgeOcrHooks; - -impl litellm_core::ocr::hooks::OcrHooks for BridgeOcrHooks { - fn intercepts_requests(&self) -> bool { - true + if let Some(payload) = &retained.payload { + payload.traverse(visit)?; + } + Ok(()) } } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 7973dc45c9b..d1fd977c5b6 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -62,13 +62,16 @@ fn call( ..OcrAdmission::all() }, ))?; - let host = PythonOcrHost::new(PythonCallState::new( - py, - args.unbind(), - kwargs.copy()?.unbind(), - asynchronous, - signature.name, - )?); + let host = PythonOcrHost::new( + PythonCallState::new( + py, + args.unbind(), + kwargs.copy()?.unbind(), + asynchronous, + signature.name, + )?, + signature, + ); run_call(py, call, host) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index a78aa5b80b7..5157f7683ed 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -10,54 +10,38 @@ use litellm_core::ocr::{ }; use litellm_python_interop::from_py_preserving_errors as from_py; +use super::document::FileDocumentInput; use super::errors::to_pyerr as ocr_error_to_pyerr; -use super::document::{FileDocumentInput, PythonFileReader}; -use crate::auth::PythonTokenProvider; +use super::host::OcrRetained; use crate::lifecycle::BoundArguments; -use crate::marshal::BoundRouteInputs; +use crate::marshal::{BoundRouteInputs, Projection}; /// Positional parameters of `ocr()` that are never projected into /// `optional_params`. const BOUND_FIELDS: &[&str] = &["model", "document", "timeout", "input_sources"]; -pub(super) struct ProjectedOcrCall { - pub request: LiteLLMOcrRequest, - pub azure_ad_token_provider: Option, - pub secret_fields: Vec<&'static str>, - pub reader: Option, -} - -enum ProjectedDocument { - File(FileDocumentInput), - Url(serde_json::Value), -} - -impl ProjectedDocument { - fn into_native(self) -> Result { - match self { - Self::File(file) => Ok(file), - Self::Url(value) => Ok(FileDocumentInput { - input: OcrDocument::try_from(value)?.into(), - reader: None, - }), - } - } -} - -fn project_document(document: &Bound<'_, PyAny>) -> PyResult { +fn project_document( + document: &Bound<'_, PyAny>, +) -> PyResult> { let kind: String = document.get_item("type")?.extract()?; if kind != "file" { - return Ok(ProjectedDocument::Url(from_py(document)?)); + let value: serde_json::Value = from_py(document)?; + return Ok( + OcrDocument::try_from(value).map(|document| FileDocumentInput { + input: document.into(), + reader: None, + }), + ); } - document.extract().map(ProjectedDocument::File) + document.extract().map(Ok) } /// Pure core assembly; every failure here is a typed `ocr::Error`. fn build_request( inputs: BoundRouteInputs, - document: ProjectedDocument, -) -> Result { - let document = document.into_native()?; + document: Result, +) -> Result, litellm_core::ocr::Error> { + let document = document?; let BoundRouteInputs { model, custom_llm_provider, @@ -83,18 +67,23 @@ fn build_request( input_sources, }, )?; - Ok(ProjectedOcrCall { - request, - azure_ad_token_provider, - secret_fields, - reader: document.reader, + Ok(Projection { + retained: OcrRetained { + model: request.model.clone(), + provider: request.provider_name(), + azure_ad_token_provider, + secret_fields, + reader: document.reader, + payload: None, + }, + native: request, }) } pub(super) fn project( py: Python<'_>, arguments: &BoundArguments<'_>, -) -> PyResult { +) -> PyResult> { let model: String = arguments.extract("model")?; let custom_llm_provider: Option = arguments.optional("custom_llm_provider")?; let document = project_document(&arguments.required("document")?)?; @@ -142,7 +131,7 @@ mod tests { ) .unwrap(); assert!(matches!( - project_document(&file).unwrap().into_native().unwrap().input, + project_document(&file).unwrap().unwrap().input, litellm_core::ocr::OcrDocumentInput::Bytes { bytes, mime_type, .. } if bytes == b"%PDF-1.4"[..] && mime_type.as_deref() == Some("application/pdf") )); @@ -155,7 +144,7 @@ mod tests { ) .unwrap(); assert!(matches!( - project_document(&original).unwrap().into_native().unwrap().input, + project_document(&original).unwrap().unwrap().input, litellm_core::ocr::OcrDocumentInput::Document(OcrDocument::DocumentUrl { document_url, .. }) if document_url == "https://example.com/a.pdf" )); @@ -169,7 +158,7 @@ mod tests { let document = py .eval(c"{'type': 'mystery', 'mystery': 'x'}", None, None) .unwrap(); - let error = project_document(&document).unwrap().into_native().err().unwrap(); + let error = project_document(&document).unwrap().err().unwrap(); assert!(error.to_string().contains("document")); }); } @@ -181,14 +170,16 @@ mod tests { let missing = py.eval(c"{}", None, None).unwrap(); assert!( project_document(&missing) - .err().unwrap() + .err() + .unwrap() .is_instance_of::(py) ); let non_string = py.eval(c"{'type': 1}", None, None).unwrap(); assert!( project_document(&non_string) - .err().unwrap() + .err() + .unwrap() .is_instance_of::(py) ); @@ -202,8 +193,9 @@ class Document: document = Document() ", ); - let error = - project_document(&locals.get_item("document").unwrap().unwrap()).err().unwrap(); + let error = project_document(&locals.get_item("document").unwrap().unwrap()) + .err() + .unwrap(); assert!( error .value(py) @@ -232,8 +224,11 @@ document = Document() ", ); let document = locals.get_item("document").unwrap().unwrap(); - let projected = project_document(&document).unwrap().into_native().unwrap(); - assert!(matches!(projected.input, litellm_core::ocr::OcrDocumentInput::Bytes { .. })); + let projected = project_document(&document).unwrap().unwrap(); + assert!(matches!( + projected.input, + litellm_core::ocr::OcrDocumentInput::Bytes { .. } + )); let reads: Vec = document.getattr("reads").unwrap().extract().unwrap(); assert_eq!(reads, ["type", "mime_type", "file"]); }); diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 40621a2f68d..d3410e5d1e5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -573,7 +573,6 @@ class Logging(LiteLLMLoggingBaseClass): self.streaming_chunks: list[Any] = [] # for generating complete stream response self.sync_streaming_chunks: list[Any] = [] # for generating complete stream response self.log_raw_request_response = log_raw_request_response - self._native_callback_fast_path: bool = False # Initialize dynamic callbacks self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 1839ab95526..c6fca6078e5 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -39,6 +39,7 @@ async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: pr ) return await fallback(*args, **kwargs) + def _decline_types() -> tuple[type[BaseException], ...]: exception_types: Final = native_exception_types() return (exception_types[0],) if exception_types is not None else () diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 2f39e6c71bc..729b3bc5c82 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3307,7 +3307,10 @@ class ProxyBaseLLMRequestProcessing: pending: Final = getattr(logging_obj, "_native_pending_logging", None) if pending is not None: logging_obj._native_pending_logging = None # rebind-ok: consume the native OCR release signal once - pending.release(not exception_raised) + if exception_raised: + pending.close() + else: + pending() _enqueue_fn: Final = getattr(logging_obj, "_enqueue_deferred_logging", None) if _enqueue_fn is None: return diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index 283f704bc56..3c858f0a69d 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -29,10 +29,12 @@ def _build_document_from_upload( filename: str | None, content_type: str | None, ) -> dict[str, str]: - mime_type: Final = content_type.split(";")[0].strip() if content_type else None - if not mime_type or mime_type == "application/octet-stream": - if filename: - mime_type = get_mime_type(filename) + supplied_mime: Final = content_type.split(";")[0].strip() if content_type else None + mime_type: Final = ( + get_mime_type(filename) + if filename and (not supplied_mime or supplied_mime == "application/octet-stream") + else supplied_mime + ) return convert_file_document_to_url_document( { # mutable-ok: OCR file document TypedDict handed to the converter diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 3f183db9c10..c65bdb79b1b 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,3 +1,4 @@ +import datetime from asyncio import Future from collections.abc import Coroutine, Mapping from typing import Never, final @@ -115,11 +116,20 @@ class TokenCounter: def acount_request(self, body: bytes) -> Future[dict[str, object]]: ... def gil_stats() -> dict[str, int]: ... +def _debug_setup( + call_type: str, + args: tuple[object, ...], + kwargs: dict[str, object], + start: datetime.datetime, + asynchronous: bool, +) -> tuple[object, dict[str, object]]: ... + __all__ = [ "ResponsesWebSocketConnection", "RustBridgeDeclined", "RustUpstreamError", "TokenCounter", + "_debug_setup", "achat_completions", "amessages", "aocr", diff --git a/litellm/rust_bridge/bindings.py b/litellm/rust_bridge/bindings.py index d16f150a2aa..32f6e9ec46b 100644 --- a/litellm/rust_bridge/bindings.py +++ b/litellm/rust_bridge/bindings.py @@ -47,3 +47,8 @@ def native_exception_types() -> tuple[type[BaseException], type[BaseException]] if not isinstance(declined, type) or not isinstance(upstream, type): return None return declined, upstream + + +def native_decline_types() -> tuple[type[BaseException], ...]: + exceptions: Final = native_exception_types() + return (exceptions[0],) if exceptions is not None else () diff --git a/litellm/rust_bridge/leaves.py b/litellm/rust_bridge/leaves.py new file mode 100644 index 00000000000..eef5a280430 --- /dev/null +++ b/litellm/rust_bridge/leaves.py @@ -0,0 +1,913 @@ +"""Leaf helpers invoked by the native callback dispatcher. + +Rust selects every target, delivery and sequence. Nothing here chooses a callback, +reads a registry, or decides whether an event fires. Each function performs one +integration-specific or interop-specific action on values Rust hands it. + +Labeled leaf helpers pending native migration (see rust-callback-inventory.md): +`prepare_success_logging`, `prepare_failure_logging`, `dispatch_named_success`, +`dispatch_named_failure`, `dispatch_callable`. They are deleted when cost and +payload construction move into core and when each string integration becomes a +CustomLogger. +""" + +from __future__ import annotations + +import datetime +import json +import traceback +from collections.abc import Awaitable, Callable, Mapping +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Final, + Literal, + Protocol, + TypeAlias, + cast, # noqa: TID251 # narrows the untyped legacy Logging and CustomLogger surfaces once +) + +from litellm.integrations.custom_logger import CustomLogger + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import StandardCallbackDynamicParams + +TerminalFamily: TypeAlias = Literal["sync_success", "async_success", "sync_failure", "async_failure"] +Family: TypeAlias = Literal["request", "deployment", TerminalFamily] +Details: TypeAlias = dict[ + str, object +] # mutable-ok: model_call_details is the shared mutable envelope callbacks write to +Timestamp: TypeAlias = datetime.datetime +LegacyCall: TypeAlias = Callable[..., object] + + +class LoggerView(Protocol): + model: str | None + messages: object + call_type: str + start_time: Timestamp + litellm_call_id: str + completion_start_time: Timestamp | None + model_call_details: Details + log_raw_request_response: bool + standard_callback_dynamic_params: StandardCallbackDynamicParams + standard_built_in_tools_params: object + + def record_api_call_start_time(self) -> None: ... + + def record_post_call( + self, original_response: object, input: object, api_key: object, additional_args: Details + ) -> None: ... + + def should_run_callback(self, callback: object, litellm_params: Details, event_hook: str) -> bool: ... + + def _pre_call(self, input: str, api_key: str | None, model: str | None, additional_args: Details) -> None: ... + + def _print_llm_call_debugging_log( + self, api_base: str, headers: Mapping[str, str], additional_args: Details + ) -> None: ... + + def _get_request_curl_command( + self, api_base: str, headers: Mapping[str, str] | None, additional_args: Details, data: object + ) -> str: ... + + def _get_masked_api_base(self, api_base: str) -> str: ... + + def _get_raw_request_body(self, data: object) -> Details: ... + + def _get_masked_headers(self, headers: Mapping[str, str]) -> Details: ... + + def _response_cost_calculator(self, result: object) -> float | None: ... + + def _build_standard_logging_payload( + self, init_response_obj: object, start_time: Timestamp, end_time: Timestamp + ) -> object: ... + + def _handle_callback_failure(self, callback: object) -> None: ... + + +class IntegrationView(Protocol): + def log_pre_api_call(self, model: str | None, messages: object, kwargs: Details) -> None: ... + + def log_post_api_call( + self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp | None + ) -> None: ... + + def log_success_event( + self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp + ) -> None: ... + + def log_failure_event( + self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp + ) -> None: ... + + def async_log_success_event( + self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp + ) -> Awaitable[None]: ... + + def async_log_failure_event( + self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp + ) -> Awaitable[None]: ... + + def logging_hook(self, kwargs: Details, result: object, call_type: str) -> tuple[Details, object]: ... + + def async_logging_hook( + self, kwargs: Details, result: object, call_type: str + ) -> Awaitable[tuple[Details, object]]: ... + + def redact_standard_logging_payload_from_model_call_details(self, model_call_details: Details) -> Details: ... + + def log_input_event( + self, model: str | None, messages: object, kwargs: Details, print_verbose: LegacyCall, callback_func: LegacyCall + ) -> None: ... + + def log_event( + self, + kwargs: Details, + response_obj: object, + start_time: Timestamp, + end_time: Timestamp, + print_verbose: LegacyCall, + callback_func: LegacyCall, + ) -> None: ... + + def async_log_event( + self, + kwargs: Details, + response_obj: object, + start_time: Timestamp, + end_time: Timestamp, + print_verbose: LegacyCall, + callback_func: LegacyCall, + ) -> Awaitable[None]: ... + + +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _details_of(value: object) -> Details: + return cast(Details, value) # cast-ok: legacy model_call_details nesting is untyped and shared by callbacks + + +def _call_of(value: object) -> LegacyCall: + return cast(LegacyCall, value) # cast-ok: legacy integration entry points are untyped + + +def _attribute(target: object, name: str) -> object: + value: Final[object] = getattr(target, name) # pyright: ignore[reportAny] # legacy attributes are untyped by definition + return value + + +def _seed(details: Details, key: str) -> Details: + existing: Final[object] = details.get(key) + if isinstance(existing, dict): + return _details_of(existing) # pyright: ignore[reportUnknownArgumentType] # isinstance yields dict[Unknown, Unknown] + seeded: Final[Details] = {} # mutable-ok: legacy envelope seed shared with callbacks + details[key] = seeded # rebind-ok: seeding a nested dict on the shared envelope is the legacy contract + return seeded + + +def _logger(logger: Logging) -> LoggerView: + return cast(LoggerView, logger) # cast-ok: legacy Logging is untyped; this protocol names the attributes we read + + +def _integration(callback: CustomLogger) -> IntegrationView: + return cast(IntegrationView, callback) # cast-ok: legacy CustomLogger methods are untyped + + +def _singleton(name: str) -> object: + from litellm.litellm_core_utils import litellm_logging + + return getattr(litellm_logging, name, None) # pyright: ignore[reportAny] # legacy module globals are rebound at runtime + + +def _print_verbose() -> LegacyCall: + from litellm.litellm_core_utils import litellm_logging + + return cast(LegacyCall, litellm_logging.print_verbose) # cast-ok: legacy debug printer is untyped + + +def _redact_string(value: str) -> str: + from litellm.litellm_core_utils import litellm_logging + + redact: Final = _call_of(litellm_logging._redact_string) # pyright: ignore[reportPrivateUsage] # private legacy helper + return str(redact(value)) + + +def _redact_result(details: Details, result: object) -> object: + from litellm.litellm_core_utils import redact_messages + + redact: Final = cast(LegacyCall, redact_messages.redact_message_input_output_from_logging) # cast-ok: legacy helper + return redact(model_call_details=details, result=result) + + +def record_pre_call( + logger: Logging, + *, + api_key: str | None, + body: Details, + headers: Mapping[str, str], + url: str, +) -> None: + view: Final = _logger(logger) + additional_args: Final[Details] = { # mutable-ok: legacy additional_args is rebound by callbacks in place + "complete_input_dict": body, + "headers": headers, + "api_base": url, + } + view._pre_call(input="OCR document processing", api_key=api_key, model=None, additional_args=additional_args) # pyright: ignore[reportPrivateUsage] # legacy state writer + view._print_llm_call_debugging_log(api_base=url, headers=headers, additional_args=additional_args) # pyright: ignore[reportPrivateUsage] # legacy debug output + _capture_raw_request(view, additional_args) + _run_logger_fn(logger) + view.record_api_call_start_time() + + +def _capture_raw_request(view: LoggerView, additional_args: Details) -> None: + import litellm + from litellm.types.utils import RawRequestTypedDict + + if not (view.log_raw_request_response or litellm.log_raw_request_response): + return + details: Final = view.model_call_details + metadata: Final = _seed(_seed(details, "litellm_params"), "metadata") + if litellm.turn_off_message_logging: + metadata["raw_request"] = "redacted by litellm. 'litellm.turn_off_message_logging=True'" + return + api_base: Final = str(additional_args.get("api_base") or "") + headers: Final = cast(Mapping[str, str], additional_args.get("headers") or _EMPTY) # cast-ok: legacy nested dict + body: Final = additional_args.get("complete_input_dict", _EMPTY) + try: + curl: Final = view._get_request_curl_command( # pyright: ignore[reportPrivateUsage] # legacy debug formatter + api_base=api_base, headers=headers, additional_args=additional_args, data=body + ) + metadata["raw_request"] = _redact_string(str(curl)) + details["raw_request_typed_dict"] = RawRequestTypedDict( + raw_request_api_base=view._get_masked_api_base(api_base), # pyright: ignore[reportPrivateUsage] # legacy masking + raw_request_body=view._get_raw_request_body(body), # pyright: ignore[reportPrivateUsage] # legacy masking + raw_request_headers=view._get_masked_headers(headers), # pyright: ignore[reportPrivateUsage] # legacy masking + error=None, + ) + except Exception as error: # noqa: BLE001 # raw-request capture is best effort by contract + details["raw_request_typed_dict"] = RawRequestTypedDict(error=str(error)) + metadata["raw_request"] = _redact_string(f"Unable to Log raw request: {error}") + + +def _run_logger_fn(logger: Logging) -> None: + from litellm._logging import verbose_logger + + logger_fn: Final = getattr(logger, "logger_fn", None) + if not callable(logger_fn): + return + try: + _call_of(logger_fn)(_logger(logger).model_call_details) + except Exception as error: # noqa: BLE001 # user logger_fn failures never fail the request + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", error) + + +def record_post_call(logger: Logging, *, original_response: object, body: Details, headers: Mapping[str, str]) -> None: + view: Final = _logger(logger) + serialized: Final = ( + json.dumps(original_response, default=str) if isinstance(original_response, dict) else original_response + ) + view.record_post_call( + original_response=serialized, + input=None, + api_key=None, + additional_args={"complete_input_dict": body, "headers": headers}, # mutable-ok: rebound in place by callbacks + ) + _run_logger_fn(logger) + _redact_result(view.model_call_details, serialized) + + +def log_pre_api_call(logger: Logging, callback: CustomLogger) -> None: + view: Final = _logger(logger) + _integration(callback).log_pre_api_call(model=view.model, messages=view.messages, kwargs=view.model_call_details) + + +def log_post_api_call(logger: Logging, callback: CustomLogger) -> None: + view: Final = _logger(logger) + _integration(callback).log_post_api_call( + kwargs=view.model_call_details, response_obj=None, start_time=view.start_time, end_time=None + ) + + +def dispatch_named_request(logger: Logging, name: str, event: Literal["pre_api_call", "post_api_call"]) -> None: + from litellm.integrations.supabase import Supabase + + view: Final = _logger(logger) + details: Final = view.model_call_details + match name: + case "supabase" if event == "pre_api_call" and isinstance(client := _singleton("supabaseClient"), Supabase): + client.input_log_event( + model=view.model, + messages=view.messages, + end_user=details.get("user", "default"), + litellm_call_id=details["litellm_call_id"], + print_verbose=_print_verbose(), + ) + case "sentry" if callable(add_breadcrumb := _singleton("add_breadcrumb")): + add_breadcrumb(category="litellm.llm_call", message=f"Model Call Details {event}: {details}", level="info") + case _: + return + + +def dispatch_callable_request(logger: Logging, callback: LegacyCall) -> None: + custom: Final = _singleton("customLogger") + if not isinstance(custom, CustomLogger): + return + view: Final = _logger(logger) + _integration(custom).log_input_event( + model=view.model, + messages=view.messages, + kwargs=view.model_call_details, + print_verbose=_print_verbose(), + callback_func=callback, + ) + + +def _typed_call_type(call_type: str) -> object: + from litellm.types.utils import CallTypes + + try: + return CallTypes(call_type) + except ValueError: + return None + + +def _awaitable(value: object) -> Awaitable[object]: + return cast(Awaitable[object], value) # cast-ok: legacy async hooks return coroutines + + +def pre_call_deployment_hook(callback: CustomLogger, kwargs: Details, call_type: str) -> Awaitable[object]: + hook: Final = _call_of(_attribute(callback, "async_pre_call_deployment_hook")) + return _awaitable(hook(kwargs, _typed_call_type(call_type))) + + +def post_call_success_deployment_hook( + callback: CustomLogger, kwargs: Details, response: object, call_type: str +) -> Awaitable[object]: + hook: Final = _call_of(_attribute(callback, "async_post_call_success_deployment_hook")) + return _awaitable(hook(kwargs, response, _typed_call_type(call_type))) + + +def failure_deployment_hook_view( + kwargs: Details, exception: BaseException +) -> tuple[Mapping[str, object], BaseException, int | None]: + from litellm import utils + + raw_depth: Final = kwargs.get("fallback_depth") + depth: Final = raw_depth if isinstance(raw_depth, int) else None + safe_request: Final = MappingProxyType({key: value for key, value in kwargs.items() if key != "attempted_targets"}) + snapshot: Final = _call_of(utils._snapshot_exception_for_hook)(exception) # pyright: ignore[reportPrivateUsage] # legacy snapshot helper + return safe_request, cast(BaseException, snapshot), depth # cast-ok: snapshot is a same-class copy of the exception + + +def post_call_failure_deployment_hook( + callback: CustomLogger, + request: Mapping[str, object], + exception: BaseException, + call_type: str, + fallback_depth: int | None, +) -> Awaitable[object]: + from litellm import utils + + hook: Final = _call_of(_attribute(callback, "async_post_call_failure_deployment_hook")) + accepts_depth: Final = _call_of(utils._accepts_fallback_depth_kwarg_for_class)(type(callback)) # pyright: ignore[reportPrivateUsage] # legacy signature probe + typed: Final = _typed_call_type(call_type) + if accepts_depth: + return _awaitable(hook(request, exception, typed, fallback_depth=fallback_depth)) + return _awaitable(hook(request, exception, typed)) + + +def report_deployment_failure_hook_error(callback: object, error: BaseException) -> None: + from litellm._logging import verbose_logger + + verbose_logger.debug("async_post_call_failure_deployment_hook error in %s: %s", type(callback).__name__, error) + + +def report_target_failure(logger: Logging, callback: object, family: Family, error: BaseException) -> None: + from litellm._logging import verbose_logger + + verbose_logger.error( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while %s logging with %s: %s", + family, + callback, + "".join(traceback.format_exception(error)), + ) + capture: Final = _singleton("capture_exception") + if callable(capture) and family in ("request", "sync_success", "sync_failure"): + capture(error) + if family not in ("request", "deployment", "sync_failure"): + _logger(logger)._handle_callback_failure(callback=callback) # pyright: ignore[reportPrivateUsage] # legacy prometheus counter + + +def prepare_success_logging(logger: Logging, response: object, start_time: Timestamp, end_time: Timestamp) -> object: + from litellm.litellm_core_utils.litellm_logging import emit_standard_logging_payload + from litellm.types.utils import StandardLoggingPayload + + view: Final = _logger(logger) + details: Final = view.model_call_details + if view.completion_start_time is None: + view.completion_start_time = end_time + details["completion_start_time"] = end_time + details["log_event_type"] = "successful_api_call" + details["end_time"] = end_time + details["cache_hit"] = None + hidden: Final = _details_of(getattr(response, "_hidden_params", None) or {}) # mutable-ok: legacy fallback envelope + if hidden and isinstance(details.get("litellm_params"), dict): + _seed(_seed(details, "litellm_params"), "metadata")["hidden_params"] = hidden + existing: Final = details.get("response_cost") + if "response_cost" in hidden: + details["response_cost"] = hidden["response_cost"] + elif existing is None or existing == 0: + details["response_cost"] = view._response_cost_calculator(result=response) # pyright: ignore[reportPrivateUsage] # labeled leaf: native cost pending + payload: Final = view._build_standard_logging_payload(response, start_time, end_time) # pyright: ignore[reportPrivateUsage] # labeled leaf: native payload pending + details["standard_logging_object"] = payload + if payload is not None: + emit_standard_logging_payload( + cast(StandardLoggingPayload, payload) # cast-ok: legacy builder returns the payload TypedDict + ) # cast-ok: legacy builder returns the payload TypedDict + return _redact_result(details, response) + + +def prepare_failure_logging( + logger: Logging, exception: BaseException, start_time: Timestamp, end_time: Timestamp +) -> str: + from litellm.litellm_core_utils import litellm_logging + + formatted: Final = "".join(traceback.format_exception(exception)) + view: Final = _logger(logger) + details: Final = view.model_call_details + if details.get("exception") is exception and details.get("standard_logging_object") is not None: + return formatted + details["log_event_type"] = "failed_api_call" + details["exception"] = exception + details["traceback_exception"] = _redact_string(formatted) + details["end_time"] = end_time + details.setdefault("original_response", None) + if details.get("combined_usage_object") is None: + details["response_cost"] = 0 + headers: Final = getattr(exception, "headers", None) + if isinstance(headers, Mapping): + _seed(_seed(details, "litellm_params"), "metadata").update( + _details_of(headers) # pyright: ignore[reportUnknownArgumentType] # exception headers carry no element types + ) + build: Final = cast( # cast-ok: labeled leaf: native payload pending + LegacyCall, litellm_logging.get_standard_logging_object_payload + ) + details["standard_logging_object"] = build( + kwargs=details, + init_response_obj=_EMPTY, + start_time=start_time, + end_time=end_time, + logging_obj=logger, + status="failure", + error_str=_redact_string(str(exception)), + original_exception=exception, + standard_built_in_tools_params=view.standard_built_in_tools_params, + ) + return formatted + + +_EVENT_HOOKS: Final[Mapping[TerminalFamily, str]] = MappingProxyType( + { + "sync_success": "success_handler", + "async_success": "async_success_handler", + "sync_failure": "failure_handler", + "async_failure": "async_failure_handler", + } +) + + +def should_run_callback(logger: Logging, callback: object, family: TerminalFamily) -> bool: + view: Final = _logger(logger) + params: Final = _details_of(view.model_call_details.get("litellm_params") or _EMPTY) + return view.should_run_callback(callback=callback, litellm_params=params, event_hook=_EVENT_HOOKS[family]) + + +def should_run_guardrail_hook(logger: Logging, callback: object) -> bool: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + if not isinstance(callback, CustomGuardrail): + return True + decide: Final = cast(LegacyCall, callback.should_run_guardrail) # cast-ok: legacy guardrail method is untyped + return decide(data=_logger(logger).model_call_details, event_type=GuardrailEventHooks.logging_only) is True + + +def logging_hook(logger: Logging, callback: CustomLogger, result: object) -> object: + view: Final = _logger(logger) + details, replaced = _integration(callback).logging_hook( + kwargs=view.model_call_details, result=result, call_type=view.call_type + ) + view.model_call_details = details + return replaced + + +async def async_logging_hook(logger: Logging, callback: CustomLogger, result: object) -> object: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.litellm_core_utils import redact_messages + + view: Final = _logger(logger) + redact: Final = cast( # cast-ok: legacy helper + LegacyCall, redact_messages.redact_message_input_output_from_custom_logger + ) + redacted: Final = ( + result + if isinstance(callback, CustomGuardrail) + else redact(result=result, litellm_logging_obj=logger, custom_logger=callback) + ) + details, replaced = await _integration(callback).async_logging_hook( + kwargs=view.model_call_details, result=redacted, call_type=view.call_type + ) + view.model_call_details = details + return replaced + + +def is_sync_request(logger: Logging) -> bool: + from litellm.litellm_core_utils.litellm_logging import Logging as LoggingClass + + params: Final = _details_of(_logger(logger).model_call_details.get("litellm_params") or _EMPTY) + decide: Final = _call_of(LoggingClass._is_sync_litellm_request) # pyright: ignore[reportPrivateUsage] # legacy predicate + return decide(params) is True + + +def mark_logged(logger: Logging, marker: str) -> None: + _logger(logger).model_call_details[marker] = True + + +def already_logged(logger: Logging, marker: str) -> bool: + return _logger(logger).model_call_details.get(marker, False) is True + + +def log_success_event( + logger: Logging, callback: CustomLogger, response: object, start_time: Timestamp, end_time: Timestamp +) -> None: + _integration(callback).log_success_event( + kwargs=_logger(logger).model_call_details, response_obj=response, start_time=start_time, end_time=end_time + ) + + +def async_log_success_event( + logger: Logging, callback: CustomLogger, response: object, start_time: Timestamp, end_time: Timestamp +) -> Awaitable[None]: + from litellm.litellm_core_utils import redact_messages + + integration: Final = _integration(callback) + details: Final = integration.redact_standard_logging_payload_from_model_call_details( + model_call_details=_logger(logger).model_call_details + ) + redact: Final = cast( # cast-ok: legacy helper + Callable[..., Details], redact_messages.redact_streaming_responses_for_custom_logger + ) + view: Final = redact(model_call_details=details, custom_logger=callback) + return integration.async_log_success_event( + kwargs=view, response_obj=response, start_time=start_time, end_time=end_time + ) + + +def log_failure_event(logger: Logging, callback: CustomLogger, start_time: Timestamp, end_time: Timestamp) -> None: + _integration(callback).log_failure_event( + kwargs=_logger(logger).model_call_details, response_obj=None, start_time=start_time, end_time=end_time + ) + + +def async_log_failure_event( + logger: Logging, callback: CustomLogger, start_time: Timestamp, end_time: Timestamp +) -> Awaitable[None]: + return _integration(callback).async_log_failure_event( + kwargs=_logger(logger).model_call_details, response_obj=None, start_time=start_time, end_time=end_time + ) + + +def _custom_logger_singleton() -> IntegrationView: + from litellm.litellm_core_utils import litellm_logging + + existing: Final = _singleton("customLogger") + if isinstance(existing, CustomLogger): + return _integration(existing) + created: Final = CustomLogger() + litellm_logging.customLogger = created # pyright: ignore[reportAttributeAccessIssue] # legacy module global + return _integration(created) + + +def dispatch_callable( + logger: Logging, + callback: LegacyCall, + family: TerminalFamily, + response: object, + start_time: Timestamp, + end_time: Timestamp, +) -> Awaitable[None] | None: + custom: Final = _custom_logger_singleton() + details: Final = _logger(logger).model_call_details + match family: + case "sync_success" | "sync_failure": + custom.log_event( + kwargs=details, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=_print_verbose(), + callback_func=callback, + ) + return None + case "async_success" | "async_failure": + return custom.async_log_event( + kwargs=details, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=_print_verbose(), + callback_func=callback, + ) + + +def dispatch_named_success( + logger: Logging, name: str, response: object, start_time: Timestamp, end_time: Timestamp +) -> Awaitable[None] | None: + from litellm.integrations.athina import AthinaLogger + from litellm.integrations.greenscale import GreenscaleLogger + from litellm.integrations.helicone import HeliconeLogger + from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger + from litellm.integrations.lunary import LunaryLogger + from litellm.integrations.openmeter import OpenMeterLogger + from litellm.integrations.prompt_layer import PromptLayerLogger + from litellm.integrations.s3 import S3Logger + from litellm.integrations.supabase import Supabase + from litellm.integrations.traceloop import TraceloopLogger + from litellm.integrations.weights_biases import WeightsBiasesLogger + + view: Final = _logger(logger) + details: Final = view.model_call_details + print_verbose: Final = _print_verbose() + without_response: Final = { # mutable-ok: legacy integrations receive a private mutable copy + key: value for key, value in details.items() if key != "original_response" + } + match name: + case "promptlayer" if isinstance(promptlayer := _singleton("promptLayerLogger"), PromptLayerLogger): + promptlayer.log_event( + kwargs=details, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + case "wandb" if isinstance(wandb := _singleton("weightsBiasesLogger"), WeightsBiasesLogger): + wandb.log_event( + kwargs=details, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + case "athina" if isinstance(athina := _singleton("athinaLogger"), AthinaLogger): + athina.log_event( + kwargs=details, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + case "logfire" if isinstance(logfire := _singleton("logfireLogger"), LogfireLogger): + logfire.log_event( + kwargs=without_response, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + level=LogfireLevel.INFO, + ) + case "greenscale" if isinstance(greenscale := _singleton("greenscaleLogger"), GreenscaleLogger): + greenscale.log_event( + kwargs=without_response, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + case "supabase" if isinstance(supabase := _singleton("supabaseClient"), Supabase): + supabase.log_event( + model=view.model, + messages=view.messages, + end_user=details.get("user", "default"), + response_obj=response, + start_time=start_time, + end_time=end_time, + litellm_call_id=details["litellm_call_id"], + print_verbose=print_verbose, + ) + case "lunary" if isinstance(lunary := _singleton("lunaryLogger"), LunaryLogger): + lunary.log_event( + kwargs=details, + type="llm", + event="end", + model=view.model, + input=details["input"], + user_id=details.get("user", "default"), + response_obj=response, + start_time=start_time, + end_time=end_time, + run_id=view.litellm_call_id, + print_verbose=print_verbose, + ) + case "helicone" if isinstance(helicone := _singleton("heliconeLogger"), HeliconeLogger): + helicone.log_success( + model=view.model, + messages=view.messages, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + kwargs=details, + ) + case "langfuse": + _langfuse( + logger, response=response, start_time=start_time, end_time=end_time, level=None, status_message=None + ) + case "traceloop" if isinstance(traceloop := _singleton("traceloopLogger"), TraceloopLogger): + traceloop.log_event( + kwargs=details, + response_obj=response, + start_time=start_time, + end_time=end_time, + user_id=details.get("user", None), + print_verbose=print_verbose, + ) + case "s3" if isinstance(s3 := _singleton("s3Logger"), S3Logger): + s3.log_event( + kwargs=details, + response_obj=response, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + case "openmeter" if isinstance(openmeter := _singleton("openMeterLogger"), OpenMeterLogger): + return openmeter.async_log_success_event( + kwargs=details, response_obj=response, start_time=start_time, end_time=end_time + ) + case "dynamodb": + return _dynamodb(details, response, start_time, end_time, print_verbose) + case _: + return None + return None + + +def _dynamodb( + details: Details, response: object, start_time: Timestamp, end_time: Timestamp, print_verbose: LegacyCall +) -> Awaitable[None]: + from litellm.integrations.dynamodb import DyanmoDBLogger + from litellm.litellm_core_utils import litellm_logging + + existing: Final = _singleton("dynamoLogger") + dynamo: Final = existing if isinstance(existing, DyanmoDBLogger) else DyanmoDBLogger() + litellm_logging.dynamoLogger = dynamo # pyright: ignore[reportAttributeAccessIssue] # legacy module global + return dynamo._async_log_event( # pyright: ignore[reportPrivateUsage] # legacy async entry point + kwargs=details, response_obj=response, start_time=start_time, end_time=end_time, print_verbose=print_verbose + ) + + +def _langfuse( + logger: Logging, + *, + response: object, + start_time: Timestamp, + end_time: Timestamp, + level: str | None, + status_message: str | None, +) -> None: + from litellm.integrations.langfuse.langfuse import LangFuseLogger + from litellm.integrations.langfuse.langfuse_handler import LangFuseHandler + from litellm.litellm_core_utils import litellm_logging + from litellm.types.utils import ModelResponse + + view: Final = _logger(logger) + kwargs: Final = { # mutable-ok: langfuse receives a private mutable copy + key: value for key, value in view.model_call_details.items() if key != "original_response" + } + global_logger: Final = _singleton("langFuseLogger") + handler: Final = LangFuseHandler.get_langfuse_logger_for_request( + globalLangfuseLogger=global_logger if isinstance(global_logger, LangFuseLogger) else None, + standard_callback_dynamic_params=view.standard_callback_dynamic_params, + in_memory_dynamic_logger_cache=litellm_logging.in_memory_dynamic_logger_cache, + ) + user: Final = kwargs.get("user") + result: Final = handler.log_event_on_langfuse( + kwargs=kwargs, + response_obj=cast(ModelResponse, response), # cast-ok: OCR responses sit outside the legacy union + start_time=start_time, + end_time=end_time, + user_id=user if isinstance(user, str) else None, + level="DEFAULT" if level is None else level, + status_message=status_message, + ) + trace_id: Final = result.get("trace_id") + if isinstance(trace_id, str): + litellm_logging.in_memory_trace_id_cache.set_cache( + litellm_call_id=view.litellm_call_id, service_name="langfuse", trace_id=trace_id + ) + + +def dispatch_named_failure( + logger: Logging, + name: str, + exception: BaseException, + formatted: str, + start_time: Timestamp, + end_time: Timestamp, +) -> None: + from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger + from litellm.integrations.lunary import LunaryLogger + from litellm.integrations.supabase import Supabase + from litellm.integrations.traceloop import TraceloopLogger + + view: Final = _logger(logger) + details: Final = view.model_call_details + print_verbose: Final = _print_verbose() + match name: + case "lunary" if isinstance(lunary := _singleton("lunaryLogger"), LunaryLogger): + lunary.log_event( + kwargs=details, + type="llm", + event="error", + user_id=details.get("user", "default"), + model=view.model, + input=details["input"], + error=formatted, + run_id=view.litellm_call_id, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + case "sentry" if callable(capture := _singleton("capture_exception")): + capture(exception) + case "supabase" if isinstance(supabase := _singleton("supabaseClient"), Supabase): + supabase.log_event( + model=view.model, + messages=view.messages, + end_user=details.get("user", "default"), + response_obj=None, + start_time=start_time, + end_time=end_time, + litellm_call_id=details["litellm_call_id"], + print_verbose=print_verbose, + ) + case "langfuse": + _langfuse( + logger, + response=None, + start_time=start_time, + end_time=end_time, + level="ERROR", + status_message=str(exception), + ) + case "traceloop" if isinstance(traceloop := _singleton("traceloopLogger"), TraceloopLogger): + traceloop.log_event( + start_time=start_time, + end_time=end_time, + response_obj=None, + user_id=details.get("user", None), + print_verbose=print_verbose, + status_message=str(exception), + level="ERROR", + kwargs=details, + ) + case "logfire" if isinstance(logfire := _singleton("logfireLogger"), LogfireLogger): + logfire.log_event( + kwargs={ # mutable-ok: logfire receives a private mutable copy + key: value for key, value in details.items() if key != "original_response" + } + | {"exception": exception}, # mutable-ok: merged into the private copy above + response_obj=None, + start_time=start_time, + end_time=end_time, + level=LogfireLevel.ERROR, + print_verbose=print_verbose, + ) + case _: + return + + +def restore_correlation_context(logger: object) -> None: + from litellm import utils + + utils._restore_correlation_context_if_supported(logger) # pyright: ignore[reportPrivateUsage] # legacy interop helper + + +def submit_worker(job: Callable[[], None]) -> None: + import contextvars + + from litellm.litellm_core_utils.thread_pool_executor import executor + + context: Final = contextvars.copy_context() + _ = executor.submit(context.run, job) + + +def enqueue_background(coroutine: Awaitable[None]) -> None: + import contextvars + + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + enqueue: Final = _call_of(_attribute(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue")) + contextvars.copy_context().run(enqueue, coroutine) diff --git a/litellm/rust_bridge/lifecycle.py b/litellm/rust_bridge/lifecycle.py index f1cc912129d..c3ab962a5cb 100644 --- a/litellm/rust_bridge/lifecycle.py +++ b/litellm/rust_bridge/lifecycle.py @@ -1,19 +1,8 @@ from __future__ import annotations -import datetime -import os -import uuid from collections.abc import Awaitable, Mapping from dataclasses import dataclass -from typing import ( - TYPE_CHECKING, - Final, - Protocol, - cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations -) - -if TYPE_CHECKING: - from litellm.litellm_core_utils.litellm_logging import Logging +from typing import Final, Protocol @dataclass(frozen=True, slots=True) @@ -53,50 +42,6 @@ async def drive(execution: Execution) -> object: execution.close() -class MetadataUpdater(Protocol): - def __call__( - self, - result: object, - logging_obj: Logging, - model: str | None, - kwargs: dict[str, object], - start_time: datetime.datetime, - end_time: datetime.datetime, - ) -> None: ... - - -@dataclass(frozen=True, slots=True) -class CallSetup: - logger: Logging - kwargs: dict[str, object] - - -def setup( - call_type: str, - args: tuple[object, ...], - kwargs: Mapping[str, object], - start_time: datetime.datetime, - asynchronous: bool, -) -> CallSetup: - from litellm import utils - from litellm.litellm_core_utils.litellm_logging import Logging - - arguments: Final = { # mutable-ok: function_setup consumes an owned kwargs dict - "litellm_call_id": str(uuid.uuid4()), - **kwargs, - } - supplied: Final = arguments.get("litellm_logging_obj") - if isinstance(supplied, Logging): - supplied._native_callback_fast_path = False # pyright: ignore[reportPrivateUsage] # supplied loggers retain all dispatch contracts - return CallSetup(supplied, arguments) - logger, prepared = utils.function_setup( - call_type, utils.Rules(), start_time, *args, is_async_call=asynchronous, **arguments - ) - if type(logger) is Logging and call_type in ("ocr", "aocr"): - logger._native_callback_fast_path = True # pyright: ignore[reportPrivateUsage] # only bridge-created OCR loggers opt into callback elision - return CallSetup(logger, prepared) - - def check_limits(kwargs: Mapping[str, object]) -> None: import litellm from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit @@ -106,100 +51,3 @@ def check_limits(kwargs: Mapping[str, object]) -> None: raise litellm.BudgetExceededError(current_cost=current_cost, max_budget=litellm.max_budget) if max_retries_per_request_hit(kwargs, litellm.num_retries_per_request): raise RuntimeError("Max retries per request hit!") - - -def finalize( - response: object, - logger: Logging, - kwargs: dict[str, object], - start_time: datetime.datetime, - end_time: datetime.datetime, -) -> None: - from litellm.litellm_core_utils.llm_response_utils import response_metadata - - model: Final = kwargs.get("model") - update: Final = cast( # cast-ok: legacy metadata function accepts concrete kwargs - MetadataUpdater, response_metadata.update_response_metadata - ) - update(response, logger, model if isinstance(model, str) else None, kwargs, start_time, end_time) - - -def deployment_callbacks_needed() -> bool: - import litellm - from litellm.integrations.custom_logger import CustomLogger - - return any(isinstance(callback, CustomLogger) for callback in litellm.callbacks) - - -def callbacks_needed(logger: Logging, phase: str) -> bool: - import litellm - from litellm._logging import ( - _is_debugging_on, # pyright: ignore[reportPrivateUsage] # use the same debug gate as Logging - ) - - if ( - _is_debugging_on() - or getattr(logger, "litellm_request_debug", False) - or os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD") - ): - return True - input_needed: Final = bool( - litellm.input_callback - or litellm._async_input_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor - or logger.dynamic_input_callbacks - or callable(getattr(logger, "logger_fn", None)) - or logger.log_raw_request_response - or litellm.log_raw_request_response - ) - match phase: - case "input": - return input_needed - case "sync_success": - return bool(litellm.success_callback or logger.dynamic_success_callbacks) - case "sync_success_async": - return bool( - (litellm.success_callback or logger.dynamic_success_callbacks) - and logger._should_run_sync_callbacks_for_async_calls() # pyright: ignore[reportPrivateUsage] # preserve async call filtering of sync callbacks - ) - case "async_success": - return bool(litellm._async_success_callback or logger.dynamic_async_success_callbacks) # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor - case "sync_failure": - return bool(litellm.failure_callback or logger.dynamic_failure_callbacks) - case "async_failure": - return bool(litellm._async_failure_callback or logger.dynamic_async_failure_callbacks) # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor - case "payload": - return bool( - input_needed - or litellm.success_callback - or litellm.failure_callback - or litellm._async_success_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor - or litellm._async_failure_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor - or logger.dynamic_success_callbacks - or logger.dynamic_async_success_callbacks - or logger.dynamic_failure_callbacks - or logger.dynamic_async_failure_callbacks - ) - case _: - return True - - -def success_bookkeeping( - logger: Logging, response: object, start: datetime.datetime, end: datetime.datetime, asynchronous: bool -) -> None: - phase: Final = "async_success" if asynchronous else "sync_success" - if logger.should_run_logging(phase): - logger._success_handler_helper_fn( # pyright: ignore[reportPrivateUsage] # retain success bookkeeping without constructing a callback payload - result=response, start_time=start, end_time=end, build_logging_payload=False - ) - logger.has_run_logging(phase) - - -def failure_bookkeeping( - logger: Logging, error: BaseException, start: datetime.datetime, end: datetime.datetime, asynchronous: bool -) -> None: - phase: Final = "async_failure" if asynchronous else "sync_failure" - if logger.should_run_logging(phase): - logger._failure_handler_helper_fn( # pyright: ignore[reportPrivateUsage] # retain failure accounting without formatting an unused traceback or payload - error, "", start, end, build_logging_payload=False - ) - logger.has_run_logging(phase) diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index 7ede521e6f3..069ec7a4ae6 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Coroutine, Mapping +from collections.abc import Callable, Coroutine, Mapping, Sequence from types import MappingProxyType from typing import Final, Protocol, cast # noqa: TID251 # validates dynamically loaded native callables @@ -8,6 +8,56 @@ from pydantic import TypeAdapter from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse from litellm.rust_bridge.bindings import NativeBinding +from litellm.types.utils import CustomPricingLiteLLMParams + + +class OcrLoggingProtocol(Protocol): + def update_from_kwargs( + self, + *, + kwargs: dict[str, object], + model: str, + optional_params: dict[str, object], + litellm_params: dict[str, object], + custom_llm_provider: str, + ) -> object: ... + + +def _redact(params: Mapping[str, object], secret_fields: Sequence[str]) -> dict[str, object]: + return { # mutable-ok: update_from_kwargs takes dict + name: "****" if name in secret_fields else value + for name, value in params.items() + if name != "proxy_server_request" + } + + +def update_logging( + logger: OcrLoggingProtocol, + kwargs: Mapping[str, object], + model: str, + custom_llm_provider: str, + optional_params: Mapping[str, object], + secret_fields: Sequence[str], + url: str, +) -> None: + logger.update_from_kwargs( + kwargs=_redact(kwargs, secret_fields), + model=model, + optional_params=_redact(optional_params, secret_fields), + litellm_params={ # mutable-ok: update_from_kwargs takes dict + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": url, + **{ + name: kwargs[name] for name in ("logger_fn", "litellm_request_debug") if name in kwargs + }, # mutable-ok: splat into the dict above + **{ # mutable-ok: splat into the dict above + name: kwargs[name] + for name in CustomPricingLiteLLMParams.model_fields + if name in kwargs and kwargs[name] is not None + }, + }, + custom_llm_provider=custom_llm_provider, + ) class RustOcr(Protocol): diff --git a/litellm/rust_bridge/setup.py b/litellm/rust_bridge/setup.py new file mode 100644 index 00000000000..428f14ba480 --- /dev/null +++ b/litellm/rust_bridge/setup.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +import datetime +from collections.abc import Callable, Mapping, MutableSequence, Sequence +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Final, + Literal, + Protocol, + TypeAlias, + cast, # noqa: TID251 # narrows legacy untyped registries at the boundary +) + +from litellm.integrations.custom_logger import CustomLogger + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging + +CallbackTarget: TypeAlias = str | Callable[..., object] | CustomLogger +RegistryName: TypeAlias = Literal[ + "input", "async_input", "success", "async_success", "failure", "async_failure", "callbacks" +] +NamedEvent: TypeAlias = Literal["success", "failure"] +Kwargs: TypeAlias = dict[str, object] # mutable-ok: the retained call kwargs are the dict callbacks receive and mutate + +_REGISTRY_ATTRIBUTES: Final[Mapping[RegistryName, str]] = MappingProxyType( + { + "input": "input_callback", + "async_input": "_async_input_callback", + "success": "success_callback", + "async_success": "_async_success_callback", + "failure": "failure_callback", + "async_failure": "_async_failure_callback", + "callbacks": "callbacks", + } +) + + +class _CallbackManager(Protocol): + def add_litellm_success_callback(self, callback: CallbackTarget) -> None: ... + + def add_litellm_failure_callback(self, callback: CallbackTarget) -> None: ... + + def add_litellm_async_success_callback(self, callback: CallbackTarget) -> None: ... + + def add_litellm_async_failure_callback(self, callback: CallbackTarget) -> None: ... + + +class _LoggingFactory(Protocol): + def __call__( + self, + *, + model: str | None, + messages: object, + stream: bool, + litellm_call_id: str, + litellm_trace_id: str | None, + function_id: str, + call_type: str, + start_time: datetime.datetime, + dynamic_success_callbacks: Sequence[CallbackTarget] | None, + dynamic_failure_callbacks: Sequence[CallbackTarget] | None, + dynamic_async_success_callbacks: Sequence[CallbackTarget] | None, + dynamic_async_failure_callbacks: Sequence[CallbackTarget] | None, + kwargs: Kwargs, + applied_guardrails: Sequence[str], + supports_correlation_logging: bool, + ) -> Logging: ... + + +class _EnvironmentUpdater(Protocol): + def __call__( + self, + *, + model: str | None, + user: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream_options: object, + ) -> None: ... + + +def _legacy(value: object) -> Callable[..., object]: + return cast(Callable[..., object], value) # cast-ok: legacy module-level helpers are untyped + + +def _mapping(value: object) -> Mapping[str, object] | None: + if not isinstance(value, Mapping): + return None + return cast(Mapping[str, object], value) # cast-ok: isinstance narrows only to Mapping[Unknown, Unknown] + + +def registry( + name: RegistryName, +) -> MutableSequence[CallbackTarget]: # mutable-ok: the public litellm registries are mutated by contract + import litellm + + return cast( # cast-ok: legacy module-level lists are untyped + MutableSequence[CallbackTarget], getattr(litellm, _REGISTRY_ATTRIBUTES[name]) + ) + + +def is_async_callable(callback: object) -> bool: + from litellm.litellm_core_utils.cached_imports import get_coroutine_checker + + return get_coroutine_checker().is_async_callable(callback) + + +def is_known_name(callback: str) -> bool: + import litellm + + known: Final = cast(Sequence[str], litellm._known_custom_logger_compatible_callbacks) # pyright: ignore[reportPrivateUsage] # cast-ok: registry list has no public typed accessor + return callback in known + + +def resolve_named_integration(callback: str) -> CustomLogger | None: + from litellm.litellm_core_utils import litellm_logging + + factory: Final = _legacy(litellm_logging._init_custom_logger_compatible_class) # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # legacy factory + resolved: Final = factory(callback, internal_usage_cache=None, llm_router=None) + return resolved if isinstance(resolved, CustomLogger) else None + + +def async_success_registry_has_type(callback: object) -> bool: + return any(type(existing) is type(callback) for existing in registry("async_success")) + + +def bootstrap_pending() -> bool: + from litellm import utils + + return not utils.callback_list + + +def bootstrap(function_id: str | None) -> None: + from litellm import utils + from litellm.litellm_core_utils import cached_imports + + combined: Final = _bootstrap_list() + utils.callback_list = combined # rebind-ok: legacy module global consumed by set_callbacks + set_callbacks: Final = _legacy(cached_imports.get_set_callbacks()) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # cached import is untyped + set_callbacks(callback_list=combined, function_id=function_id) + + +def _bootstrap_list() -> list[str]: # mutable-ok: set_callbacks and the legacy module global expect a list + names: Final = frozenset({*registry("input"), *registry("success"), *registry("failure")}) + consumed: Final = cast(Sequence[str], names) # cast-ok: legacy list annotation is narrower than its contents + return list(consumed) # mutable-ok: consumed by set_callbacks + + +def expand_named(callback: str, event: NamedEvent) -> None: + from litellm import utils + + utils._add_custom_logger_callback_to_specific_event(callback, event) # pyright: ignore[reportPrivateUsage] # legacy expansion helper + + +def append_registry(name: RegistryName, callback: CallbackTarget) -> None: + import litellm + + manager: Final = cast(_CallbackManager, litellm.logging_callback_manager) # cast-ok: legacy manager is untyped + match name: + case "input" | "async_input": + registry(name).append(callback) + case "success": + manager.add_litellm_success_callback(callback) + case "async_success": + manager.add_litellm_async_success_callback(callback) + case "failure": + manager.add_litellm_failure_callback(callback) + case "async_failure": + manager.add_litellm_async_failure_callback(callback) + case "callbacks": + raise KeyError(name) + + +def remove_registry(name: RegistryName, callback: CallbackTarget) -> None: + target: Final = registry(name) + for index in range(len(target) - 1, -1, -1): + if target[index] is callback or target[index] == callback: + del target[index] + return + + +def logger_fn(callback: object) -> None: + from litellm import utils + + utils.user_logger_fn = callback # rebind-ok: legacy module global read by pre_call + + +def breadcrumb(kwargs: Mapping[str, object]) -> None: + from litellm import utils + + if utils.add_breadcrumb is None: + return + import litellm + from litellm.litellm_core_utils import core_helpers + + deep_copy: Final = _legacy(core_helpers.safe_deep_copy) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # legacy helper + copied: Final = _copied(deep_copy, kwargs) + hidden: Final = frozenset(("messages", "input", "prompt")) if litellm.turn_off_message_logging else frozenset[str]() + details: Final = MappingProxyType({key: value for key, value in copied.items() if key not in hidden}) + _legacy(utils.add_breadcrumb)(category="litellm.llm_call", message=f"Keyword Args: {details}", level="info") + + +def _copied(deep_copy: Callable[..., object], kwargs: Mapping[str, object]) -> Mapping[str, object]: + try: + copied: Final = deep_copy(dict(kwargs)) # mutable-ok: safe_deep_copy expects a dict + except Exception: # noqa: BLE001 # legacy breadcrumb falls back to the live mapping + return kwargs + return _mapping(copied) or kwargs + + +def prepare_environment() -> None: + from litellm import utils + + utils.custom_llm_setup() + + +def applied_guardrails(kwargs: Mapping[str, object]) -> Sequence[str]: + from litellm.utils import get_applied_guardrails + + return tuple(get_applied_guardrails(dict(kwargs))) # mutable-ok: legacy helper expects a dict + + +def _litellm_params(kwargs: Mapping[str, object]) -> Mapping[str, object]: + metadata: Final = kwargs.get("metadata") + litellm_metadata: Final = kwargs.get("litellm_metadata") + base: Final = MappingProxyType({"api_base": ""}) + with_metadata: Final = MappingProxyType({**base, "metadata": metadata}) if "metadata" in kwargs else base + typed_metadata: Final = _mapping(litellm_metadata) + if typed_metadata is None: + return with_metadata + with_litellm_metadata: Final = MappingProxyType({**with_metadata, "litellm_metadata": typed_metadata}) + if metadata: + return with_litellm_metadata + copied_metadata: Final = dict(typed_metadata) # mutable-ok: callbacks may mutate this copy + return MappingProxyType({**with_litellm_metadata, "metadata": copied_metadata}) + + +def _owned( + callbacks: Sequence[CallbackTarget] | None, +) -> list[CallbackTarget] | None: # mutable-ok: Logging appends to these lists + return list(callbacks) if callbacks is not None else None # mutable-ok: Logging appends to these lists + + +def build_logging( + *, + call_type: str, + model: str | None, + kwargs: Kwargs, + start_time: datetime.datetime, + asynchronous: bool, + dynamic_success: Sequence[CallbackTarget] | None, + dynamic_async_success: Sequence[CallbackTarget] | None, + dynamic_failure: Sequence[CallbackTarget] | None, + guardrails: Sequence[str], +) -> Logging: + from litellm.litellm_core_utils.cached_imports import get_litellm_logging_class + + function_id: Final = kwargs.get("id") + trace_id: Final = kwargs.get("litellm_trace_id") + factory: Final = cast(_LoggingFactory, get_litellm_logging_class()) # cast-ok: legacy constructor is untyped + logger: Final = factory( + model=model, + messages="default-message-value", + stream=False, + litellm_call_id=str(kwargs["litellm_call_id"]), + litellm_trace_id=trace_id if isinstance(trace_id, str) else None, + function_id=function_id if isinstance(function_id, str) else "", + call_type=call_type, + start_time=start_time, + dynamic_success_callbacks=_owned(dynamic_success), + dynamic_failure_callbacks=_owned(dynamic_failure), + dynamic_async_success_callbacks=_owned(dynamic_async_success), + dynamic_async_failure_callbacks=None, + kwargs=kwargs, + applied_guardrails=list(guardrails), # mutable-ok: legacy constructor annotation is list + supports_correlation_logging=asynchronous, + ) + update: Final = cast(_EnvironmentUpdater, logger.update_environment_variables) # cast-ok: legacy method is untyped + update( + model=model, + user="", + optional_params=MappingProxyType({}), + litellm_params=_litellm_params(kwargs), + stream_options=kwargs.get("stream_options"), + ) + return logger diff --git a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py index 6295469c066..2c407f6c3a0 100644 --- a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py +++ b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py @@ -461,10 +461,16 @@ def test_native_pending_logging_is_released_only_for_ocr(call_type: str, excepti ) if call_type in ("ocr", "aocr"): - pending.release.assert_called_once_with(not exception_raised) + if exception_raised: + pending.close.assert_called_once_with() + pending.assert_not_called() + else: + pending.assert_called_once_with() + pending.close.assert_not_called() assert logger._native_pending_logging is None else: - pending.release.assert_not_called() + pending.assert_not_called() + pending.close.assert_not_called() assert logger._native_pending_logging is pending if exception_raised: enqueue.assert_not_called() diff --git a/tests/test_litellm/rust_bridge/test_bindings.py b/tests/test_litellm/rust_bridge/test_bindings.py index 88036a5a556..34e5e2c04ae 100644 --- a/tests/test_litellm/rust_bridge/test_bindings.py +++ b/tests/test_litellm/rust_bridge/test_bindings.py @@ -33,3 +33,17 @@ def test_binding_validates_native_attribute( binding: Final = bindings.NativeBinding("route", validate=lambda item: item if isinstance(item, int) else None) assert binding.load() == expected + + +@pytest.mark.parametrize("available", [False, True]) +def test_decline_accessor_only_catches_admission_declines(monkeypatch: pytest.MonkeyPatch, available: bool) -> None: + class Declined(Exception): + pass + + class Upstream(Exception): + pass + + native: Final = SimpleNamespace(RustBridgeDeclined=Declined, RustUpstreamError=Upstream) if available else None + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + assert bindings.native_decline_types() == ((Declined,) if available else ()) + assert not isinstance(Upstream("already dispatched"), bindings.native_decline_types()) diff --git a/tests/test_litellm/rust_bridge/test_leaves.py b/tests/test_litellm/rust_bridge/test_leaves.py new file mode 100644 index 00000000000..a8f594da3f4 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_leaves.py @@ -0,0 +1,282 @@ +import asyncio +import datetime +import inspect +from collections.abc import Iterator +from typing import Final +from unittest.mock import Mock + +import pytest + +from litellm.integrations.athina import AthinaLogger +from litellm.integrations.dynamodb import DyanmoDBLogger +from litellm.integrations.greenscale import GreenscaleLogger +from litellm.integrations.helicone import HeliconeLogger +from litellm.integrations.langfuse.langfuse import LangFuseLogger +from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger +from litellm.integrations.lunary import LunaryLogger +from litellm.integrations.openmeter import OpenMeterLogger +from litellm.integrations.prompt_layer import PromptLayerLogger +from litellm.integrations.s3 import S3Logger +from litellm.integrations.supabase import Supabase +from litellm.integrations.traceloop import TraceloopLogger +from litellm.integrations.weights_biases import WeightsBiasesLogger +from litellm.litellm_core_utils import litellm_logging +from litellm.rust_bridge import leaves + +START: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) +END: Final = START + datetime.timedelta(seconds=1) +DETAILS: Final[dict[str, object]] = { + "litellm_call_id": "call-id", + "input": "document", + "user": "user-1", + "original_response": "raw", + "litellm_params": {}, +} + + +def _logger() -> Mock: + return Mock( + model="mistral/mistral-ocr-latest", + messages="document", + litellm_call_id="call-id", + model_call_details=dict(DETAILS), + standard_callback_dynamic_params={}, + ) + + +def _stub(cls: type, method: str) -> tuple[object, Mock]: + instance: Final[object] = object.__new__(cls) + recorder: Final = Mock(name=f"{cls.__name__}.{method}") + setattr(instance, method, recorder) + return instance, recorder + + +def _assert_binds(cls: type, method: str, recorder: Mock, instance: object) -> None: + recorder.assert_called_once() + target: Final[object] = getattr(cls, method) + assert callable(target) + signature: Final = inspect.signature(target) + bound: Final = signature.bind(instance, *recorder.call_args.args, **recorder.call_args.kwargs) + required: Final = frozenset( + name for name, parameter in signature.parameters.items() if parameter.default is inspect.Parameter.empty + ) + assert required <= set(bound.arguments) + + +SUCCESS_CASES: Final[tuple[tuple[str, str, type, str], ...]] = ( + ("promptlayer", "promptLayerLogger", PromptLayerLogger, "log_event"), + ("wandb", "weightsBiasesLogger", WeightsBiasesLogger, "log_event"), + ("athina", "athinaLogger", AthinaLogger, "log_event"), + ("logfire", "logfireLogger", LogfireLogger, "log_event"), + ("greenscale", "greenscaleLogger", GreenscaleLogger, "log_event"), + ("supabase", "supabaseClient", Supabase, "log_event"), + ("lunary", "lunaryLogger", LunaryLogger, "log_event"), + ("helicone", "heliconeLogger", HeliconeLogger, "log_success"), + ("traceloop", "traceloopLogger", TraceloopLogger, "log_event"), + ("s3", "s3Logger", S3Logger, "log_event"), +) + +FAILURE_CASES: Final[tuple[tuple[str, str, type, str], ...]] = ( + ("lunary", "lunaryLogger", LunaryLogger, "log_event"), + ("supabase", "supabaseClient", Supabase, "log_event"), + ("traceloop", "traceloopLogger", TraceloopLogger, "log_event"), + ("logfire", "logfireLogger", LogfireLogger, "log_event"), +) + + +def _install(monkeypatch: pytest.MonkeyPatch, global_name: str, value: object) -> None: + monkeypatch.setattr(litellm_logging, global_name, value, raising=False) + + +@pytest.mark.parametrize(("name", "global_name", "cls", "method"), SUCCESS_CASES) +def test_named_success_calls_integration_with_its_real_signature( + monkeypatch: pytest.MonkeyPatch, name: str, global_name: str, cls: type, method: str +) -> None: + instance, recorder = _stub(cls, method) + _install(monkeypatch, global_name, instance) + response: Final = object() + + assert leaves.dispatch_named_success(_logger(), name, response, START, END) is None + + _assert_binds(cls, method, recorder, instance) + kwargs: Final = recorder.call_args.kwargs + assert kwargs["start_time"] is START and kwargs["end_time"] is END + assert kwargs["response_obj"] is response + assert callable(kwargs["print_verbose"]) + + +@pytest.mark.parametrize(("name", "global_name", "cls", "method"), FAILURE_CASES) +def test_named_failure_calls_integration_with_its_real_signature( + monkeypatch: pytest.MonkeyPatch, name: str, global_name: str, cls: type, method: str +) -> None: + instance, recorder = _stub(cls, method) + _install(monkeypatch, global_name, instance) + error: Final = RuntimeError("boom") + + leaves.dispatch_named_failure(_logger(), name, error, "formatted", START, END) + + _assert_binds(cls, method, recorder, instance) + kwargs: Final = recorder.call_args.kwargs + assert kwargs["start_time"] is START and kwargs["end_time"] is END + assert kwargs.get("response_obj") is None + + +def test_logfire_receives_enum_levels_and_exception_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: + instance, recorder = _stub(LogfireLogger, "log_event") + _install(monkeypatch, "logfireLogger", instance) + error: Final = RuntimeError("boom") + + leaves.dispatch_named_success(_logger(), "logfire", object(), START, END) + assert recorder.call_args.kwargs["level"] is LogfireLevel.INFO + assert "original_response" not in recorder.call_args.kwargs["kwargs"] + + leaves.dispatch_named_failure(_logger(), "logfire", error, "formatted", START, END) + assert recorder.call_args.kwargs["level"] is LogfireLevel.ERROR + assert recorder.call_args.kwargs["kwargs"]["exception"] is error + assert "original_response" not in recorder.call_args.kwargs["kwargs"] + + +def test_lunary_marks_success_and_error_events(monkeypatch: pytest.MonkeyPatch) -> None: + instance, recorder = _stub(LunaryLogger, "log_event") + _install(monkeypatch, "lunaryLogger", instance) + + leaves.dispatch_named_success(_logger(), "lunary", object(), START, END) + assert recorder.call_args.kwargs["event"] == "end" + assert recorder.call_args.kwargs["run_id"] == "call-id" + + leaves.dispatch_named_failure(_logger(), "lunary", RuntimeError("boom"), "formatted", START, END) + assert recorder.call_args.kwargs["event"] == "error" + assert recorder.call_args.kwargs["error"] == "formatted" + + +def test_supabase_request_hook_only_fires_pre_call(monkeypatch: pytest.MonkeyPatch) -> None: + instance, recorder = _stub(Supabase, "input_log_event") + _install(monkeypatch, "supabaseClient", instance) + + leaves.dispatch_named_request(_logger(), "supabase", "post_api_call") + recorder.assert_not_called() + + leaves.dispatch_named_request(_logger(), "supabase", "pre_api_call") + _assert_binds(Supabase, "input_log_event", recorder, instance) + assert recorder.call_args.kwargs["end_user"] == "user-1" + + +def test_sentry_hooks_are_called_when_configured(monkeypatch: pytest.MonkeyPatch) -> None: + breadcrumb: Final = Mock() + capture: Final = Mock() + _install(monkeypatch, "add_breadcrumb", breadcrumb) + _install(monkeypatch, "capture_exception", capture) + error: Final = RuntimeError("boom") + + leaves.dispatch_named_request(_logger(), "sentry", "pre_api_call") + assert breadcrumb.call_args.kwargs["category"] == "litellm.llm_call" + + leaves.dispatch_named_failure(_logger(), "sentry", error, "formatted", START, END) + capture.assert_called_once_with(error) + + +@pytest.mark.parametrize(("name", "global_name"), [(case[0], case[1]) for case in SUCCESS_CASES]) +def test_unconfigured_or_foreign_singleton_is_skipped( + monkeypatch: pytest.MonkeyPatch, name: str, global_name: str +) -> None: + _install(monkeypatch, global_name, None) + assert leaves.dispatch_named_success(_logger(), name, object(), START, END) is None + + foreign: Final = Mock() + _install(monkeypatch, global_name, foreign) + assert leaves.dispatch_named_success(_logger(), name, object(), START, END) is None + assert not foreign.method_calls + + +def _run(awaitable: object) -> None: + async def consume() -> None: + assert inspect.isawaitable(awaitable) + _ = await awaitable + + asyncio.run(consume()) + + +def test_openmeter_returns_the_async_coroutine(monkeypatch: pytest.MonkeyPatch) -> None: + instance: Final = object.__new__(OpenMeterLogger) + recorder: Final = Mock() + + async def async_log_success_event(**kwargs: object) -> None: + recorder(**kwargs) + + monkeypatch.setattr(instance, "async_log_success_event", async_log_success_event) + _install(monkeypatch, "openMeterLogger", instance) + response: Final = object() + + _run(leaves.dispatch_named_success(_logger(), "openmeter", response, START, END)) + + _assert_binds(OpenMeterLogger, "async_log_success_event", recorder, instance) + assert recorder.call_args.kwargs["response_obj"] is response + + +def test_dynamodb_reuses_the_existing_singleton(monkeypatch: pytest.MonkeyPatch) -> None: + instance: Final = object.__new__(DyanmoDBLogger) + recorder: Final = Mock() + + async def _async_log_event(**kwargs: object) -> None: + recorder(**kwargs) + + monkeypatch.setattr(instance, "_async_log_event", _async_log_event) + _install(monkeypatch, "dynamoLogger", instance) + + _run(leaves.dispatch_named_success(_logger(), "dynamodb", object(), START, END)) + + _assert_binds(DyanmoDBLogger, "_async_log_event", recorder, instance) + assert litellm_logging.dynamoLogger is instance + + +@pytest.fixture +def langfuse_handler(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[LangFuseLogger, Mock, Mock]]: + handler: Final = object.__new__(LangFuseLogger) + recorder: Final = Mock(return_value={"trace_id": "trace-1"}) + monkeypatch.setattr(handler, "log_event_on_langfuse", recorder) + _install(monkeypatch, "langFuseLogger", handler) + cache: Final = Mock() + _install(monkeypatch, "in_memory_trace_id_cache", cache) + yield handler, recorder, cache + + +def test_langfuse_success_logs_and_caches_trace_id(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None: + handler, recorder, cache = langfuse_handler + response: Final = object() + + leaves.dispatch_named_success(_logger(), "langfuse", response, START, END) + + _assert_binds(LangFuseLogger, "log_event_on_langfuse", recorder, handler) + kwargs: Final = recorder.call_args.kwargs + assert kwargs["response_obj"] is response + assert kwargs["user_id"] == "user-1" + assert kwargs["level"] == "DEFAULT" and kwargs["status_message"] is None + assert "original_response" not in kwargs["kwargs"] + cache.set_cache.assert_called_once_with(litellm_call_id="call-id", service_name="langfuse", trace_id="trace-1") + + +def test_langfuse_failure_logs_error_level(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None: + handler, recorder, _ = langfuse_handler + + leaves.dispatch_named_failure(_logger(), "langfuse", RuntimeError("boom"), "formatted", START, END) + + _assert_binds(LangFuseLogger, "log_event_on_langfuse", recorder, handler) + kwargs: Final = recorder.call_args.kwargs + assert kwargs["response_obj"] is None + assert kwargs["level"] == "ERROR" and kwargs["status_message"] == "boom" + + +def test_langfuse_skips_cache_without_trace_id(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None: + _, recorder, cache = langfuse_handler + recorder.return_value = {} + + leaves.dispatch_named_success(_logger(), "langfuse", object(), START, END) + + cache.set_cache.assert_not_called() + + +def test_signature_binding_rejects_drift() -> None: + instance, recorder = _stub(Supabase, "log_event") + recorder(model="m", end_userr="typo", response_obj=None, start_time=START, end_time=END) + with pytest.raises(TypeError): + _assert_binds(Supabase, "log_event", recorder, instance) diff --git a/tests/test_litellm/rust_bridge/test_ocr_lifecycle.py b/tests/test_litellm/rust_bridge/test_ocr_lifecycle.py index baf5295d571..272974b2689 100644 --- a/tests/test_litellm/rust_bridge/test_ocr_lifecycle.py +++ b/tests/test_litellm/rust_bridge/test_ocr_lifecycle.py @@ -8,7 +8,8 @@ import litellm from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.ocr import legacy from litellm.rust_bridge import bindings, configuration -from litellm.rust_bridge.ocr import NATIVE_AOCR, NATIVE_OCR +from litellm.rust_bridge.leaves import record_post_call, record_pre_call +from litellm.rust_bridge.ocr import NATIVE_AOCR, NATIVE_OCR, update_logging @pytest.fixture(autouse=True) @@ -21,6 +22,73 @@ def isolated_ocr_configuration(monkeypatch: pytest.MonkeyPatch) -> Generator[Non configuration.reset_rust_configuration() +def test_logging_redacts_views_and_preserves_opaque_arguments_and_pricing() -> None: + logger: Final = Mock() + opaque: Final = object() + logger_fn: Final = object() + kwargs: Final = { + "vertex_credentials": "secret", + "proxy_server_request": opaque, + "metadata": opaque, + "logger_fn": logger_fn, + "litellm_request_debug": False, + "litellm_call_id": "call-id", + "input_cost_per_token": 0, + "output_cost_per_token": None, + } + optional: Final = {"vertex_credentials": "secret", "pages": [1], "proxy_server_request": opaque} + update_logging(logger, kwargs, "model", "vertex_ai", optional, ("vertex_credentials",), "https://provider") + logger.update_from_kwargs.assert_called_once_with( + kwargs={ + "vertex_credentials": "****", + "metadata": opaque, + "logger_fn": logger_fn, + "litellm_request_debug": False, + "litellm_call_id": "call-id", + "input_cost_per_token": 0, + "output_cost_per_token": None, + }, + model="model", + custom_llm_provider="vertex_ai", + optional_params={"vertex_credentials": "****", "pages": [1]}, + litellm_params={ + "litellm_call_id": "call-id", + "api_base": "https://provider", + "logger_fn": logger_fn, + "litellm_request_debug": False, + "input_cost_per_token": 0, + }, + ) + assert kwargs["vertex_credentials"] == optional["vertex_credentials"] == "secret" + assert kwargs["proxy_server_request"] is optional["proxy_server_request"] is opaque + assert logger.update_from_kwargs.call_args.kwargs["kwargs"]["metadata"] is opaque + assert logger.update_from_kwargs.call_args.kwargs["optional_params"]["pages"] is optional["pages"] + + +def test_logging_callbacks_receive_captured_payload_roots_and_propagate_errors() -> None: + logger: Final = Mock(log_raw_request_response=False, logger_fn=None, model_call_details={}) + body: Final[dict[str, object]] = {"document": "original"} + headers: Final = {"authorization": "key"} + response: Final = object() + record_pre_call(logger, api_key="key", body=body, headers=headers, url="https://provider") + record_post_call(logger, original_response=response, body=body, headers=headers) + logger._pre_call.assert_called_once_with( + input="OCR document processing", + api_key="key", + model=None, + additional_args={"complete_input_dict": body, "headers": headers, "api_base": "https://provider"}, + ) + for callback in (logger._pre_call, logger.record_post_call): + assert callback.call_args.kwargs["additional_args"]["complete_input_dict"] is body + assert callback.call_args.kwargs["additional_args"]["headers"] is headers + assert logger.record_post_call.call_args.kwargs["original_response"] is response + failure: Final = RuntimeError("callback failed") + failing_logger: Final = Mock(_pre_call=Mock(side_effect=failure)) + with pytest.raises(RuntimeError) as caught: + record_pre_call(failing_logger, api_key=None, body=body, headers=headers, url="https://provider") + assert caught.value is failure + + @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True]) async def test_unavailable_native_uses_legacy(monkeypatch: pytest.MonkeyPatch, asynchronous: bool) -> None: diff --git a/tests/test_litellm/rust_bridge/test_setup.py b/tests/test_litellm/rust_bridge/test_setup.py new file mode 100644 index 00000000000..ecfd68349c8 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_setup.py @@ -0,0 +1,169 @@ +import datetime +from collections.abc import Iterator +from typing import Final + +import pytest + +import litellm +from litellm import utils +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.rust_bridge.loader import native_bridge_available + +pytestmark = pytest.mark.skipif(not native_bridge_available(), reason="requires the Rust extension") + +REGISTRIES: Final = ( + "input_callback", + "_async_input_callback", + "success_callback", + "_async_success_callback", + "failure_callback", + "_async_failure_callback", + "callbacks", +) + + +@pytest.fixture +def clean_registries(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + for name in REGISTRIES: + monkeypatch.setattr(litellm, name, []) + monkeypatch.setattr(utils, "callback_list", []) + monkeypatch.setattr(utils, "user_logger_fn", None) + monkeypatch.setenv("OPENMETER_API_KEY", "test") + monkeypatch.setenv("OPENMETER_API_ENDPOINT", "http://127.0.0.1:9") + yield + + +class SyncLogger(CustomLogger): + pass + + +class AsyncOnly(CustomLogger): + pass + + +def sync_fn(*args: object, **kwargs: object) -> None: + del args, kwargs + + +async def async_fn(*args: object, **kwargs: object) -> None: + del args, kwargs + + +def snapshot() -> dict[str, list[object]]: + return {name: list(getattr(litellm, name)) for name in REGISTRIES} + + +def run_legacy(kwargs: dict[str, object]) -> tuple[Logging, dict[str, object]]: + logger, prepared = utils.function_setup( + "ocr", + utils.Rules(), + datetime.datetime.now(), + is_async_call=False, + **{"litellm_call_id": "legacy", **kwargs}, + ) + assert isinstance(logger, Logging) + return logger, prepared + + +def run_native(kwargs: dict[str, object]) -> tuple[Logging, dict[str, object]]: + from litellm.rust_bridge import _native + + logger, prepared = _native._debug_setup( + "ocr", (), {"litellm_call_id": "native", **kwargs}, datetime.datetime.now(), False + ) + assert isinstance(logger, Logging) + return logger, prepared + + +@pytest.mark.parametrize( + "globals_before, kwargs", + [ + ({}, {}), + ({"callbacks": [SyncLogger()]}, {}), + ({"callbacks": [async_fn]}, {}), + ({}, {"callbacks": [SyncLogger(), sync_fn]}), + ({"success_callback": [async_fn, "openmeter", sync_fn]}, {}), + ({"failure_callback": [async_fn, sync_fn]}, {}), + ({"input_callback": [async_fn, sync_fn]}, {}), + ({}, {"success_callback": [sync_fn, async_fn, "s3", "dynamodb"], "failure_callback": [sync_fn]}), + ({"callbacks": [SyncLogger()]}, {"callbacks": [SyncLogger()], "success_callback": [async_fn]}), + ], + ids=[ + "empty", + "global-custom-logger", + "global-async-callable", + "dynamic-callbacks", + "success-safety-net", + "failure-safety-net", + "input-safety-net", + "per-call-success-failure-split", + "mixed", + ], +) +def test_native_setup_registry_side_effects_match_function_setup( + clean_registries: None, globals_before: dict[str, list[object]], kwargs: dict[str, object] +) -> None: + for name, values in globals_before.items(): + getattr(litellm, name).extend(values) + legacy_logger, legacy_kwargs = run_legacy( + {key: list(value) if isinstance(value, list) else value for key, value in kwargs.items()} + ) + legacy_snapshot: Final = snapshot() + legacy_bootstrap: Final = list(utils.callback_list or []) + + for name in REGISTRIES: + getattr(litellm, name).clear() + utils.callback_list = [] + for name, values in globals_before.items(): + getattr(litellm, name).extend(values) + native_logger, native_kwargs = run_native( + {key: list(value) if isinstance(value, list) else value for key, value in kwargs.items()} + ) + + assert snapshot() == legacy_snapshot + assert sorted(map(repr, utils.callback_list or [])) == sorted(map(repr, legacy_bootstrap)) + assert set(native_kwargs) - {"litellm_call_id"} == set(legacy_kwargs) - {"litellm_call_id"} + for attribute in ( + "dynamic_success_callbacks", + "dynamic_async_success_callbacks", + "dynamic_failure_callbacks", + "dynamic_async_failure_callbacks", + "call_type", + "stream", + "model", + ): + assert getattr(native_logger, attribute) == getattr(legacy_logger, attribute), attribute + + +def test_native_setup_honours_caller_supplied_logging_object(clean_registries: None) -> None: + class Supplied(Logging): + pass + + supplied: Final = Supplied( + model="mistral/mistral-ocr-latest", + messages=[], + stream=False, + call_type="ocr", + start_time=datetime.datetime.now(), + litellm_call_id="supplied", + function_id="", + ) + litellm.callbacks.append(SyncLogger()) + logger, kwargs = run_native({"litellm_logging_obj": supplied, "callbacks": [SyncLogger()]}) + assert logger is supplied + assert kwargs["callbacks"] is not None + assert litellm.success_callback == [] + + +def test_native_setup_records_logger_fn_and_metadata(clean_registries: None) -> None: + def logger_fn(details: object) -> None: + del details + + logger, kwargs = run_native( + {"model": "mistral/mistral-ocr-latest", "logger_fn": logger_fn, "metadata": {"source": "test"}} + ) + assert utils.user_logger_fn is logger_fn + assert logger.litellm_params["metadata"] == {"source": "test"} + assert logger.model == "mistral/mistral-ocr-latest" + assert kwargs["metadata"] == {"source": "test"} diff --git a/tests/test_litellm_rust/ocr/test_callbacks.py b/tests/test_litellm_rust/ocr/test_callbacks.py index 1cfd04b1bff..904504150a7 100644 --- a/tests/test_litellm_rust/ocr/test_callbacks.py +++ b/tests/test_litellm_rust/ocr/test_callbacks.py @@ -10,6 +10,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.ocr.transformation import OCRResponse from tests.test_litellm_rust.support.callback_recorder import RecordingLogger +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec from tests.test_litellm_rust.support.requests import ( OCR_DOCUMENT, OCR_RESPONSE, @@ -18,7 +19,6 @@ from tests.test_litellm_rust.support.requests import ( request_body, request_headers, ) -from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec pytestmark = pytest.mark.requires_rust_extension @@ -119,69 +119,6 @@ def test_native_ocr_pre_call_header_rebinding_does_not_replace_execution_root(oc assert "x-rebound" not in ocr_server.requests[0].headers -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) -async def test_native_ocr_pre_call_nested_document_edit_updates_caller_callback_and_provider_references( - ocr_server: RecordingServer, asynchronous: bool -) -> None: - original: Final = dict(OCR_DOCUMENT) - replacement_url: Final = "data:application/pdf;base64,ZGVm" - retained: Final = [] - aliases: Final = [] - - class Retain(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - aliases.append(request_body(kwargs)["document"] is original) - retained.append(request_body(kwargs)["document"]) - - class Edit(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - original["document_url"] = replacement_url - - arguments: Final = { - "model": "mistral/mistral-ocr-latest", - "document": original, - "api_key": "test-key", - "api_base": ocr_server.base_url, - "callbacks": [Retain(), Edit()], - } - response: Final = ( - await call_native_aocr(ocr_server, **arguments) - if asynchronous - else call_native_ocr(ocr_server, **arguments) - ) - - assert aliases == [True] - assert retained[0]["document_url"] == replacement_url - assert original["document_url"] == replacement_url - assert ocr_server.requests[0].body["document"]["document_url"] == replacement_url - assert response.pages[0].markdown == "native OCR response" - - -def test_native_ocr_pre_call_document_replacement_does_not_mutate_original_document( - ocr_server: RecordingServer, -) -> None: - original: Final = dict(OCR_DOCUMENT) - replacement: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,ZGVm"} - retained: Final = [] - - class RetainAndReplace(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - body = request_body(kwargs) - retained.append(body["document"]) - body["document"] = replacement - - call_native_ocr( - ocr_server, - document=original, - callbacks=[RetainAndReplace()], - ) - - assert retained[0] is original - assert original["document_url"] == OCR_DOCUMENT["document_url"] - assert ocr_server.requests[0].body["document"] == replacement - - def test_native_ocr_pre_call_body_rebinding_is_visible_to_callbacks_but_not_provider( ocr_server: RecordingServer, ) -> None: @@ -372,6 +309,7 @@ async def test_native_azure_ocr_resolves_token_before_pre_call_on_caller_context asynchronous: bool, ) -> None: from contextvars import ContextVar + context: Final = ContextVar("azure-token-context", default="missing") context.set("caller") caller_thread: Final = threading.current_thread() @@ -400,9 +338,7 @@ async def test_native_azure_ocr_resolves_token_before_pre_call_on_caller_context "callbacks": [Edit()], } response: Final = ( - await call_native_aocr(ocr_server, **arguments) - if asynchronous - else call_native_ocr(ocr_server, **arguments) + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) ) assert response.pages[0].markdown == "native OCR response" assert observations == ["token", "pre_call"] @@ -431,9 +367,7 @@ async def test_native_azure_ocr_token_provider_can_make_nested_native_ocr_call( "azure_ad_token_provider": provider, } response: Final = ( - await call_native_aocr(ocr_server, **arguments) - if asynchronous - else call_native_ocr(ocr_server, **arguments) + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) ) assert response.pages[0].markdown == "native OCR response" assert calls == ["token"] @@ -488,7 +422,9 @@ async def test_native_azure_ocr_releases_token_provider_after_terminal_outcome( ) -> None: import gc import weakref + from tests.test_litellm_rust.support.callback_recorder import drain_logging + class Provider: def __call__(self) -> str: if outcome == "failure": @@ -534,3 +470,274 @@ async def test_native_azure_ocr_releases_token_provider_after_terminal_outcome( await asyncio.sleep(0) gc.collect() assert reference() is None + + +FORBIDDEN_ORCHESTRATION: Final = ( + "pre_call", + "post_call", + "success_handler", + "async_success_handler", + "failure_handler", + "async_failure_handler", + "_success_handler_body", + "_async_success_handler_body", + "_failure_handler_body", + "_async_failure_handler_body", + "dispatch_success_handlers", + "dispatch_failure_handlers", + "handle_sync_success_callbacks_for_async_calls", +) + + +@pytest.fixture +def legacy_orchestration_disabled(monkeypatch: pytest.MonkeyPatch) -> list[str]: + from litellm import utils + from litellm.litellm_core_utils.litellm_logging import Logging + + reached: Final[list[str]] = [] + + def forbid(name: str): + def method(self, *args, **kwargs): + reached.append(name) + raise AssertionError(f"legacy orchestration reached: {name}") + + return method + + for name in FORBIDDEN_ORCHESTRATION: + monkeypatch.setattr(Logging, name, forbid(name)) + + def forbid_utils(name: str): + async def hook(*args, **kwargs): + reached.append(name) + raise AssertionError(f"legacy orchestration reached: {name}") + + def setup(*args, **kwargs): + reached.append(name) + raise AssertionError(f"legacy orchestration reached: {name}") + + return setup if name == "function_setup" else hook + + for name in ( + "function_setup", + "async_pre_call_deployment_hook", + "async_post_call_success_deployment_hook", + "async_post_call_failure_deployment_hook", + ): + monkeypatch.setattr(utils, name, forbid_utils(name)) + return reached + + +class DeploymentRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.hooks: list[str] = [] + + async def async_pre_call_deployment_hook(self, kwargs, call_type): + self.hooks.append(f"pre:{call_type.value}") + return {**kwargs, "metadata": {**(kwargs.get("metadata") or {}), "deployment": "seen"}} + + async def async_post_call_success_deployment_hook(self, request_data, response, call_type): + self.hooks.append(f"post:{request_data['metadata']['deployment']}") + return response + + async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None): + self.hooks.append(f"failure:{type(exception).__name__}:{fallback_depth}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_ocr_success_runs_integrations_without_legacy_orchestration( + ocr_server: RecordingServer, legacy_orchestration_disabled: list[str], asynchronous: bool +) -> None: + recorder: Final = RecordingLogger() + deployment: Final = DeploymentRecorder() + litellm.callbacks.append(deployment) + arguments: Final = {"callbacks": [recorder]} + response: Final = ( + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + success_event: Final = "async_log_success_event" if asynchronous else "log_success_event" + events: Final = await recorder.wait_for_async(success_event) + assert legacy_orchestration_disabled == [] + assert deployment.hooks == (["pre:aocr", "post:seen"] if asynchronous else []) + assert recorder.names.count("log_pre_api_call") == 1 + assert events[0].kwargs["standard_logging_object"]["status"] == "success" + assert events[0].kwargs["response_cost"] is not None + assert events[0].kwargs["litellm_params"]["api_base"] == f"{ocr_server.base_url}/v1/ocr" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_ocr_failure_runs_integrations_without_legacy_orchestration( + ocr_server: RecordingServer, legacy_orchestration_disabled: list[str], asynchronous: bool +) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500)) + recorder: Final = RecordingLogger() + deployment: Final = DeploymentRecorder() + litellm.callbacks.append(deployment) + arguments: Final = {"callbacks": [recorder]} + with pytest.raises(litellm.InternalServerError) as caught: + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) + assert legacy_orchestration_disabled == [] + assert deployment.hooks == (["pre:aocr", "failure:InternalServerError:None"] if asynchronous else []) + failures: Final = tuple(event for event in recorder.events if event.name.endswith("log_failure_event")) + assert [event.name for event in failures] == ( + ["log_failure_event", "async_log_failure_event"] if asynchronous else ["log_failure_event"] + ) + assert all(event.kwargs["exception"] is caught.value for event in failures) + assert all(event.kwargs["standard_logging_object"]["status"] == "failure" for event in failures) + assert "log_success_event" not in recorder.names + + +def test_native_ocr_success_hooks_all_run_before_any_success_dispatch(ocr_server: RecordingServer) -> None: + order: Final = [] + finished: Final = threading.Event() + + class Hooked(CustomLogger): + def __init__(self, name: str) -> None: + super().__init__() + self.name = name + + def logging_hook(self, kwargs, result, call_type): + order.append(("hook", self.name)) + kwargs[f"seen-by-{self.name}"] = True + return kwargs, result + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + order.append(("log", self.name, kwargs.get("seen-by-a"), kwargs.get("seen-by-b"))) + if self.name == "b": + finished.set() + + call_native_ocr_with_callbacks(ocr_server, [Hooked("a"), Hooked("b")]) + + assert finished.wait(10) + assert order == [("hook", "a"), ("hook", "b"), ("log", "a", True, True), ("log", "b", True, True)] + + +def test_native_ocr_hook_failure_is_contained_and_target_still_dispatches(ocr_server: RecordingServer) -> None: + order: Final = [] + finished: Final = threading.Event() + + class Broken(CustomLogger): + def logging_hook(self, kwargs, result, call_type): + raise RuntimeError("hook failed") + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + order.append("broken-log") + + class Healthy(CustomLogger): + def logging_hook(self, kwargs, result, call_type): + order.append("healthy-hook") + return kwargs, result + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + order.append("healthy-log") + finished.set() + + call_native_ocr_with_callbacks(ocr_server, [Broken(), Healthy()]) + + assert finished.wait(10) + assert order == ["healthy-hook", "broken-log", "healthy-log"] + + +@pytest.mark.asyncio +async def test_native_aocr_hook_replacement_of_result_reaches_success_dispatch(ocr_server: RecordingServer) -> None: + replacement: Final = object() + observed: Final = [] + + class Replace(CustomLogger): + async def async_logging_hook(self, kwargs, result, call_type): + return kwargs, replacement + + class Observe(CustomLogger): + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + observed.append(response_obj) + + recorder: Final = RecordingLogger() + await call_native_aocr_with_callbacks(ocr_server, [Replace(), Observe(), recorder]) + await recorder.wait_for_async("async_log_success_event") + + assert observed == [replacement] + + +@pytest.mark.asyncio +async def test_native_aocr_shared_logging_object_dispatches_success_once(ocr_server: RecordingServer) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + + ocr_server.expected_requests = 2 + recorder: Final = RecordingLogger() + litellm.callbacks.append(recorder) + logger: Final = Logging( + model="mistral-ocr-latest", + messages=[], + stream=False, + call_type="aocr", + start_time=__import__("datetime").datetime.now(), + litellm_call_id="shared", + function_id="shared", + ) + logger.dynamic_async_success_callbacks = [recorder] + + await call_native_aocr(ocr_server, litellm_logging_obj=logger) + await call_native_aocr(ocr_server, litellm_logging_obj=logger) + await recorder.wait_for_async("async_log_success_event") + from tests.test_litellm_rust.support.callback_recorder import drain_logging + + await drain_logging() + + assert logger.model_call_details["has_logged_async_success"] is True + assert recorder.names.count("async_log_success_event") == 1 + + +def test_native_ocr_logging_preparation_failure_does_not_fail_request( + ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.litellm_core_utils import litellm_logging + + def broken_payload(*args, **kwargs): + raise RuntimeError("payload unavailable") + + monkeypatch.setattr(litellm_logging, "get_standard_logging_object_payload", broken_payload) + unraisable: Final = [] + monkeypatch.setattr(__import__("sys"), "unraisablehook", lambda event: unraisable.append(event.exc_value)) + recorder: Final = RecordingLogger() + + response: Final = call_native_ocr_with_callbacks(ocr_server, [recorder]) + events: Final = recorder.wait_for("log_success_event") + + assert response.pages[0].markdown == "native OCR response" + assert len(events) == 1 + assert events[0].thread is not threading.current_thread() + assert any(str(error) == "payload unavailable" for error in unraisable) + + +def test_native_ocr_writes_success_marker_and_honours_existing_marker( + ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.rust_bridge import setup as native_setup + + ocr_server.expected_requests = 2 + loggers: Final = [] + original_build: Final = native_setup.build_logging + + def build_logging(**kwargs): + logger = original_build(**kwargs) + if loggers: + logger.model_call_details["has_logged_sync_success"] = True + loggers.append(logger) + return logger + + monkeypatch.setattr(native_setup, "build_logging", build_logging) + recorder: Final = RecordingLogger() + + call_native_ocr_with_callbacks(ocr_server, [recorder]) + recorder.wait_for("log_success_event") + assert loggers[0].model_call_details["has_logged_sync_success"] is True + + call_native_ocr_with_callbacks(ocr_server, [recorder]) + from litellm.litellm_core_utils.thread_pool_executor import executor + + executor.submit(lambda: None).result(10) + assert recorder.names.count("log_success_event") == 1 + assert recorder.names.count("logging_hook") == 1 diff --git a/tests/test_litellm_rust/ocr/test_lifecycle.py b/tests/test_litellm_rust/ocr/test_lifecycle.py index b0df9168719..04cc6eb41fc 100644 --- a/tests/test_litellm_rust/ocr/test_lifecycle.py +++ b/tests/test_litellm_rust/ocr/test_lifecycle.py @@ -79,7 +79,7 @@ async def test_native_bindings_run_callbacks_and_send_their_mutations( class Observe(Logging): def pre_call(self, input, api_key, additional_args): body: Final = additional_args["complete_input_dict"] - assert body["pages"] is pages + assert body["pages"] == pages and body["pages"] is not pages observed.append(self.model_call_details["litellm_params"]["metadata"]["marker"]) body["pages"].append(2) additional_args["headers"]["x-callback"] = "native" @@ -98,6 +98,7 @@ async def test_native_bindings_run_callbacks_and_send_their_mutations( assert response.pages[0].markdown == "native OCR response" assert observed == [marker] and observed[0] is marker + assert pages == [0] assert len(ocr_server.requests) == 1 assert ocr_server.requests[0].body["pages"] == [0, 2] assert ocr_server.requests[0].headers["x-callback"] == "native" @@ -263,7 +264,6 @@ async def test_deployment_hook_replaces_complete_routing_request(ocr_server: Rec assert response.pages[0].markdown == "native OCR response" assert observed == [(replacement, "replacement-key")] - assert observed[0][0] is replacement assert replacement == original assert replacement is not original assert original == {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"} @@ -621,7 +621,7 @@ def test_sync_pre_call_can_make_nested_native_request(ocr_server: RecordingServe @pytest.mark.asyncio -async def test_retained_argument_aliases_and_body_roots_survive_envelope_replacement( +async def test_callback_body_roots_survive_envelope_replacement( ocr_server: RecordingServer, ) -> None: pages: Final = [0] @@ -633,8 +633,8 @@ async def test_retained_argument_aliases_and_body_roots_survive_envelope_replace def pre_call(self, input, api_key, additional_args): body: Final = additional_args["complete_input_dict"] headers: Final = additional_args["headers"] - observed.append((body["document"] is document, body["pages"] is pages)) - pages.append(2) + observed.append((body["document"] == document, body["pages"] == pages)) + body["pages"].append(2) headers["x-retained"] = "yes" additional_args["complete_input_dict"] = {"discarded": True} additional_args["headers"] = {} @@ -672,6 +672,7 @@ async def test_retained_argument_aliases_and_body_roots_survive_envelope_replace assert observed[0] == (False, False, True) assert observed[1] == (True, True) assert observed[3] == (True, True) + assert pages == [0] assert ocr_server.requests[0].body["pages"] == [0, 2] assert ocr_server.requests[0].headers["x-retained"] == "yes" @@ -933,72 +934,6 @@ async def test_response_limit_is_enforced_at_the_public_boundary( assert "max_response_bytes" not in body -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True]) -@pytest.mark.parametrize("failure", [False, True]) -async def test_empty_callbacks_keep_bookkeeping_without_optional_dispatch( - ocr_server: RecordingServer, - monkeypatch: pytest.MonkeyPatch, - asynchronous: bool, - failure: bool, - created_loggers: list[Logging], -) -> None: - from litellm import utils - from litellm.litellm_core_utils import litellm_logging, logging_worker - - class DispatchProbe: - deployments = 0 - submissions = 0 - enqueues = 0 - - def deployment(self, *args: object, **kwargs: object) -> None: - self.deployments += 1 - - def submit(self, *args: object, **kwargs: object) -> None: - self.submissions += 1 - - def ensure_initialized_and_enqueue(self, coroutine: Coroutine[object, object, object]) -> None: - self.enqueues += 1 - coroutine.close() - - probe: Final = DispatchProbe() - for name in ( - "async_pre_call_deployment_hook", - "async_post_call_success_deployment_hook", - "async_post_call_failure_deployment_hook", - ): - monkeypatch.setattr(utils, name, probe.deployment) - monkeypatch.setattr(litellm_logging, "executor", probe) - monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", probe) - if failure: - ocr_server.enqueue(ResponseSpec(body={"message": "provider failed"}, status=500)) - trace_id_var.set("callback-free-parent") - arguments: Final = {"litellm_trace_id": "callback-free-call", "litellm_call_id": "callback-free-id"} - if failure: - with pytest.raises(litellm.InternalServerError): - await call_aocr(ocr_server, **arguments) if asynchronous else call_ocr(ocr_server, **arguments) - else: - response: Final = ( - await call_aocr(ocr_server, **arguments) if asynchronous else call_ocr(ocr_server, **arguments) - ) - assert response.pages[0].markdown == "native OCR response" - assert response._hidden_params["litellm_call_id"] == "callback-free-id" - assert response._hidden_params["response_cost"] is not None - assert response._hidden_params["_response_ms"] > 0 - assert trace_id_var.get() == "callback-free-parent" - assert probe.deployments == probe.submissions == probe.enqueues == 0 - assert len(created_loggers) == 1 - logger: Final = created_loggers[0] - assert not hasattr(logger, "_native_pending_logging") - assert logger.model_call_details["first_api_call_start_time"] <= logger.model_call_details["end_time"] - assert "standard_logging_object" not in logger.model_call_details - assert ( - "original_response" not in logger.model_call_details or logger.model_call_details["original_response"] is None - ) - assert "complete_input_dict" not in logger.model_call_details.get("additional_args", {}) - assert logger.model_call_details["response_cost"] == (0 if failure else response._hidden_params["response_cost"]) - - @pytest.mark.asyncio @pytest.mark.parametrize( "registration", ["success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"] @@ -1032,26 +967,18 @@ async def test_terminal_registration_added_during_http_is_observed( @pytest.fixture def created_loggers(monkeypatch: pytest.MonkeyPatch) -> list[Logging]: - from litellm import utils + from litellm.rust_bridge import setup as native_setup - original_setup: Final = utils.function_setup + original_build: Final = native_setup.build_logging loggers: Final[list[Logging]] = [] - def setup( - call_type: str, - rules: utils.Rules, - start: datetime.datetime, - *args: object, - is_async_call: bool = True, - **kwargs: object, - ) -> tuple[Logging, dict[str, object]]: - logger, prepared = original_setup(call_type, rules, start, *args, is_async_call=is_async_call, **kwargs) - assert isinstance(logger, Logging) + def build_logging(**kwargs: object) -> Logging: + logger: Final = original_build(**kwargs) # pyright: ignore[reportArgumentType] # passthrough of the factory signature setattr(logger, "_defer_async_logging", True) loggers.append(logger) - return logger, prepared + return logger - monkeypatch.setattr(utils, "function_setup", setup) + monkeypatch.setattr(native_setup, "build_logging", build_logging) return loggers @@ -1076,30 +1003,3 @@ async def test_explicit_logging_consumers_keep_request_and_response_payloads( assert details["raw_request_typed_dict"]["raw_request_body"]["model"] == "mistral-ocr-latest" if consumer == "logger_fn": assert [item["log_event_type"] for item in snapshots] == ["pre_api_call", "post_api_call"] - - -@pytest.mark.asyncio -async def test_registration_removed_before_deferred_release_skips_queue( - ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch, created_loggers: list[Logging] -) -> None: - from litellm.litellm_core_utils import logging_worker - - class QueueProbe: - enqueues = 0 - - def ensure_initialized_and_enqueue(self, coroutine: Coroutine[object, object, object]) -> None: - self.enqueues += 1 - coroutine.close() - - observer: Final = RecordingLogger() - litellm._async_success_callback.append(observer) - await call_aocr(ocr_server) - logger: Final = created_loggers[0] - assert hasattr(logger, "_native_pending_logging") - litellm._async_success_callback.clear() - probe: Final = QueueProbe() - monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", probe) - ProxyBaseLLMRequestProcessing._flush_deferred_async_logging(logger, False) - assert probe.enqueues == 0 - assert not observer.names - assert logger.model_call_details["response_cost"] is not None diff --git a/tests/test_litellm_rust/ocr/test_requests.py b/tests/test_litellm_rust/ocr/test_requests.py index 5bcc65d75af..47eb4f0f577 100644 --- a/tests/test_litellm_rust/ocr/test_requests.py +++ b/tests/test_litellm_rust/ocr/test_requests.py @@ -12,6 +12,7 @@ from tests.test_litellm_rust.support.requests import ( OCR_RESPONSE, call_native_aocr, call_native_ocr, + request_body, ) pytestmark = pytest.mark.requires_rust_extension @@ -535,7 +536,7 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen class Observer(RecordingLogger): def log_pre_api_call(self, model, messages, kwargs): super().log_pre_api_call(model, messages, kwargs) - pages.append(2) + request_body(kwargs)["pages"].append(2) arguments: Final = { "model": "mistral/mistral-ocr-latest", @@ -548,3 +549,4 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen assert response.pages[0].markdown == "native OCR response" assert ocr_server.requests[0].headers["authorization"] == f"Bearer {expected_key}" assert ocr_server.requests[0].body["pages"] == [0, 2] + assert pages == [0] diff --git a/tests/test_litellm_rust/test_ocr.py b/tests/test_litellm_rust/test_ocr.py index a83748c0dd6..86913f388db 100644 --- a/tests/test_litellm_rust/test_ocr.py +++ b/tests/test_litellm_rust/test_ocr.py @@ -1,14 +1,15 @@ -import json import asyncio import base64 +import gc +import json +import os import threading import weakref -import gc -from pathlib import Path -from collections.abc import Generator +from collections.abc import Coroutine, Generator from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from io import BytesIO +from pathlib import Path from typing import Final, Protocol import httpx @@ -340,7 +341,12 @@ def test_native_lifecycle_core_encodes_python_file_input( @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("source", ["path", "pathlike", "reader", "text", "bytes"]) @pytest.mark.asyncio -async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, asynchronous, source): +async def test_native_file_sources_preserve_sdk_behavior( + ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]], + tmp_path: Path, + asynchronous: bool, + source: str, +) -> None: server, requests = ocr_server path: Final = tmp_path / "scan.png" content: Final = b"document bytes" @@ -388,9 +394,12 @@ async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, a "api_base": f"http://127.0.0.1:{server.server_port}", } response: Final = await litellm.aocr(**kwargs) if asynchronous else litellm.ocr(**kwargs) + assert isinstance(response, OCRResponse) assert response.pages[0].markdown == "native OCR response" assert len(requests) == 1 - assert requests[0]["body"]["document"] == { + body: Final = requests[0]["body"] + assert isinstance(body, dict) + assert body["document"] == { "type": "image_url", "image_url": "data:image/png;base64," + base64.b64encode(content).decode(), } @@ -400,7 +409,11 @@ async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, a @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.asyncio -async def test_native_file_failures_preserve_identity_and_do_not_send(ocr_server, tmp_path, asynchronous): +async def test_native_file_failures_preserve_identity_and_do_not_send( + ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]], + tmp_path: Path, + asynchronous: bool, +) -> None: from litellm.rust_bridge import _native server, requests = ocr_server @@ -416,37 +429,47 @@ async def test_native_file_failures_preserve_identity_and_do_not_send(ocr_server reader: Final = Reader() path: Final = tmp_path / "missing.pdf" - kwargs: Final = { - "model": "mistral/mistral-ocr-latest", - "api_key": "test-key", - "api_base": f"http://127.0.0.1:{server.server_port}", - } + + async def invoke(source: Reader | Path) -> None: + if asynchronous: + await _native.aocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": source}, + api_key="test-key", + api_base=f"http://127.0.0.1:{server.server_port}", + ) + else: + _native.ocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": source}, + api_key="test-key", + api_base=f"http://127.0.0.1:{server.server_port}", + ) + for source, error in ((reader, KeyError), (path, FileNotFoundError)): with pytest.raises(error) as caught: - if asynchronous: - await _native.aocr(document={"type": "file", "file": source}, **kwargs) - else: - _native.ocr(document={"type": "file", "file": source}, **kwargs) + await invoke(source) if source is reader: assert caught.value is failure else: - assert caught.value.filename == str(path) + assert isinstance(caught.value, FileNotFoundError) + assert getattr(caught.value, "filename") == str(path) assert reader.calls == 1 assert requests == [] -def test_unstarted_native_file_call_does_not_read_and_releases_reader(): +def test_unstarted_native_file_call_does_not_read_and_releases_reader() -> None: from litellm.rust_bridge import _native class Reader: + pending: Coroutine[object, object, OCRResponse] | None = None + def read(self) -> bytes: raise AssertionError("unstarted call consumed its document") - def create_call(): + def create_call() -> tuple[Coroutine[object, object, OCRResponse], weakref.ReferenceType[Reader]]: reader: Final = Reader() - pending: Final = _native.aocr( - model="mistral/mistral-ocr-latest", document={"type": "file", "file": reader} - ) + pending: Final = _native.aocr(model="mistral/mistral-ocr-latest", document={"type": "file", "file": reader}) reader.pending = pending return pending, weakref.ref(reader) @@ -457,6 +480,49 @@ def test_unstarted_native_file_call_does_not_read_and_releases_reader(): assert reference() is None +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="requires Unix named pipes") +@pytest.mark.asyncio +async def test_native_path_cancellation_waits_for_read_completion( + ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]], tmp_path: Path +) -> None: + from litellm.rust_bridge import _native + + server, requests = ocr_server + path: Final = tmp_path / "document.pdf" + os.mkfifo(path) + entered: Final = threading.Event() + release: Final = threading.Event() + + def supply_document() -> None: + with path.open("wb") as stream: + stream.write(b"document") + stream.flush() + entered.set() + release.wait(5) + + writer: Final = threading.Thread(target=supply_document, daemon=True) + writer.start() + task: Final = asyncio.create_task( + _native.aocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": path}, + api_key="test-key", + api_base=f"http://127.0.0.1:{server.server_port}", + ) + ) + try: + assert await asyncio.to_thread(entered.wait, 3) + task.cancel() + await asyncio.sleep(0.05) + assert not task.done() + finally: + release.set() + await asyncio.to_thread(writer.join, 3) + with pytest.raises(asyncio.CancelledError): + await task + assert requests == [] + + @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("model", ["mistral/mistral-ocr-latest", "azure_ai/doc-intelligence/prebuilt-read"]) @pytest.mark.asyncio