mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(callbacks): move lifecycle dispatch into Rust
This commit is contained in:
parent
d5acf2f9e1
commit
7073f5c0b8
49 changed files with 5793 additions and 1464 deletions
857
litellm-rust/crates/core/src/call_lifecycle/callbacks.rs
Normal file
857
litellm-rust/crates/core/src/call_lifecycle/callbacks.rs
Normal file
|
|
@ -0,0 +1,857 @@
|
|||
use super::host::HostPhase;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct CallbackId(pub u64);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Delivery {
|
||||
Inline,
|
||||
Await,
|
||||
Worker,
|
||||
Background,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum CallbackKind {
|
||||
CustomLogger,
|
||||
Callable { internal: bool },
|
||||
Named { known: bool },
|
||||
Opaque,
|
||||
}
|
||||
|
||||
impl CallbackKind {
|
||||
fn runs_sync_handler_for_async_call(self) -> bool {
|
||||
match self {
|
||||
Self::Callable { internal } => !internal,
|
||||
Self::Named { known } => !known,
|
||||
Self::CustomLogger | Self::Opaque => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum CallbackMethod {
|
||||
LogPreApiCall,
|
||||
LogPostApiCall,
|
||||
PreCallDeploymentHook,
|
||||
PostCallSuccessDeploymentHook,
|
||||
PostCallFailureDeploymentHook,
|
||||
LoggingHook,
|
||||
AsyncLoggingHook,
|
||||
LogSuccessEvent,
|
||||
AsyncLogSuccessEvent,
|
||||
LogFailureEvent,
|
||||
AsyncLogFailureEvent,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct CallbackInvocation {
|
||||
pub target: CallbackId,
|
||||
pub method: CallbackMethod,
|
||||
pub delivery: Delivery,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum TargetErrorPolicy {
|
||||
Contain,
|
||||
Propagate,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum LoggedMarker {
|
||||
SyncSuccess,
|
||||
AsyncSuccess,
|
||||
SyncFailure,
|
||||
AsyncFailure,
|
||||
}
|
||||
|
||||
impl LoggedMarker {
|
||||
pub const fn key(self) -> &'static str {
|
||||
match self {
|
||||
Self::SyncSuccess => "has_logged_sync_success",
|
||||
Self::AsyncSuccess => "has_logged_async_success",
|
||||
Self::SyncFailure => "has_logged_sync_failure",
|
||||
Self::AsyncFailure => "has_logged_async_failure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum CallbackFamily {
|
||||
RequestPreCall,
|
||||
RequestPostCall,
|
||||
DeploymentPreCall,
|
||||
DeploymentPostCall,
|
||||
DeploymentFailure,
|
||||
SyncSuccess,
|
||||
AsyncSuccess,
|
||||
SyncFailure,
|
||||
AsyncFailure,
|
||||
}
|
||||
|
||||
impl CallbackFamily {
|
||||
pub const fn dispatch_method(self) -> CallbackMethod {
|
||||
match self {
|
||||
Self::RequestPreCall => CallbackMethod::LogPreApiCall,
|
||||
Self::RequestPostCall => CallbackMethod::LogPostApiCall,
|
||||
Self::DeploymentPreCall => CallbackMethod::PreCallDeploymentHook,
|
||||
Self::DeploymentPostCall => CallbackMethod::PostCallSuccessDeploymentHook,
|
||||
Self::DeploymentFailure => CallbackMethod::PostCallFailureDeploymentHook,
|
||||
Self::SyncSuccess => CallbackMethod::LogSuccessEvent,
|
||||
Self::AsyncSuccess => CallbackMethod::AsyncLogSuccessEvent,
|
||||
Self::SyncFailure => CallbackMethod::LogFailureEvent,
|
||||
Self::AsyncFailure => CallbackMethod::AsyncLogFailureEvent,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn hook_method(self) -> Option<CallbackMethod> {
|
||||
match self {
|
||||
Self::SyncSuccess => Some(CallbackMethod::LoggingHook),
|
||||
Self::AsyncSuccess => Some(CallbackMethod::AsyncLoggingHook),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn marker(self) -> Option<LoggedMarker> {
|
||||
match self {
|
||||
Self::SyncSuccess => Some(LoggedMarker::SyncSuccess),
|
||||
Self::AsyncSuccess => Some(LoggedMarker::AsyncSuccess),
|
||||
Self::SyncFailure => Some(LoggedMarker::SyncFailure),
|
||||
Self::AsyncFailure => Some(LoggedMarker::AsyncFailure),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn prepares_logging(self) -> bool {
|
||||
self.marker().is_some()
|
||||
}
|
||||
|
||||
pub const fn error_policy(self) -> TargetErrorPolicy {
|
||||
match self {
|
||||
Self::DeploymentPreCall | Self::DeploymentPostCall => TargetErrorPolicy::Propagate,
|
||||
_ => TargetErrorPolicy::Contain,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn targets(self, global: &[CallbackId], dynamic: Option<&[CallbackId]>) -> Vec<CallbackId> {
|
||||
match self {
|
||||
Self::RequestPreCall | Self::RequestPostCall => global
|
||||
.iter()
|
||||
.chain(dynamic.unwrap_or_default())
|
||||
.copied()
|
||||
.collect(),
|
||||
Self::DeploymentPreCall | Self::DeploymentPostCall | Self::DeploymentFailure => {
|
||||
global.to_vec()
|
||||
}
|
||||
Self::SyncSuccess | Self::AsyncSuccess | Self::SyncFailure | Self::AsyncFailure => {
|
||||
let Some(dynamic) = dynamic else {
|
||||
return global.to_vec();
|
||||
};
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
dynamic
|
||||
.iter()
|
||||
.chain(global)
|
||||
.copied()
|
||||
.filter(|id| seen.insert(*id))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ReleaseGate {
|
||||
Immediate,
|
||||
Deferred,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Dispatch {
|
||||
pub family: CallbackFamily,
|
||||
pub delivery: Delivery,
|
||||
pub gate: ReleaseGate,
|
||||
}
|
||||
|
||||
impl Dispatch {
|
||||
pub const fn immediate(family: CallbackFamily, delivery: Delivery) -> Self {
|
||||
Self {
|
||||
family,
|
||||
delivery,
|
||||
gate: ReleaseGate::Immediate,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct SuccessFacts {
|
||||
pub asynchronous: bool,
|
||||
pub internal: bool,
|
||||
pub fallbacks: bool,
|
||||
pub deferred: bool,
|
||||
pub sync_target_kinds: Vec<CallbackKind>,
|
||||
}
|
||||
|
||||
pub fn plan_success(facts: &SuccessFacts) -> Vec<Dispatch> {
|
||||
if !facts.asynchronous {
|
||||
return vec![Dispatch {
|
||||
family: CallbackFamily::SyncSuccess,
|
||||
delivery: Delivery::Worker,
|
||||
gate: ReleaseGate::Immediate,
|
||||
}];
|
||||
}
|
||||
let background = (!facts.internal && !facts.fallbacks).then_some(Dispatch {
|
||||
family: CallbackFamily::AsyncSuccess,
|
||||
delivery: Delivery::Background,
|
||||
gate: if facts.deferred {
|
||||
ReleaseGate::Deferred
|
||||
} else {
|
||||
ReleaseGate::Immediate
|
||||
},
|
||||
});
|
||||
let worker = facts
|
||||
.sync_target_kinds
|
||||
.iter()
|
||||
.any(|kind| kind.runs_sync_handler_for_async_call())
|
||||
.then_some(Dispatch {
|
||||
family: CallbackFamily::SyncSuccess,
|
||||
delivery: Delivery::Worker,
|
||||
gate: ReleaseGate::Immediate,
|
||||
});
|
||||
background.into_iter().chain(worker).collect()
|
||||
}
|
||||
|
||||
pub fn plan_failure(phase: HostPhase, asynchronous: bool, internal: bool) -> Option<Dispatch> {
|
||||
if asynchronous && internal {
|
||||
return None;
|
||||
}
|
||||
match phase {
|
||||
HostPhase::Failure => Some(Dispatch::immediate(
|
||||
CallbackFamily::SyncFailure,
|
||||
Delivery::Inline,
|
||||
)),
|
||||
HostPhase::AsyncFailure => Some(Dispatch::immediate(
|
||||
CallbackFamily::AsyncFailure,
|
||||
Delivery::Await,
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn plan_request(family: CallbackFamily) -> Dispatch {
|
||||
let delivery = match family {
|
||||
CallbackFamily::RequestPreCall | CallbackFamily::RequestPostCall => Delivery::Inline,
|
||||
_ => Delivery::Await,
|
||||
};
|
||||
Dispatch::immediate(family, delivery)
|
||||
}
|
||||
|
||||
pub fn object_target_eligible(
|
||||
sync_request: bool,
|
||||
method: CallbackMethod,
|
||||
kind: CallbackKind,
|
||||
) -> bool {
|
||||
match (method, kind) {
|
||||
(
|
||||
CallbackMethod::LogSuccessEvent | CallbackMethod::LogFailureEvent,
|
||||
CallbackKind::CustomLogger | CallbackKind::Callable { .. },
|
||||
) => sync_request,
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
pub trait DispatchFacts {
|
||||
fn eligible(&mut self, target: CallbackId, method: CallbackMethod) -> bool;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum DispatchStep {
|
||||
PrepareLogging,
|
||||
Invoke(CallbackInvocation),
|
||||
MarkLogged(LoggedMarker),
|
||||
Complete { aborted: bool },
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum InvocationOutcome {
|
||||
Completed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Position {
|
||||
Prepare,
|
||||
Hook(usize),
|
||||
Mark,
|
||||
Dispatch(usize),
|
||||
Complete { aborted: bool },
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct CursorFacts {
|
||||
pub already_logged: bool,
|
||||
pub stream: bool,
|
||||
pub sync_request: bool,
|
||||
}
|
||||
|
||||
pub struct DispatchCursor {
|
||||
dispatch: Dispatch,
|
||||
targets: Vec<CallbackId>,
|
||||
facts: CursorFacts,
|
||||
position: Position,
|
||||
}
|
||||
|
||||
impl DispatchCursor {
|
||||
pub fn start(dispatch: Dispatch, targets: Vec<CallbackId>, facts: CursorFacts) -> Self {
|
||||
let family = dispatch.family;
|
||||
let position = if family.marker().is_some() && facts.already_logged {
|
||||
Position::Complete { aborted: false }
|
||||
} else if family.prepares_logging() {
|
||||
Position::Prepare
|
||||
} else {
|
||||
Position::Dispatch(0)
|
||||
};
|
||||
Self {
|
||||
dispatch,
|
||||
targets,
|
||||
facts,
|
||||
position,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn object_target_eligible(&self, method: CallbackMethod, kind: CallbackKind) -> bool {
|
||||
object_target_eligible(self.facts.sync_request, method, kind)
|
||||
}
|
||||
|
||||
pub const fn family(&self) -> CallbackFamily {
|
||||
self.dispatch.family
|
||||
}
|
||||
|
||||
pub const fn delivery(&self) -> Delivery {
|
||||
self.dispatch.delivery
|
||||
}
|
||||
|
||||
pub fn targets(&self) -> &[CallbackId] {
|
||||
&self.targets
|
||||
}
|
||||
|
||||
pub fn accept(&mut self, outcome: InvocationOutcome) {
|
||||
if outcome == InvocationOutcome::Failed
|
||||
&& self.dispatch.family.error_policy() == TargetErrorPolicy::Propagate
|
||||
{
|
||||
self.position = Position::Complete { aborted: true };
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next(&mut self, facts: &mut dyn DispatchFacts) -> DispatchStep {
|
||||
loop {
|
||||
match self.position {
|
||||
Position::Prepare => {
|
||||
self.position = self.after_prepare();
|
||||
return DispatchStep::PrepareLogging;
|
||||
}
|
||||
Position::Hook(index) => {
|
||||
let Some(method) = self.dispatch.family.hook_method() else {
|
||||
self.position = Position::Mark;
|
||||
continue;
|
||||
};
|
||||
let Some(target) = self.targets.get(index).copied() else {
|
||||
self.position = Position::Mark;
|
||||
continue;
|
||||
};
|
||||
self.position = Position::Hook(index + 1);
|
||||
if facts.eligible(target, method) {
|
||||
return DispatchStep::Invoke(self.invocation(target, method));
|
||||
}
|
||||
}
|
||||
Position::Mark => {
|
||||
self.position = Position::Dispatch(0);
|
||||
match self.dispatch.family.marker() {
|
||||
Some(marker) if !self.facts.stream => {
|
||||
return DispatchStep::MarkLogged(marker);
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
Position::Dispatch(index) => {
|
||||
let Some(target) = self.targets.get(index).copied() else {
|
||||
self.position = Position::Complete { aborted: false };
|
||||
continue;
|
||||
};
|
||||
self.position = Position::Dispatch(index + 1);
|
||||
let method = self.dispatch.family.dispatch_method();
|
||||
if facts.eligible(target, method) {
|
||||
return DispatchStep::Invoke(self.invocation(target, method));
|
||||
}
|
||||
}
|
||||
Position::Complete { aborted } => return DispatchStep::Complete { aborted },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn after_prepare(&self) -> Position {
|
||||
if self.dispatch.family.hook_method().is_some() {
|
||||
Position::Hook(0)
|
||||
} else {
|
||||
Position::Mark
|
||||
}
|
||||
}
|
||||
|
||||
const fn invocation(&self, target: CallbackId, method: CallbackMethod) -> CallbackInvocation {
|
||||
CallbackInvocation {
|
||||
target,
|
||||
method,
|
||||
delivery: self.dispatch.delivery,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
struct AllEligible;
|
||||
|
||||
impl DispatchFacts for AllEligible {
|
||||
fn eligible(&mut self, _: CallbackId, _: CallbackMethod) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
struct Gate<F: FnMut(CallbackId, CallbackMethod) -> bool>(F);
|
||||
|
||||
impl<F: FnMut(CallbackId, CallbackMethod) -> bool> DispatchFacts for Gate<F> {
|
||||
fn eligible(&mut self, target: CallbackId, method: CallbackMethod) -> bool {
|
||||
(self.0)(target, method)
|
||||
}
|
||||
}
|
||||
|
||||
fn drain(cursor: &mut DispatchCursor, facts: &mut dyn DispatchFacts) -> Vec<DispatchStep> {
|
||||
let mut steps = Vec::new();
|
||||
loop {
|
||||
let step = cursor.next(facts);
|
||||
steps.push(step);
|
||||
match step {
|
||||
DispatchStep::Complete { .. } => return steps,
|
||||
DispatchStep::Invoke(_) => cursor.accept(InvocationOutcome::Completed),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn ids(values: &[u64]) -> Vec<CallbackId> {
|
||||
values.iter().copied().map(CallbackId).collect()
|
||||
}
|
||||
|
||||
fn delivery_for(family: CallbackFamily) -> Delivery {
|
||||
match family {
|
||||
CallbackFamily::SyncSuccess => Delivery::Worker,
|
||||
CallbackFamily::AsyncSuccess => Delivery::Background,
|
||||
CallbackFamily::SyncFailure
|
||||
| CallbackFamily::RequestPreCall
|
||||
| CallbackFamily::RequestPostCall => Delivery::Inline,
|
||||
_ => Delivery::Await,
|
||||
}
|
||||
}
|
||||
|
||||
fn start(
|
||||
family: CallbackFamily,
|
||||
targets: Vec<CallbackId>,
|
||||
already_logged: bool,
|
||||
stream: bool,
|
||||
) -> DispatchCursor {
|
||||
DispatchCursor::start(
|
||||
Dispatch::immediate(family, delivery_for(family)),
|
||||
targets,
|
||||
CursorFacts {
|
||||
already_logged,
|
||||
stream,
|
||||
sync_request: true,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_families_order_dynamic_before_global_and_keep_first_duplicate() {
|
||||
let combined = CallbackFamily::SyncSuccess.targets(&ids(&[3, 1, 4]), Some(&ids(&[1, 2])));
|
||||
assert_eq!(combined, ids(&[1, 2, 3, 4]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_families_copy_global_without_dedup_when_dynamic_is_absent() {
|
||||
let combined = CallbackFamily::AsyncFailure.targets(&ids(&[3, 3, 1]), None);
|
||||
assert_eq!(combined, ids(&[3, 3, 1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_families_order_global_before_dynamic_without_dedup() {
|
||||
let combined = CallbackFamily::RequestPreCall.targets(&ids(&[1, 2]), Some(&ids(&[2, 3])));
|
||||
assert_eq!(combined, ids(&[1, 2, 2, 3]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_runs_every_hook_before_any_dispatch_and_marks_between_passes() {
|
||||
let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1, 2]), false, false);
|
||||
let steps = drain(&mut cursor, &mut AllEligible);
|
||||
let invocation = |target, method| {
|
||||
DispatchStep::Invoke(CallbackInvocation {
|
||||
target: CallbackId(target),
|
||||
method,
|
||||
delivery: Delivery::Worker,
|
||||
})
|
||||
};
|
||||
assert_eq!(
|
||||
steps,
|
||||
vec![
|
||||
DispatchStep::PrepareLogging,
|
||||
invocation(1, CallbackMethod::LoggingHook),
|
||||
invocation(2, CallbackMethod::LoggingHook),
|
||||
DispatchStep::MarkLogged(LoggedMarker::SyncSuccess),
|
||||
invocation(1, CallbackMethod::LogSuccessEvent),
|
||||
invocation(2, CallbackMethod::LogSuccessEvent),
|
||||
DispatchStep::Complete { aborted: false },
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn async_success_uses_async_leaf_methods_and_background_delivery() {
|
||||
let mut cursor = start(CallbackFamily::AsyncSuccess, ids(&[7]), false, false);
|
||||
let steps = drain(&mut cursor, &mut AllEligible);
|
||||
let methods: Vec<_> = steps
|
||||
.iter()
|
||||
.filter_map(|step| match step {
|
||||
DispatchStep::Invoke(invocation) => {
|
||||
assert_eq!(invocation.delivery, Delivery::Background);
|
||||
Some(invocation.method)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
methods,
|
||||
[
|
||||
CallbackMethod::AsyncLoggingHook,
|
||||
CallbackMethod::AsyncLogSuccessEvent
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_and_request_families_have_no_hook_pass() {
|
||||
for (family, delivery, method) in [
|
||||
(
|
||||
CallbackFamily::SyncFailure,
|
||||
Delivery::Inline,
|
||||
CallbackMethod::LogFailureEvent,
|
||||
),
|
||||
(
|
||||
CallbackFamily::AsyncFailure,
|
||||
Delivery::Await,
|
||||
CallbackMethod::AsyncLogFailureEvent,
|
||||
),
|
||||
] {
|
||||
let mut cursor = start(family, ids(&[1, 2]), false, false);
|
||||
let steps = drain(&mut cursor, &mut AllEligible);
|
||||
assert_eq!(steps[0], DispatchStep::PrepareLogging);
|
||||
assert!(matches!(steps[1], DispatchStep::MarkLogged(_)));
|
||||
assert_eq!(
|
||||
&steps[2..],
|
||||
&[
|
||||
DispatchStep::Invoke(CallbackInvocation {
|
||||
target: CallbackId(1),
|
||||
method,
|
||||
delivery
|
||||
}),
|
||||
DispatchStep::Invoke(CallbackInvocation {
|
||||
target: CallbackId(2),
|
||||
method,
|
||||
delivery
|
||||
}),
|
||||
DispatchStep::Complete { aborted: false },
|
||||
]
|
||||
);
|
||||
}
|
||||
let mut cursor = start(CallbackFamily::RequestPreCall, ids(&[1]), false, false);
|
||||
let steps = drain(&mut cursor, &mut AllEligible);
|
||||
assert_eq!(
|
||||
steps,
|
||||
vec![
|
||||
DispatchStep::Invoke(CallbackInvocation {
|
||||
target: CallbackId(1),
|
||||
method: CallbackMethod::LogPreApiCall,
|
||||
delivery: Delivery::Inline,
|
||||
}),
|
||||
DispatchStep::Complete { aborted: false },
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_logged_marker_skips_the_whole_terminal_family_but_not_request_families() {
|
||||
let mut cursor = start(CallbackFamily::AsyncSuccess, ids(&[1]), true, false);
|
||||
assert_eq!(
|
||||
cursor.next(&mut AllEligible),
|
||||
DispatchStep::Complete { aborted: false }
|
||||
);
|
||||
let mut cursor = start(CallbackFamily::RequestPostCall, ids(&[1]), true, false);
|
||||
assert!(matches!(
|
||||
cursor.next(&mut AllEligible),
|
||||
DispatchStep::Invoke(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_skips_the_marker_write_but_still_dispatches() {
|
||||
let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1]), false, true);
|
||||
let steps = drain(&mut cursor, &mut AllEligible);
|
||||
assert!(
|
||||
!steps
|
||||
.iter()
|
||||
.any(|step| matches!(step, DispatchStep::MarkLogged(_)))
|
||||
);
|
||||
assert_eq!(
|
||||
steps
|
||||
.iter()
|
||||
.filter(|step| matches!(step, DispatchStep::Invoke(_)))
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ineligible_targets_are_skipped_per_method_without_affecting_others() {
|
||||
let mut cursor = start(CallbackFamily::SyncSuccess, ids(&[1, 2]), false, false);
|
||||
let mut facts = Gate(|target, method| {
|
||||
!(target == CallbackId(1) && method == CallbackMethod::LoggingHook)
|
||||
&& !(target == CallbackId(2) && method == CallbackMethod::LogSuccessEvent)
|
||||
});
|
||||
let invoked: Vec<_> = drain(&mut cursor, &mut facts)
|
||||
.into_iter()
|
||||
.filter_map(|step| match step {
|
||||
DispatchStep::Invoke(invocation) => Some((invocation.target.0, invocation.method)),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
invoked,
|
||||
[
|
||||
(2, CallbackMethod::LoggingHook),
|
||||
(1, CallbackMethod::LogSuccessEvent)
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn contained_failures_continue_and_propagating_failures_abort() {
|
||||
let mut cursor = start(CallbackFamily::SyncFailure, ids(&[1, 2]), false, false);
|
||||
assert_eq!(cursor.next(&mut AllEligible), DispatchStep::PrepareLogging);
|
||||
assert!(matches!(
|
||||
cursor.next(&mut AllEligible),
|
||||
DispatchStep::MarkLogged(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
cursor.next(&mut AllEligible),
|
||||
DispatchStep::Invoke(_)
|
||||
));
|
||||
cursor.accept(InvocationOutcome::Failed);
|
||||
assert!(matches!(
|
||||
cursor.next(&mut AllEligible),
|
||||
DispatchStep::Invoke(CallbackInvocation {
|
||||
target: CallbackId(2),
|
||||
..
|
||||
})
|
||||
));
|
||||
|
||||
let mut cursor = start(
|
||||
CallbackFamily::DeploymentPreCall,
|
||||
ids(&[1, 2]),
|
||||
false,
|
||||
false,
|
||||
);
|
||||
assert!(matches!(
|
||||
cursor.next(&mut AllEligible),
|
||||
DispatchStep::Invoke(_)
|
||||
));
|
||||
cursor.accept(InvocationOutcome::Failed);
|
||||
assert_eq!(
|
||||
cursor.next(&mut AllEligible),
|
||||
DispatchStep::Complete { aborted: true }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_sdk_success_selects_one_worker_dispatch() {
|
||||
let plan = plan_success(&SuccessFacts {
|
||||
asynchronous: false,
|
||||
internal: true,
|
||||
fallbacks: true,
|
||||
deferred: true,
|
||||
sync_target_kinds: vec![],
|
||||
});
|
||||
assert_eq!(
|
||||
plan,
|
||||
[Dispatch {
|
||||
family: CallbackFamily::SyncSuccess,
|
||||
delivery: Delivery::Worker,
|
||||
gate: ReleaseGate::Immediate,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn async_sdk_success_enqueues_background_then_worker_only_for_external_sync_targets() {
|
||||
let base = SuccessFacts {
|
||||
asynchronous: true,
|
||||
internal: false,
|
||||
fallbacks: false,
|
||||
deferred: false,
|
||||
sync_target_kinds: vec![
|
||||
CallbackKind::CustomLogger,
|
||||
CallbackKind::Named { known: true },
|
||||
],
|
||||
};
|
||||
assert_eq!(
|
||||
plan_success(&base),
|
||||
[Dispatch {
|
||||
family: CallbackFamily::AsyncSuccess,
|
||||
delivery: Delivery::Background,
|
||||
gate: ReleaseGate::Immediate,
|
||||
}]
|
||||
);
|
||||
let with_external = SuccessFacts {
|
||||
sync_target_kinds: vec![
|
||||
CallbackKind::CustomLogger,
|
||||
CallbackKind::Callable { internal: false },
|
||||
],
|
||||
deferred: true,
|
||||
..base.clone()
|
||||
};
|
||||
assert_eq!(
|
||||
plan_success(&with_external),
|
||||
[
|
||||
Dispatch {
|
||||
family: CallbackFamily::AsyncSuccess,
|
||||
delivery: Delivery::Background,
|
||||
gate: ReleaseGate::Deferred,
|
||||
},
|
||||
Dispatch {
|
||||
family: CallbackFamily::SyncSuccess,
|
||||
delivery: Delivery::Worker,
|
||||
gate: ReleaseGate::Immediate,
|
||||
},
|
||||
]
|
||||
);
|
||||
let internal_or_fallback = SuccessFacts {
|
||||
internal: true,
|
||||
sync_target_kinds: vec![CallbackKind::Named { known: false }],
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
plan_success(&internal_or_fallback),
|
||||
[Dispatch {
|
||||
family: CallbackFamily::SyncSuccess,
|
||||
delivery: Delivery::Worker,
|
||||
gate: ReleaseGate::Immediate,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_and_internal_targets_never_trigger_the_worker_for_async_calls() {
|
||||
let plan = plan_success(&SuccessFacts {
|
||||
asynchronous: true,
|
||||
internal: false,
|
||||
fallbacks: true,
|
||||
deferred: false,
|
||||
sync_target_kinds: vec![
|
||||
CallbackKind::Opaque,
|
||||
CallbackKind::Callable { internal: true },
|
||||
],
|
||||
});
|
||||
assert!(plan.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_families_follow_the_phase_and_skip_internal_async_calls() {
|
||||
assert_eq!(
|
||||
plan_failure(HostPhase::Failure, false, true),
|
||||
Some(Dispatch::immediate(
|
||||
CallbackFamily::SyncFailure,
|
||||
Delivery::Inline
|
||||
))
|
||||
);
|
||||
assert_eq!(
|
||||
plan_failure(HostPhase::AsyncFailure, true, false),
|
||||
Some(Dispatch::immediate(
|
||||
CallbackFamily::AsyncFailure,
|
||||
Delivery::Await
|
||||
))
|
||||
);
|
||||
assert_eq!(plan_failure(HostPhase::Failure, true, true), None);
|
||||
assert_eq!(plan_failure(HostPhase::Success, false, false), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delivery_is_selected_by_the_plan_and_carried_on_every_invocation() {
|
||||
let mut cursor = DispatchCursor::start(
|
||||
Dispatch::immediate(CallbackFamily::SyncFailure, Delivery::Worker),
|
||||
ids(&[1]),
|
||||
CursorFacts {
|
||||
already_logged: false,
|
||||
stream: false,
|
||||
sync_request: true,
|
||||
},
|
||||
);
|
||||
let deliveries: Vec<_> = drain(&mut cursor, &mut AllEligible)
|
||||
.into_iter()
|
||||
.filter_map(|step| match step {
|
||||
DispatchStep::Invoke(invocation) => Some(invocation.delivery),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(deliveries, [Delivery::Worker]);
|
||||
assert_eq!(
|
||||
plan_request(CallbackFamily::RequestPreCall).delivery,
|
||||
Delivery::Inline
|
||||
);
|
||||
assert_eq!(
|
||||
plan_request(CallbackFamily::DeploymentPreCall).delivery,
|
||||
Delivery::Await
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_leaf_methods_skip_object_targets_on_async_requests() {
|
||||
let asynchronous = DispatchCursor::start(
|
||||
Dispatch::immediate(CallbackFamily::SyncSuccess, Delivery::Worker),
|
||||
ids(&[1]),
|
||||
CursorFacts {
|
||||
already_logged: false,
|
||||
stream: false,
|
||||
sync_request: false,
|
||||
},
|
||||
);
|
||||
for kind in [
|
||||
CallbackKind::CustomLogger,
|
||||
CallbackKind::Callable { internal: false },
|
||||
] {
|
||||
assert!(!asynchronous.object_target_eligible(CallbackMethod::LogSuccessEvent, kind));
|
||||
assert!(!asynchronous.object_target_eligible(CallbackMethod::LogFailureEvent, kind));
|
||||
assert!(asynchronous.object_target_eligible(CallbackMethod::LoggingHook, kind));
|
||||
assert!(
|
||||
asynchronous.object_target_eligible(CallbackMethod::AsyncLogSuccessEvent, kind)
|
||||
);
|
||||
}
|
||||
assert!(asynchronous.object_target_eligible(
|
||||
CallbackMethod::LogSuccessEvent,
|
||||
CallbackKind::Named { known: false }
|
||||
));
|
||||
let synchronous = start(CallbackFamily::SyncSuccess, ids(&[1]), false, false);
|
||||
assert!(
|
||||
synchronous.object_target_eligible(
|
||||
CallbackMethod::LogSuccessEvent,
|
||||
CallbackKind::CustomLogger
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -223,9 +223,11 @@ mod tests {
|
|||
] {
|
||||
assert_eq!(lifecycle.phase(), phase);
|
||||
assert!(
|
||||
lifecycle.accept(Err(HostFailure::Error(crate::ocr::Error::InvalidRequest(
|
||||
"callback".into()
|
||||
)))).is_none()
|
||||
lifecycle
|
||||
.accept(Err(HostFailure::Error(crate::ocr::Error::InvalidRequest(
|
||||
"callback".into()
|
||||
))))
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert_eq!(lifecycle.phase(), HostPhase::Complete);
|
||||
|
|
|
|||
|
|
@ -1,9 +1,17 @@
|
|||
use std::future::Future;
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
pub mod callbacks;
|
||||
pub mod host;
|
||||
pub mod registration;
|
||||
pub mod types;
|
||||
|
||||
pub use callbacks::{
|
||||
CallbackFamily, CallbackId, CallbackInvocation, CallbackKind, CallbackMethod, CursorFacts,
|
||||
Delivery, Dispatch, DispatchCursor, DispatchFacts, DispatchStep, InvocationOutcome,
|
||||
LoggedMarker, ReleaseGate, SuccessFacts, TargetErrorPolicy, object_target_eligible,
|
||||
plan_failure, plan_request, plan_success,
|
||||
};
|
||||
pub use types::{
|
||||
CallLifecycleContext, CallLifecyclePhase, CallLifecyclePhaseTiming, CallLifecycleRequest,
|
||||
CallLifecycleTiming,
|
||||
|
|
|
|||
374
litellm-rust/crates/core/src/call_lifecycle/registration.rs
Normal file
374
litellm-rust/crates/core/src/call_lifecycle/registration.rs
Normal file
|
|
@ -0,0 +1,374 @@
|
|||
use std::collections::HashSet;
|
||||
|
||||
use super::callbacks::CallbackId;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum Registry {
|
||||
Input,
|
||||
AsyncInput,
|
||||
Success,
|
||||
AsyncSuccess,
|
||||
Failure,
|
||||
AsyncFailure,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Registration {
|
||||
Named { known: bool, async_only: bool },
|
||||
Object { asynchronous: bool },
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Entry {
|
||||
pub id: CallbackId,
|
||||
pub registration: Registration,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Candidate {
|
||||
pub resolved: Option<Entry>,
|
||||
pub duplicate_type: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Default)]
|
||||
pub struct RegistrationFacts {
|
||||
pub candidates: Vec<Candidate>,
|
||||
pub input: Vec<Entry>,
|
||||
pub success: Vec<Entry>,
|
||||
pub failure: Vec<Entry>,
|
||||
pub async_success: Vec<CallbackId>,
|
||||
pub async_failure: Vec<CallbackId>,
|
||||
pub bootstrap_pending: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum NamedEvent {
|
||||
Success,
|
||||
Failure,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RegistryMutation {
|
||||
Append(Registry, CallbackId),
|
||||
Remove(Registry, CallbackId),
|
||||
ExpandNamed(NamedEvent, CallbackId),
|
||||
Bootstrap,
|
||||
}
|
||||
|
||||
struct Planner {
|
||||
input: Vec<Entry>,
|
||||
success: Vec<Entry>,
|
||||
failure: Vec<Entry>,
|
||||
async_success: HashSet<CallbackId>,
|
||||
async_failure: HashSet<CallbackId>,
|
||||
mutations: Vec<RegistryMutation>,
|
||||
}
|
||||
|
||||
impl Planner {
|
||||
fn contains(&self, registry: Registry, id: CallbackId) -> bool {
|
||||
match registry {
|
||||
Registry::Input => self.input.iter().any(|entry| entry.id == id),
|
||||
Registry::Success => self.success.iter().any(|entry| entry.id == id),
|
||||
Registry::Failure => self.failure.iter().any(|entry| entry.id == id),
|
||||
Registry::AsyncSuccess => self.async_success.contains(&id),
|
||||
Registry::AsyncFailure => self.async_failure.contains(&id),
|
||||
Registry::AsyncInput => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn append(&mut self, registry: Registry, entry: Entry) {
|
||||
if self.contains(registry, entry.id) {
|
||||
return;
|
||||
}
|
||||
match registry {
|
||||
Registry::Input => self.input.push(entry),
|
||||
Registry::Success => self.success.push(entry),
|
||||
Registry::Failure => self.failure.push(entry),
|
||||
Registry::AsyncSuccess => {
|
||||
self.async_success.insert(entry.id);
|
||||
}
|
||||
Registry::AsyncFailure => {
|
||||
self.async_failure.insert(entry.id);
|
||||
}
|
||||
Registry::AsyncInput => {}
|
||||
}
|
||||
self.mutations
|
||||
.push(RegistryMutation::Append(registry, entry.id));
|
||||
}
|
||||
|
||||
fn record(&mut self, mutation: RegistryMutation) {
|
||||
self.mutations.push(mutation);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_asynchronous(registration: Registration) -> bool {
|
||||
matches!(registration, Registration::Object { asynchronous: true })
|
||||
}
|
||||
|
||||
pub fn plan_registration(facts: &RegistrationFacts) -> Vec<RegistryMutation> {
|
||||
let mut planner = Planner {
|
||||
input: facts.input.clone(),
|
||||
success: facts.success.clone(),
|
||||
failure: facts.failure.clone(),
|
||||
async_success: facts.async_success.iter().copied().collect(),
|
||||
async_failure: facts.async_failure.iter().copied().collect(),
|
||||
mutations: Vec::new(),
|
||||
};
|
||||
|
||||
for candidate in &facts.candidates {
|
||||
let Some(entry) = candidate.resolved else {
|
||||
continue;
|
||||
};
|
||||
if candidate.duplicate_type {
|
||||
continue;
|
||||
}
|
||||
planner.append(Registry::Input, entry);
|
||||
if !is_asynchronous(entry.registration) {
|
||||
planner.append(Registry::Success, entry);
|
||||
planner.append(Registry::Failure, entry);
|
||||
}
|
||||
planner.append(Registry::AsyncSuccess, entry);
|
||||
planner.append(Registry::AsyncFailure, entry);
|
||||
}
|
||||
|
||||
if facts.bootstrap_pending
|
||||
&& !(planner.input.is_empty() && planner.success.is_empty() && planner.failure.is_empty())
|
||||
{
|
||||
planner.record(RegistryMutation::Bootstrap);
|
||||
}
|
||||
|
||||
let input = planner.input.clone();
|
||||
for entry in input
|
||||
.iter()
|
||||
.filter(|entry| is_asynchronous(entry.registration))
|
||||
{
|
||||
planner.record(RegistryMutation::Append(Registry::AsyncInput, entry.id));
|
||||
planner.record(RegistryMutation::Remove(Registry::Input, entry.id));
|
||||
}
|
||||
|
||||
let success = planner.success.clone();
|
||||
for entry in &success {
|
||||
match entry.registration {
|
||||
Registration::Object { asynchronous: true }
|
||||
| Registration::Named {
|
||||
async_only: true, ..
|
||||
} => {
|
||||
planner.append(Registry::AsyncSuccess, *entry);
|
||||
planner.record(RegistryMutation::Remove(Registry::Success, entry.id));
|
||||
}
|
||||
Registration::Named { known: true, .. } => {
|
||||
planner.record(RegistryMutation::ExpandNamed(NamedEvent::Success, entry.id));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let failure = planner.failure.clone();
|
||||
for entry in &failure {
|
||||
match entry.registration {
|
||||
Registration::Object { asynchronous: true } => {
|
||||
planner.append(Registry::AsyncFailure, *entry);
|
||||
planner.record(RegistryMutation::Remove(Registry::Failure, entry.id));
|
||||
}
|
||||
Registration::Named { known: true, .. } => {
|
||||
planner.record(RegistryMutation::ExpandNamed(NamedEvent::Failure, entry.id));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
planner.mutations
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum DynamicSuccessSlot {
|
||||
Sync,
|
||||
Async,
|
||||
}
|
||||
|
||||
pub fn classify_dynamic_success(entry: Entry, named_async: bool) -> DynamicSuccessSlot {
|
||||
match entry.registration {
|
||||
Registration::Object { asynchronous: true } => DynamicSuccessSlot::Async,
|
||||
Registration::Named { .. } if named_async => DynamicSuccessSlot::Async,
|
||||
_ => DynamicSuccessSlot::Sync,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn object(id: u64, asynchronous: bool) -> Entry {
|
||||
Entry {
|
||||
id: CallbackId(id),
|
||||
registration: Registration::Object { asynchronous },
|
||||
}
|
||||
}
|
||||
|
||||
fn named(id: u64, known: bool, async_only: bool) -> Entry {
|
||||
Entry {
|
||||
id: CallbackId(id),
|
||||
registration: Registration::Named { known, async_only },
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate(entry: Entry) -> Candidate {
|
||||
Candidate {
|
||||
resolved: Some(entry),
|
||||
duplicate_type: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_callback_in_callbacks_registers_in_every_list_once() {
|
||||
let facts = RegistrationFacts {
|
||||
candidates: vec![candidate(object(1, false)), candidate(object(1, false))],
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert_eq!(
|
||||
plan_registration(&facts),
|
||||
[
|
||||
RegistryMutation::Append(Registry::Input, CallbackId(1)),
|
||||
RegistryMutation::Append(Registry::Success, CallbackId(1)),
|
||||
RegistryMutation::Append(Registry::Failure, CallbackId(1)),
|
||||
RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(1)),
|
||||
RegistryMutation::Append(Registry::AsyncFailure, CallbackId(1)),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn async_callable_in_callbacks_skips_sync_lists_and_moves_out_of_input() {
|
||||
let facts = RegistrationFacts {
|
||||
candidates: vec![candidate(object(2, true))],
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert_eq!(
|
||||
plan_registration(&facts),
|
||||
[
|
||||
RegistryMutation::Append(Registry::Input, CallbackId(2)),
|
||||
RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(2)),
|
||||
RegistryMutation::Append(Registry::AsyncFailure, CallbackId(2)),
|
||||
RegistryMutation::Append(Registry::AsyncInput, CallbackId(2)),
|
||||
RegistryMutation::Remove(Registry::Input, CallbackId(2)),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unresolved_and_duplicate_type_named_candidates_are_skipped() {
|
||||
let facts = RegistrationFacts {
|
||||
candidates: vec![
|
||||
Candidate {
|
||||
resolved: None,
|
||||
duplicate_type: false,
|
||||
},
|
||||
Candidate {
|
||||
resolved: Some(object(3, false)),
|
||||
duplicate_type: true,
|
||||
},
|
||||
],
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert!(plan_registration(&facts).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_registered_callbacks_are_not_appended_again() {
|
||||
let facts = RegistrationFacts {
|
||||
candidates: vec![candidate(object(1, false))],
|
||||
input: vec![object(1, false)],
|
||||
success: vec![object(1, false)],
|
||||
failure: vec![object(1, false)],
|
||||
async_success: vec![CallbackId(1)],
|
||||
async_failure: vec![CallbackId(1)],
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert!(plan_registration(&facts).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_runs_once_when_any_public_list_is_populated() {
|
||||
let empty = RegistrationFacts {
|
||||
bootstrap_pending: true,
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert!(plan_registration(&empty).is_empty());
|
||||
let populated = RegistrationFacts {
|
||||
bootstrap_pending: true,
|
||||
candidates: vec![candidate(object(1, false))],
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert!(plan_registration(&populated).contains(&RegistryMutation::Bootstrap));
|
||||
let already = RegistrationFacts {
|
||||
bootstrap_pending: false,
|
||||
success: vec![object(1, false)],
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert!(!plan_registration(&already).contains(&RegistryMutation::Bootstrap));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_safety_net_moves_async_and_async_only_names_and_expands_known_names() {
|
||||
let facts = RegistrationFacts {
|
||||
success: vec![
|
||||
object(1, true),
|
||||
named(2, false, true),
|
||||
named(3, true, false),
|
||||
named(4, false, false),
|
||||
object(5, false),
|
||||
],
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert_eq!(
|
||||
plan_registration(&facts),
|
||||
[
|
||||
RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(1)),
|
||||
RegistryMutation::Remove(Registry::Success, CallbackId(1)),
|
||||
RegistryMutation::Append(Registry::AsyncSuccess, CallbackId(2)),
|
||||
RegistryMutation::Remove(Registry::Success, CallbackId(2)),
|
||||
RegistryMutation::ExpandNamed(NamedEvent::Success, CallbackId(3)),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_safety_net_ignores_async_only_names() {
|
||||
let facts = RegistrationFacts {
|
||||
failure: vec![
|
||||
object(1, true),
|
||||
named(2, false, true),
|
||||
named(3, true, false),
|
||||
],
|
||||
async_failure: vec![CallbackId(1)],
|
||||
..RegistrationFacts::default()
|
||||
};
|
||||
assert_eq!(
|
||||
plan_registration(&facts),
|
||||
[
|
||||
RegistryMutation::Remove(Registry::Failure, CallbackId(1)),
|
||||
RegistryMutation::ExpandNamed(NamedEvent::Failure, CallbackId(3)),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_success_split_follows_async_callables_and_selected_names() {
|
||||
assert_eq!(
|
||||
classify_dynamic_success(object(1, true), false),
|
||||
DynamicSuccessSlot::Async
|
||||
);
|
||||
assert_eq!(
|
||||
classify_dynamic_success(object(1, false), false),
|
||||
DynamicSuccessSlot::Sync
|
||||
);
|
||||
assert_eq!(
|
||||
classify_dynamic_success(named(2, true, false), true),
|
||||
DynamicSuccessSlot::Async
|
||||
);
|
||||
assert_eq!(
|
||||
classify_dynamic_success(named(2, true, true), false),
|
||||
DynamicSuccessSlot::Sync
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -816,7 +816,8 @@ mod tests {
|
|||
"type":"document_url",
|
||||
"document_url":"https://example.com/document.pdf"
|
||||
}))
|
||||
.unwrap().into();
|
||||
.unwrap()
|
||||
.into();
|
||||
|
||||
perform_ocr(request).await.unwrap();
|
||||
server.await.unwrap();
|
||||
|
|
|
|||
|
|
@ -361,10 +361,12 @@ mod tests {
|
|||
}
|
||||
}),
|
||||
);
|
||||
let request = request.with_document(serde_json::from_value(json!({
|
||||
"type":"image_url","image_url":"https://example.com/original.png"
|
||||
}))
|
||||
.unwrap());
|
||||
let request = request.with_document(
|
||||
serde_json::from_value(json!({
|
||||
"type":"image_url","image_url":"https://example.com/original.png"
|
||||
}))
|
||||
.unwrap(),
|
||||
);
|
||||
let request = crate::ocr::prepare::prepare_request(request);
|
||||
let http = CohereParseConfig
|
||||
.prepare_request(&request, &crate::ocr::test_support::ocr_client())
|
||||
|
|
@ -503,10 +505,12 @@ mod tests {
|
|||
"https://example.com",
|
||||
json!({"output_format":null,"req_format":null}),
|
||||
);
|
||||
let request = request.with_document(serde_json::from_value(
|
||||
let request = request.with_document(
|
||||
serde_json::from_value(
|
||||
json!({"type":"image_url","image_url":"https://example.com/a.png"}),
|
||||
)
|
||||
.unwrap());
|
||||
.unwrap(),
|
||||
);
|
||||
assert_eq!(
|
||||
request.response_format().unwrap(),
|
||||
crate::ocr::types::OcrResponseFormat::Litellm
|
||||
|
|
|
|||
|
|
@ -738,7 +738,8 @@ mod tests {
|
|||
"result":{"chunks":[]}
|
||||
}))])
|
||||
.await;
|
||||
let request = crate::ocr::test_support::with_source(wire_request(model, &base, options), source);
|
||||
let request =
|
||||
crate::ocr::test_support::with_source(wire_request(model, &base, options), source);
|
||||
|
||||
perform_ocr(request).await.unwrap();
|
||||
server.await.unwrap();
|
||||
|
|
@ -857,7 +858,9 @@ mod tests {
|
|||
#[tokio::test]
|
||||
async fn rejects_invalid_document_sources_before_network(#[case] source: &str) {
|
||||
let request = crate::ocr::test_support::with_source(
|
||||
wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})), source);
|
||||
wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})),
|
||||
source,
|
||||
);
|
||||
assert!(perform_ocr(request).await.is_err());
|
||||
}
|
||||
|
||||
|
|
@ -913,7 +916,9 @@ mod tests {
|
|||
let raw = json!({"job_id":"job-1","result":{"chunks":[]}});
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(raw)]).await;
|
||||
let mut request = crate::ocr::test_support::with_source(
|
||||
wire_request("reducto/parse-v3", &base, json!({})), "reducto://ready.pdf");
|
||||
wire_request("reducto/parse-v3", &base, json!({})),
|
||||
"reducto://ready.pdf",
|
||||
);
|
||||
request.transport.extra_headers = vec![("authorization".into(), "Bearer existing".into())];
|
||||
|
||||
let response = perform_ocr(request).await.unwrap();
|
||||
|
|
@ -984,6 +989,7 @@ mod tests {
|
|||
request: OcrDuringCallRequest,
|
||||
) -> OcrHookFuture<'_, OcrDuringCallRequest> {
|
||||
Box::pin(async move {
|
||||
assert_eq!(request.optional_params["use_cache"], json!(true));
|
||||
assert_eq!(
|
||||
request.body["document_url"],
|
||||
"data:application/pdf;base64,YWJj"
|
||||
|
|
@ -1000,7 +1006,7 @@ mod tests {
|
|||
async fn guardrail_rewrites_document_before_upload() {
|
||||
let (base, seen, server) =
|
||||
mock_server(vec![MockResponse::json(json!({"result":{"chunks":[]}}))]).await;
|
||||
let mut request = wire_request("reducto/parse-v3", &base, json!({}));
|
||||
let mut request = wire_request("reducto/parse-v3", &base, json!({"use_cache":true}));
|
||||
request.hooks = Arc::new(RewriteDocument);
|
||||
|
||||
perform_ocr(request).await.unwrap();
|
||||
|
|
|
|||
|
|
@ -338,8 +338,12 @@ mod tests {
|
|||
options.clone(),
|
||||
);
|
||||
let vertex = wire_request("vertex_ai/mistral-ocr-maas", "https://vertex.test", options);
|
||||
let direct = crate::ocr::prepare::prepare_request(crate::ocr::test_support::resolved_request(direct));
|
||||
let vertex = crate::ocr::prepare::prepare_request(crate::ocr::test_support::resolved_request(vertex));
|
||||
let direct = crate::ocr::prepare::prepare_request(
|
||||
crate::ocr::test_support::resolved_request(direct),
|
||||
);
|
||||
let vertex = crate::ocr::prepare::prepare_request(
|
||||
crate::ocr::test_support::resolved_request(vertex),
|
||||
);
|
||||
let direct_http = MistralOCRConfig
|
||||
.prepare_request(&direct, &client)
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -39,9 +39,10 @@ impl OcrClient {
|
|||
) -> Result<LiteLLMOcrResponse, crate::ocr::Error> {
|
||||
use super::{
|
||||
NativeOutcome, OcrAdmission, OcrCall, OcrCallStep, OcrHookHost, OcrHost,
|
||||
OcrHostOperation, OcrHostResult,
|
||||
OcrHostOperation, OcrHostResult, OcrProjectedRequest,
|
||||
};
|
||||
|
||||
let intercepts_requests = request.hooks.intercepts_requests();
|
||||
let host = OcrHookHost::new(request.hooks.clone());
|
||||
let mut request = Some(request);
|
||||
let NativeOutcome::Completed(mut call) = OcrCall::admit(self.clone(), OcrAdmission::all())
|
||||
|
|
@ -54,14 +55,15 @@ impl OcrClient {
|
|||
loop {
|
||||
match call.resume(result.take()).await? {
|
||||
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => {
|
||||
result = Some(OcrHostResult::Request(Ok((
|
||||
Box::new(request.take().ok_or_else(|| {
|
||||
result = Some(OcrHostResult::Request(Ok(OcrProjectedRequest {
|
||||
request: Box::new(request.take().ok_or_else(|| {
|
||||
crate::ocr::Error::InvalidRequest(
|
||||
"OCR request was already projected".into(),
|
||||
)
|
||||
})?),
|
||||
false,
|
||||
))))
|
||||
intercepts_requests,
|
||||
host_token_provider: false,
|
||||
})))
|
||||
}
|
||||
OcrCallStep::Host(operation) => result = Some(host.invoke(operation).await),
|
||||
OcrCallStep::Complete(response) => return Ok(response),
|
||||
|
|
|
|||
|
|
@ -15,7 +15,10 @@ pub(crate) fn read_path_document(
|
|||
) -> Result<OcrDocument, super::Error> {
|
||||
let mut bytes = Vec::new();
|
||||
std::fs::File::open(path)
|
||||
.and_then(|file| file.take(OCR_INLINE_MAX_BYTES as u64 + 1).read_to_end(&mut bytes))
|
||||
.and_then(|file| {
|
||||
file.take(OCR_INLINE_MAX_BYTES as u64 + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
})
|
||||
.map_err(|source| super::Error::FileRead {
|
||||
path: path.to_owned(),
|
||||
source: std::sync::Arc::new(source),
|
||||
|
|
@ -201,9 +204,14 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn path_preparation_preserves_io_causes_and_enforces_the_inline_limit() {
|
||||
let path = std::env::temp_dir().join(format!("ocr-document-{:032x}.png", rand::random::<u128>()));
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("ocr-document-{:032x}.png", rand::random::<u128>()));
|
||||
let error = read_path_document(&path, None).unwrap_err();
|
||||
let super::super::Error::FileRead { path: failed_path, source } = error else {
|
||||
let super::super::Error::FileRead {
|
||||
path: failed_path,
|
||||
source,
|
||||
} = error
|
||||
else {
|
||||
panic!("missing path must produce a typed file error");
|
||||
};
|
||||
assert_eq!(failed_path, path);
|
||||
|
|
@ -215,9 +223,16 @@ mod tests {
|
|||
let image = read_path_document(&path, None).unwrap();
|
||||
let overridden = read_path_document(&path, Some("application/pdf")).unwrap();
|
||||
std::fs::remove_file(&path).unwrap();
|
||||
assert!(matches!(oversized, Err(super::super::Error::InlineDocumentTooLarge)));
|
||||
assert!(matches!(image, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2UgYnl0ZXM="));
|
||||
assert!(matches!(overridden, OcrDocument::DocumentUrl { document_url, .. } if document_url == "data:application/pdf;base64,aW1hZ2UgYnl0ZXM="));
|
||||
assert!(matches!(
|
||||
oversized,
|
||||
Err(super::super::Error::InlineDocumentTooLarge)
|
||||
));
|
||||
assert!(
|
||||
matches!(image, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2UgYnl0ZXM=")
|
||||
);
|
||||
assert!(
|
||||
matches!(overridden, OcrDocument::DocumentUrl { document_url, .. } if document_url == "data:application/pdf;base64,aW1hZ2UgYnl0ZXM=")
|
||||
);
|
||||
}
|
||||
|
||||
fn document(source: &str) -> OcrDocument {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ pub enum Error {
|
|||
},
|
||||
#[error("File is empty or could not be read")]
|
||||
EmptyFile,
|
||||
#[error("Host OCR document read failed")]
|
||||
HostDocumentRead,
|
||||
#[error("Failed to read OCR file {}: {source}", path.display())]
|
||||
FileRead {
|
||||
path: std::path::PathBuf,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use std::sync::Arc;
|
|||
|
||||
use super::OcrClient;
|
||||
use super::hooks::{OcrHooks, OcrLifecycleHooks, OcrPostCallRequest};
|
||||
use super::types::{ResolvedOcrRequest, LiteLLMOcrResponse, PreparedOcrRequest};
|
||||
use super::types::{LiteLLMOcrResponse, PreparedOcrRequest, ResolvedOcrRequest};
|
||||
use crate::call_lifecycle::{CallLifecycle, CallLifecycleContext};
|
||||
use crate::llms::base_llm::ocr::transformation::OcrResponseContext;
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ pub struct OcrPreCallRequest {
|
|||
pub struct OcrDuringCallRequest {
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub optional_params: Value,
|
||||
pub api_key: Option<String>,
|
||||
pub url: String,
|
||||
pub headers: Vec<(String, String)>,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
|
|
@ -9,8 +10,8 @@ use super::hooks::{
|
|||
OcrDuringCallRequest, OcrHookFuture, OcrHooks, OcrLogFuture, OcrPostCallRequest,
|
||||
OcrPreCallRequest,
|
||||
};
|
||||
use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient, OcrDocumentInput, OcrFileContent};
|
||||
use super::types::ResolvedOcrRequest;
|
||||
use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient, OcrDocumentInput, OcrFileContent};
|
||||
use crate::call_lifecycle::host::{
|
||||
HostCall, HostCallFuture, HostCallStep, HostFailure, HostLifecycle, HostPhase,
|
||||
};
|
||||
|
|
@ -82,8 +83,14 @@ impl OcrHostOperation {
|
|||
}
|
||||
}
|
||||
|
||||
pub struct OcrProjectedRequest {
|
||||
pub request: Box<LiteLLMOcrRequest>,
|
||||
pub intercepts_requests: bool,
|
||||
pub host_token_provider: bool,
|
||||
}
|
||||
|
||||
pub enum OcrHostResult {
|
||||
Request(Result<(Box<LiteLLMOcrRequest>, bool), super::Error>),
|
||||
Request(Result<OcrProjectedRequest, super::Error>),
|
||||
Document(Result<OcrFileContent, super::Error>),
|
||||
Lifecycle(Result<(), HostFailure<super::Error>>),
|
||||
AzureAdToken(Result<ResolvedCredential, litellm_auth::Error>),
|
||||
|
|
@ -160,9 +167,10 @@ impl OcrCall {
|
|||
Some(OcrHostResult::Request(result)) if self.projecting => {
|
||||
self.projecting = false;
|
||||
match result {
|
||||
Ok((request, azure_ad_token_provider)) => {
|
||||
self.execution.request = Some(*request);
|
||||
self.execution.azure_ad_token_provider = azure_ad_token_provider;
|
||||
Ok(projected) => {
|
||||
self.execution.set_request(*projected.request);
|
||||
self.execution.intercepts_requests = projected.intercepts_requests;
|
||||
self.execution.host_token_provider = projected.host_token_provider;
|
||||
}
|
||||
Err(error) => self.accept(Err(HostFailure::Error(error))),
|
||||
}
|
||||
|
|
@ -268,6 +276,15 @@ impl OcrCall {
|
|||
fn accept(&mut self, result: Result<(), HostFailure<super::Error>>) {
|
||||
let cancelled = matches!(&result, Err(HostFailure::Cancelled(_)));
|
||||
if let Some(error) = self.lifecycle.accept(result) {
|
||||
if let Some((_, timing)) = self
|
||||
.execution
|
||||
.terminal
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.as_mut()
|
||||
{
|
||||
timing.end_time = epoch_seconds();
|
||||
}
|
||||
if cancelled {
|
||||
self.error = Some(error);
|
||||
} else {
|
||||
|
|
@ -333,11 +350,33 @@ struct OcrExecution {
|
|||
pending_result: Option<oneshot::Sender<OcrHostResult>>,
|
||||
execution: Option<tokio::task::JoinHandle<Result<LiteLLMOcrResponse, super::Error>>>,
|
||||
completed: bool,
|
||||
azure_ad_token_provider: bool,
|
||||
intercepts_requests: bool,
|
||||
host_token_provider: bool,
|
||||
terminal: Arc<std::sync::Mutex<Option<(CallLifecycleContext, CallLifecycleTiming)>>>,
|
||||
}
|
||||
|
||||
impl OcrExecution {
|
||||
fn set_request(&mut self, mut request: LiteLLMOcrRequest) {
|
||||
let call_id = request
|
||||
.litellm_call_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("ocr-{:032x}", rand::random::<u128>()));
|
||||
let context = CallLifecycleContext::new(
|
||||
"ocr",
|
||||
request.model.clone(),
|
||||
request.provider_name(),
|
||||
call_id.clone(),
|
||||
);
|
||||
request.litellm_call_id = Some(call_id);
|
||||
let start = epoch_seconds();
|
||||
*self
|
||||
.terminal
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner()) =
|
||||
Some((context, CallLifecycleTiming::new(start, start, Vec::new())));
|
||||
self.request = Some(request);
|
||||
}
|
||||
|
||||
fn new(client: OcrClient) -> Self {
|
||||
let (operations_tx, operations_rx) = mpsc::unbounded_channel();
|
||||
Self {
|
||||
|
|
@ -350,7 +389,8 @@ impl OcrExecution {
|
|||
pending_result: None,
|
||||
execution: None,
|
||||
completed: false,
|
||||
azure_ad_token_provider: false,
|
||||
intercepts_requests: false,
|
||||
host_token_provider: false,
|
||||
terminal: Arc::default(),
|
||||
}
|
||||
}
|
||||
|
|
@ -366,11 +406,16 @@ impl OcrExecution {
|
|||
}
|
||||
let result = if self.reading {
|
||||
let Some(OcrHostResult::Document(content)) = result else {
|
||||
return Err(super::Error::InvalidRequest("OCR document read result is required".into()));
|
||||
return Err(super::Error::InvalidRequest(
|
||||
"OCR document read result is required".into(),
|
||||
));
|
||||
};
|
||||
self.reading = false;
|
||||
let content = content?;
|
||||
let request = self.request.take().expect("pending document read has a request");
|
||||
let request = self
|
||||
.request
|
||||
.take()
|
||||
.expect("pending document read has a request");
|
||||
let OcrDocumentInput::HostReader { mime_type } = &request.document else {
|
||||
unreachable!("only host readers request document reads");
|
||||
};
|
||||
|
|
@ -428,7 +473,10 @@ impl OcrExecution {
|
|||
|
||||
async fn prepare(&mut self) -> Result<Option<OcrHostOperation>, super::Error> {
|
||||
if self.preparation.is_none() {
|
||||
let request = self.request.take().expect("admitted OCR call has a request");
|
||||
let request = self
|
||||
.request
|
||||
.take()
|
||||
.expect("admitted OCR call has a request");
|
||||
if matches!(request.document, OcrDocumentInput::HostReader { .. }) {
|
||||
self.request = Some(request);
|
||||
self.reading = true;
|
||||
|
|
@ -444,15 +492,25 @@ impl OcrExecution {
|
|||
OcrDocumentInput::Path { path, mime_type } => {
|
||||
super::document::read_path_document(path, mime_type.as_deref())?
|
||||
}
|
||||
OcrDocumentInput::Bytes { bytes, file_name, mime_type } => {
|
||||
super::document::encode_file_document(bytes, file_name.as_deref(), mime_type.as_deref())?
|
||||
}
|
||||
OcrDocumentInput::Bytes {
|
||||
bytes,
|
||||
file_name,
|
||||
mime_type,
|
||||
} => super::document::encode_file_document(
|
||||
bytes,
|
||||
file_name.as_deref(),
|
||||
mime_type.as_deref(),
|
||||
)?,
|
||||
_ => unreachable!("only native file inputs require preparation"),
|
||||
};
|
||||
Ok(request.with_document(document))
|
||||
}));
|
||||
}
|
||||
let result = self.preparation.as_mut().expect("document preparation started").await;
|
||||
let result = self
|
||||
.preparation
|
||||
.as_mut()
|
||||
.expect("document preparation started")
|
||||
.await;
|
||||
self.preparation = None;
|
||||
let request = result.map_err(|error| super::Error::DocumentTask(Arc::new(error)))??;
|
||||
self.start(request);
|
||||
|
|
@ -461,8 +519,7 @@ impl OcrExecution {
|
|||
|
||||
fn start(&mut self, mut request: ResolvedOcrRequest) {
|
||||
let client = self.client.take().expect("admitted OCR call has a client");
|
||||
let intercepts_requests = request.hooks.intercepts_requests();
|
||||
if self.azure_ad_token_provider {
|
||||
if self.host_token_provider {
|
||||
request.azure_ad_token_provider = Some(TokenProviderHandle::new(Arc::new(
|
||||
OcrAzureAdTokenProvider {
|
||||
operations: self.operations_tx.clone(),
|
||||
|
|
@ -471,7 +528,7 @@ impl OcrExecution {
|
|||
}
|
||||
request.hooks = Arc::new(ProtocolHooks {
|
||||
operations: self.operations_tx.clone(),
|
||||
intercepts_requests,
|
||||
intercepts_requests: self.intercepts_requests,
|
||||
terminal: self.terminal.clone(),
|
||||
});
|
||||
self.execution = Some(tokio::spawn(async move {
|
||||
|
|
@ -517,6 +574,13 @@ struct ProtocolHooks {
|
|||
terminal: Arc<std::sync::Mutex<Option<(CallLifecycleContext, CallLifecycleTiming)>>>,
|
||||
}
|
||||
|
||||
fn epoch_seconds() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs_f64()
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct OcrAzureAdTokenProvider {
|
||||
operations: mpsc::UnboundedSender<PendingOperation>,
|
||||
|
|
@ -743,36 +807,60 @@ mod tests {
|
|||
use crate::ocr::test_support::{MockResponse, mock_server, perform_ocr, wire_request};
|
||||
use crate::ocr::{
|
||||
LiteLLMOcrRequest, NativeOutcome, NoopOcrHost, OcrAdmission, OcrCall, OcrCallStep,
|
||||
OcrDecline, OcrDocument, OcrHost, OcrHostOperation, OcrHostResult,
|
||||
OcrDecline, OcrDocument, OcrHost, OcrHostOperation, OcrHostResult, OcrProjectedRequest,
|
||||
};
|
||||
|
||||
fn projected_request(request: LiteLLMOcrRequest) -> OcrHostResult {
|
||||
OcrHostResult::Request(Ok(OcrProjectedRequest {
|
||||
intercepts_requests: request.hooks.intercepts_requests(),
|
||||
request: Box::new(request),
|
||||
host_token_provider: false,
|
||||
}))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sdk_paths_are_read_during_execution_and_hooks_observe_normalized_documents() {
|
||||
struct CaptureDocument(Arc<Mutex<Vec<OcrDocument>>>);
|
||||
impl OcrHooks for CaptureDocument {
|
||||
fn intercepts_requests(&self) -> bool { true }
|
||||
fn intercepts_requests(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> {
|
||||
self.0.lock().unwrap().push(request.document.clone());
|
||||
Box::pin(async { Ok(request) })
|
||||
}
|
||||
}
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
let path = std::env::temp_dir().join(format!("ocr-sdk-{:032x}.pdf", rand::random::<u128>()));
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("ocr-sdk-{:032x}.pdf", rand::random::<u128>()));
|
||||
let documents = Arc::new(Mutex::new(Vec::new()));
|
||||
let request = LiteLLMOcrRequest::from_inputs(
|
||||
"mistral/model".into(), path.clone(), None, Default::default(),
|
||||
crate::ocr::OcrConnectionInputs { api_base: Some(base), api_key: Some("test-key".into()), ..Default::default() },
|
||||
).unwrap().with_host_hooks(Arc::new(CaptureDocument(documents.clone())), None);
|
||||
"mistral/model".into(),
|
||||
path.clone(),
|
||||
None,
|
||||
Default::default(),
|
||||
crate::ocr::OcrConnectionInputs {
|
||||
api_base: Some(base),
|
||||
api_key: Some("test-key".into()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.with_host_hooks(Arc::new(CaptureDocument(documents.clone())), None);
|
||||
std::fs::write(&path, b"sdk document").unwrap();
|
||||
let result = perform_ocr(request).await;
|
||||
std::fs::remove_file(&path).unwrap();
|
||||
result.unwrap();
|
||||
server.await.unwrap();
|
||||
let expected = json!({"type":"document_url","document_url":"data:application/pdf;base64,c2RrIGRvY3VtZW50"});
|
||||
assert_eq!(serde_json::to_value(&documents.lock().unwrap()[0]).unwrap(), expected);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&documents.lock().unwrap()[0]).unwrap(),
|
||||
expected
|
||||
);
|
||||
let requests = seen.lock().unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
let body: Value = serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap();
|
||||
let body: Value =
|
||||
serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap();
|
||||
assert_eq!(body["document"], expected);
|
||||
}
|
||||
|
||||
|
|
@ -781,9 +869,18 @@ mod tests {
|
|||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
let request = wire_request("mistral/model", &base, json!({}))
|
||||
.with_document(crate::ocr::OcrDocumentInput::HostReader { mime_type: None })
|
||||
.with_host_hooks(Arc::new(AdmissionSpy { effects: Arc::new(Mutex::new(0)) }), None);
|
||||
.with_host_hooks(
|
||||
Arc::new(AdmissionSpy {
|
||||
effects: Arc::new(Mutex::new(0)),
|
||||
}),
|
||||
None,
|
||||
);
|
||||
let mut request = Some(request);
|
||||
let NativeOutcome::Completed(mut call) = OcrCall::admit(crate::ocr::test_support::ocr_client(), OcrAdmission::all()) else { panic!("admission declined"); };
|
||||
let NativeOutcome::Completed(mut call) =
|
||||
OcrCall::admit(crate::ocr::test_support::ocr_client(), OcrAdmission::all())
|
||||
else {
|
||||
panic!("admission declined");
|
||||
};
|
||||
let mut result = None;
|
||||
let mut reads = 0;
|
||||
let mut pre_calls = 0;
|
||||
|
|
@ -791,18 +888,21 @@ mod tests {
|
|||
result = Some(match operation {
|
||||
OcrHostOperation::ProjectRequest => {
|
||||
assert_eq!(reads, 0);
|
||||
OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false)))
|
||||
projected_request(request.take().unwrap())
|
||||
}
|
||||
OcrHostOperation::ReadDocument => {
|
||||
reads += 1;
|
||||
OcrHostResult::Document(Ok(crate::ocr::OcrFileContent {
|
||||
bytes: bytes::Bytes::from_static(b"image"), file_name: Some("scan.png".into()),
|
||||
bytes: bytes::Bytes::from_static(b"image"),
|
||||
file_name: Some("scan.png".into()),
|
||||
}))
|
||||
}
|
||||
OcrHostOperation::PreCall(request) => {
|
||||
assert_eq!(reads, 1);
|
||||
pre_calls += 1;
|
||||
assert!(matches!(&request.document, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2U="));
|
||||
assert!(
|
||||
matches!(&request.document, OcrDocument::ImageUrl { image_url, .. } if image_url == "data:image/png;base64,aW1hZ2U=")
|
||||
);
|
||||
OcrHostResult::PreCall(Ok(request))
|
||||
}
|
||||
operation => NoopOcrHost.invoke(operation).await,
|
||||
|
|
@ -812,6 +912,37 @@ mod tests {
|
|||
assert_eq!((reads, pre_calls, seen.lock().unwrap().len()), (1, 1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sdk_file_failure_dispatches_the_typed_error_once() {
|
||||
struct CaptureFailure(Arc<Mutex<Vec<crate::ocr::Error>>>);
|
||||
impl OcrHooks for CaptureFailure {
|
||||
fn failure<'a>(
|
||||
&'a self,
|
||||
_: &'a CallLifecycleContext,
|
||||
error: &'a crate::ocr::Error,
|
||||
_: &'a CallLifecycleTiming,
|
||||
) -> OcrLogFuture<'a> {
|
||||
self.0.lock().unwrap().push(error.clone());
|
||||
Box::pin(async {})
|
||||
}
|
||||
}
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("missing-ocr-{:032x}.pdf", rand::random::<u128>()));
|
||||
let failures = Arc::new(Mutex::new(Vec::new()));
|
||||
let request = wire_request("mistral/model", "http://127.0.0.1:1", json!({}))
|
||||
.with_document(path.clone().into())
|
||||
.with_host_hooks(Arc::new(CaptureFailure(failures.clone())), None);
|
||||
let error = perform_ocr(request).await.unwrap_err();
|
||||
assert!(
|
||||
matches!(error, crate::ocr::Error::FileRead { path: actual, .. } if actual == path)
|
||||
);
|
||||
let failures = failures.lock().unwrap();
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(
|
||||
matches!(&failures[0], crate::ocr::Error::FileRead { source, .. } if source.kind() == std::io::ErrorKind::NotFound)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_acknowledges_blocking_preparation_completion() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
|
@ -831,7 +962,8 @@ mod tests {
|
|||
std::future::poll_fn(|cx| {
|
||||
assert!(stop.as_mut().poll(cx).is_pending());
|
||||
std::task::Poll::Ready(())
|
||||
}).await;
|
||||
})
|
||||
.await;
|
||||
assert!(!finished.load(Ordering::SeqCst));
|
||||
release_tx.send(()).unwrap();
|
||||
stop.await;
|
||||
|
|
@ -1029,6 +1161,8 @@ mod tests {
|
|||
mut request: OcrDuringCallRequest,
|
||||
) -> OcrHookFuture<'_, OcrDuringCallRequest> {
|
||||
Box::pin(async move {
|
||||
assert_eq!(request.optional_params["pages"], json!([0]));
|
||||
assert_eq!(request.optional_params["extra_body"]["pages"], json!([2]));
|
||||
assert_eq!(request.body["pages"], json!([2]));
|
||||
assert_eq!(request.body.get("future"), Some(&Value::Null));
|
||||
request.body.as_object_mut().unwrap().remove("future");
|
||||
|
|
@ -1282,10 +1416,7 @@ mod tests {
|
|||
result = Some(OcrHostResult::Lifecycle(Ok(())))
|
||||
}
|
||||
OcrHostOperation::ProjectRequest => {
|
||||
result = Some(OcrHostResult::Request(Ok((
|
||||
Box::new(request.take().unwrap()),
|
||||
false,
|
||||
))))
|
||||
result = Some(projected_request(request.take().unwrap()))
|
||||
}
|
||||
OcrHostOperation::AcquireAzureAdToken => {
|
||||
panic!("test request has no token provider")
|
||||
|
|
@ -1307,7 +1438,9 @@ mod tests {
|
|||
Ok(request)
|
||||
}));
|
||||
}
|
||||
OcrHostOperation::PostCall(_) | OcrHostOperation::ReadDocument => panic!("transport should not be reached"),
|
||||
OcrHostOperation::PostCall(_) | OcrHostOperation::ReadDocument => {
|
||||
panic!("transport should not be reached")
|
||||
}
|
||||
},
|
||||
Err(error) => break error,
|
||||
Ok(OcrCallStep::Complete(_)) => panic!("failed call completed"),
|
||||
|
|
@ -1340,10 +1473,7 @@ mod tests {
|
|||
let error = loop {
|
||||
match call.resume(result.take()).await {
|
||||
Ok(OcrCallStep::Host(OcrHostOperation::ProjectRequest)) => {
|
||||
result = Some(OcrHostResult::Request(Ok((
|
||||
Box::new(request.take().unwrap()),
|
||||
false,
|
||||
))));
|
||||
result = Some(projected_request(request.take().unwrap()));
|
||||
}
|
||||
Ok(OcrCallStep::Host(operation)) => {
|
||||
if let OcrHostOperation::PostCall(request) = &operation {
|
||||
|
|
@ -1407,7 +1537,7 @@ mod tests {
|
|||
});
|
||||
result = Some(match operation {
|
||||
OcrHostOperation::ProjectRequest => {
|
||||
OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false)))
|
||||
projected_request(request.take().unwrap())
|
||||
}
|
||||
operation => host.invoke(operation).await,
|
||||
});
|
||||
|
|
@ -1467,7 +1597,9 @@ mod tests {
|
|||
OcrHostResult::Lifecycle(Err(HostFailure::Error(selected.clone())))
|
||||
}
|
||||
OcrHostOperation::Failure { error, .. } => {
|
||||
assert!(matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed"));
|
||||
assert!(
|
||||
matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed")
|
||||
);
|
||||
failures.push("sync");
|
||||
OcrHostResult::Lifecycle(Err(HostFailure::Error(
|
||||
crate::ocr::Error::InvalidRequest("failure callback failed".into()),
|
||||
|
|
@ -1483,7 +1615,7 @@ mod tests {
|
|||
panic!("finalization failure used provider/success dispatch")
|
||||
}
|
||||
OcrHostOperation::ProjectRequest => {
|
||||
OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false)))
|
||||
projected_request(request.take().unwrap())
|
||||
}
|
||||
operation => host.invoke(operation).await,
|
||||
});
|
||||
|
|
@ -1493,7 +1625,9 @@ mod tests {
|
|||
}
|
||||
};
|
||||
server.await.unwrap();
|
||||
assert!(matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed"));
|
||||
assert!(
|
||||
matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "public metadata failed")
|
||||
);
|
||||
assert_eq!(failures, ["sync", "async"]);
|
||||
assert_eq!(seen.lock().unwrap().len(), 1);
|
||||
}
|
||||
|
|
@ -1520,10 +1654,7 @@ mod tests {
|
|||
match call.resume(result.take()).await.unwrap() {
|
||||
OcrCallStep::Host(OcrHostOperation::PreCall(_)) => break,
|
||||
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => {
|
||||
result = Some(OcrHostResult::Request(Ok((
|
||||
Box::new(request.take().unwrap()),
|
||||
false,
|
||||
))))
|
||||
result = Some(projected_request(request.take().unwrap()))
|
||||
}
|
||||
OcrCallStep::Host(operation) => result = Some(host.invoke(operation).await),
|
||||
OcrCallStep::Complete(_) => panic!("provider executed before pre-call result"),
|
||||
|
|
@ -1746,7 +1877,7 @@ mod tests {
|
|||
_ = entered.notified() => break,
|
||||
step = call.resume(result.take()) => {
|
||||
result = Some(match step.unwrap() {
|
||||
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false))),
|
||||
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => projected_request(request.take().unwrap()),
|
||||
OcrCallStep::Host(operation) => NoopOcrHost.invoke(operation).await,
|
||||
OcrCallStep::Complete(_) => panic!("pending provider completed"),
|
||||
});
|
||||
|
|
@ -1773,7 +1904,9 @@ mod tests {
|
|||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(result, Err(crate::ocr::Error::InvalidRequest(message)) if message == "cancelled"));
|
||||
assert!(
|
||||
matches!(result, Err(crate::ocr::Error::InvalidRequest(message)) if message == "cancelled")
|
||||
);
|
||||
assert!(
|
||||
dropped.load(Ordering::SeqCst),
|
||||
"cancellation returned while provider captures were still alive"
|
||||
|
|
|
|||
|
|
@ -18,12 +18,13 @@ pub use client::{OcrClient, ocr};
|
|||
pub use document::{encode_file_document, mime_type_for_name, upload_mime_type};
|
||||
pub use lifecycle::{
|
||||
NativeOutcome, NativeResult, NoopOcrHost, OcrAdmission, OcrCall, OcrCallStep, OcrDecline,
|
||||
OcrHookHost, OcrHost, OcrHostOperation, OcrHostResult,
|
||||
OcrHookHost, OcrHost, OcrHostOperation, OcrHostResult, OcrProjectedRequest,
|
||||
};
|
||||
pub use provider_config::{get_api_key_env_var, get_health_check_document};
|
||||
pub use types::{
|
||||
LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrConnectionInputs, OcrCredentialInputs,
|
||||
OcrDocument, OcrDocumentInput, OcrFileContent, OcrPage, OcrPageDimensions, OcrPageImage, OcrTransportConfig, OcrUsageInfo,
|
||||
OcrDocument, OcrDocumentInput, OcrFileContent, OcrPage, OcrPageDimensions, OcrPageImage,
|
||||
OcrTransportConfig, OcrUsageInfo,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use serde_json::Value;
|
|||
|
||||
use super::OcrClient;
|
||||
use super::hooks::OcrDuringCallRequest;
|
||||
use super::types::{ResolvedOcrRequest, OcrConnection, OcrDocument, PreparedOcrRequest};
|
||||
use super::types::{OcrConnection, OcrDocument, PreparedOcrRequest, ResolvedOcrRequest};
|
||||
|
||||
pub(crate) async fn transform_request_body<B>(
|
||||
client: &OcrClient,
|
||||
|
|
@ -28,6 +28,7 @@ where
|
|||
.during_call(OcrDuringCallRequest {
|
||||
model: request.model.clone(),
|
||||
custom_llm_provider: request.provider_name().into(),
|
||||
optional_params: Value::Object(request.optional_params.clone().into()),
|
||||
api_key: request.connection.api_key.clone(),
|
||||
url: url.into(),
|
||||
headers: headers.to_vec(),
|
||||
|
|
@ -78,6 +79,7 @@ pub(crate) async fn guardrail_document(
|
|||
.during_call(OcrDuringCallRequest {
|
||||
model: request.model.clone(),
|
||||
custom_llm_provider: request.provider_name().into(),
|
||||
optional_params: Value::Object(request.optional_params.clone().into()),
|
||||
api_key: request.connection.api_key.clone(),
|
||||
url: url.into(),
|
||||
headers: headers.to_vec(),
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ use std::path::PathBuf;
|
|||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use serde_with::serde_as;
|
||||
|
||||
|
|
@ -93,7 +93,10 @@ impl From<OcrDocument> for OcrDocumentInput {
|
|||
|
||||
impl From<PathBuf> for OcrDocumentInput {
|
||||
fn from(path: PathBuf) -> Self {
|
||||
Self::Path { path, mime_type: None }
|
||||
Self::Path {
|
||||
path,
|
||||
mime_type: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -324,7 +327,6 @@ impl LiteLLMOcrRequest {
|
|||
config,
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
impl<D> LiteLLMOcrRequest<D> {
|
||||
|
|
@ -383,7 +385,6 @@ impl<D> LiteLLMOcrRequest<D> {
|
|||
..self
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
impl LiteLLMOcrRequest {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use litellm_python_interop::release_count;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
use pyo3::types::{PyDict, PyTuple};
|
||||
|
||||
#[pyfunction]
|
||||
fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
|
|
@ -9,6 +9,27 @@ fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
|
|||
Ok(stats.into_any().unbind())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn _debug_setup(
|
||||
py: Python<'_>,
|
||||
call_type: String,
|
||||
args: Bound<'_, PyTuple>,
|
||||
kwargs: Bound<'_, PyDict>,
|
||||
start: Py<PyAny>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
|
||||
let leaked: &'static str = Box::leak(call_type.into_boxed_str());
|
||||
let result = crate::lifecycle::debug_setup(
|
||||
py,
|
||||
leaked,
|
||||
&args.unbind(),
|
||||
&kwargs.unbind(),
|
||||
&start,
|
||||
asynchronous,
|
||||
)?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[cfg(feature = "panic-test")]
|
||||
#[pyfunction]
|
||||
fn _panic_for_test() {
|
||||
|
|
@ -17,6 +38,7 @@ fn _panic_for_test() {
|
|||
|
||||
pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
module.add_function(wrap_pyfunction!(gil_stats, module)?)?;
|
||||
module.add_function(wrap_pyfunction!(_debug_setup, module)?)?;
|
||||
#[cfg(feature = "panic-test")]
|
||||
module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?;
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
use pyo3::exceptions::PyBaseException;
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyTuple};
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
pub(crate) struct PythonLogger(Py<PyAny>);
|
||||
|
|
@ -19,34 +18,6 @@ impl PythonLogger {
|
|||
visit.call(&self.0)
|
||||
}
|
||||
|
||||
pub(crate) fn callbacks_needed(&self, py: Python<'_>, phase: &str) -> PyResult<bool> {
|
||||
if !self
|
||||
.object(py)
|
||||
.getattr("_native_callback_fast_path")
|
||||
.is_ok_and(|value| value.is_truthy().unwrap_or(false))
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
py.import("litellm.rust_bridge.lifecycle")?
|
||||
.getattr("callbacks_needed")?
|
||||
.call1((self.object(py), phase))?
|
||||
.extract()
|
||||
}
|
||||
|
||||
pub(super) fn success_bookkeeping(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<()> {
|
||||
py.import("litellm.rust_bridge.lifecycle")?
|
||||
.getattr("success_bookkeeping")?
|
||||
.call1((self.object(py), response, start, end, asynchronous))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn defers_async_logging(&self, py: Python<'_>) -> bool {
|
||||
self.object(py)
|
||||
.getattr("_defer_async_logging")
|
||||
|
|
@ -56,150 +27,10 @@ impl PythonLogger {
|
|||
pub(super) fn defer_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
pending: Py<super::PendingLogging>,
|
||||
pending: Py<super::DeferredSuccess>,
|
||||
) -> PyResult<()> {
|
||||
self.object(py).setattr("_native_pending_logging", pending)
|
||||
}
|
||||
|
||||
pub(super) fn sync_success_for_async_call(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()> {
|
||||
if !self.callbacks_needed(py, "sync_success_async")? {
|
||||
return Ok(());
|
||||
}
|
||||
self.object(py).call_method1(
|
||||
"handle_sync_success_callbacks_for_async_calls",
|
||||
(response, start, end),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn failure(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
error: &Py<PyBaseException>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Option<Py<PyAny>>> {
|
||||
if !self.callbacks_needed(
|
||||
py,
|
||||
if asynchronous {
|
||||
"async_failure"
|
||||
} else {
|
||||
"sync_failure"
|
||||
},
|
||||
)? {
|
||||
py.import("litellm.rust_bridge.lifecycle")?
|
||||
.getattr("failure_bookkeeping")?
|
||||
.call1((self.object(py), error, start, end, asynchronous))?;
|
||||
return Ok(None);
|
||||
}
|
||||
let trace = py
|
||||
.import("traceback")?
|
||||
.getattr("format_exception")?
|
||||
.call1((error,))?;
|
||||
let trace = pyo3::types::PyString::new(py, "").call_method1("join", (trace,))?;
|
||||
let value = self.object(py).call_method1(
|
||||
if asynchronous {
|
||||
"async_failure_handler"
|
||||
} else {
|
||||
"failure_handler"
|
||||
},
|
||||
(error, trace, start, end),
|
||||
)?;
|
||||
Ok(asynchronous.then(|| value.unbind()))
|
||||
}
|
||||
|
||||
pub(super) fn restore_context(&self, py: Python<'_>) -> PyResult<()> {
|
||||
py.import("litellm.utils")?
|
||||
.getattr("_restore_correlation_context_if_supported")?
|
||||
.call1((self.object(py),))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn submit_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()> {
|
||||
if !self.callbacks_needed(py, "sync_success")? {
|
||||
return self.success_bookkeeping(py, response, start, end, false);
|
||||
}
|
||||
let context = py.import("contextvars")?.call_method0("copy_context")?;
|
||||
py.import("litellm.litellm_core_utils.litellm_logging")?
|
||||
.getattr("executor")?
|
||||
.call_method1(
|
||||
"submit",
|
||||
(
|
||||
context.getattr("run")?,
|
||||
self.object(py).getattr("success_handler")?,
|
||||
response,
|
||||
start,
|
||||
end,
|
||||
),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn enqueue_success(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()> {
|
||||
if !self.callbacks_needed(py, "async_success")? {
|
||||
return self.success_bookkeeping(py, response, start, end, true);
|
||||
}
|
||||
let context = py.import("contextvars")?.call_method0("copy_context")?;
|
||||
let worker = py
|
||||
.import("litellm.litellm_core_utils.logging_worker")?
|
||||
.getattr("GLOBAL_LOGGING_WORKER")?
|
||||
.getattr("ensure_initialized_and_enqueue")?;
|
||||
let coroutine = self
|
||||
.object(py)
|
||||
.call_method1("async_success_handler", (response, start, end))?;
|
||||
let enqueue = context.call_method1("run", (worker, &coroutine));
|
||||
if enqueue.is_err()
|
||||
&& let Err(error) = coroutine.call_method0("close")
|
||||
{
|
||||
error.write_unraisable(py, Some(&coroutine));
|
||||
}
|
||||
enqueue.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct SetupResult<'py>(Bound<'py, PyAny>);
|
||||
|
||||
impl SetupResult<'_> {
|
||||
pub(super) fn logger(&self) -> PyResult<PythonLogger> {
|
||||
self.0.getattr("logger")?.extract()
|
||||
}
|
||||
|
||||
pub(super) fn kwargs(&self) -> PyResult<Py<PyDict>> {
|
||||
Ok(self.0.getattr("kwargs")?.extract()?)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn setup<'py>(
|
||||
py: Python<'py>,
|
||||
call_type: &str,
|
||||
args: &Py<PyTuple>,
|
||||
kwargs: &Py<PyDict>,
|
||||
start: &Py<PyAny>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<SetupResult<'py>> {
|
||||
py.import("litellm.rust_bridge.lifecycle")?
|
||||
.getattr("setup")?
|
||||
.call1((call_type, args, kwargs, start, asynchronous))
|
||||
.map(SetupResult)
|
||||
}
|
||||
|
||||
pub(super) fn finalize(
|
||||
|
|
@ -210,9 +41,11 @@ pub(super) fn finalize(
|
|||
start: &Py<PyAny>,
|
||||
end: &Option<Py<PyAny>>,
|
||||
) -> PyResult<()> {
|
||||
py.import("litellm.rust_bridge.lifecycle")?
|
||||
.getattr("finalize")?
|
||||
.call1((response, logger.object(py), kwargs, start, end))?;
|
||||
let model = kwargs.bind(py).get_item("model")?;
|
||||
let model = model.filter(|value| value.is_instance_of::<pyo3::types::PyString>());
|
||||
py.import("litellm.litellm_core_utils.llm_response_utils.response_metadata")?
|
||||
.getattr("update_response_metadata")?
|
||||
.call1((response, logger.object(py), model, kwargs, start, end))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -222,170 +55,3 @@ pub(super) fn is_internal_call(py: Python<'_>) -> PyResult<bool> {
|
|||
.call_method0("get")?
|
||||
.extract()
|
||||
}
|
||||
|
||||
pub(super) struct DeploymentHooks;
|
||||
|
||||
impl DeploymentHooks {
|
||||
pub(super) fn needed(py: Python<'_>) -> PyResult<bool> {
|
||||
py.import("litellm.rust_bridge.lifecycle")?
|
||||
.getattr("deployment_callbacks_needed")?
|
||||
.call0()?
|
||||
.extract()
|
||||
}
|
||||
|
||||
pub(super) fn before_call(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
py.import("litellm.utils")?
|
||||
.getattr("async_pre_call_deployment_hook")?
|
||||
.call1((kwargs, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
pub(super) fn after_success(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
response: &Option<Py<PyAny>>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
py.import("litellm.utils")?
|
||||
.getattr("async_post_call_success_deployment_hook")?
|
||||
.call1((kwargs, response, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
pub(super) fn after_failure(
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
error: &Py<PyBaseException>,
|
||||
call_type: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
py.import("litellm.utils")?
|
||||
.getattr("async_post_call_failure_deployment_hook")?
|
||||
.call1((kwargs, error, call_type))
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pyo3::exceptions::PyTypeError;
|
||||
|
||||
#[test]
|
||||
fn setup_fields_are_checked_in_order_without_eager_logger_method_reads() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
pyo3::ffi::c_str!(
|
||||
r#"
|
||||
reads = []
|
||||
class Logger:
|
||||
def __getattribute__(self, name):
|
||||
reads.append(name)
|
||||
raise AssertionError('logger methods must remain lazy')
|
||||
logger = Logger()
|
||||
class Setup:
|
||||
@property
|
||||
def logger(self):
|
||||
reads.append('logger')
|
||||
return logger
|
||||
@property
|
||||
def kwargs(self):
|
||||
reads.append('kwargs')
|
||||
return []
|
||||
result = Setup()
|
||||
"#
|
||||
),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let result = SetupResult(locals.get_item("result").unwrap().unwrap());
|
||||
let logger = result.logger().unwrap();
|
||||
assert!(
|
||||
logger
|
||||
.object(py)
|
||||
.is(locals.get_item("logger").unwrap().unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
locals
|
||||
.get_item("reads")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract::<Vec<String>>()
|
||||
.unwrap(),
|
||||
["logger"]
|
||||
);
|
||||
assert!(
|
||||
result
|
||||
.kwargs()
|
||||
.unwrap_err()
|
||||
.is_instance_of::<PyTypeError>(py)
|
||||
);
|
||||
assert_eq!(
|
||||
locals
|
||||
.get_item("reads")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract::<Vec<String>>()
|
||||
.unwrap(),
|
||||
["logger", "kwargs"]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logger_resolves_each_callback_at_invocation_and_preserves_arguments() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
pyo3::ffi::c_str!(
|
||||
r#"
|
||||
calls = []
|
||||
response, start, end = object(), object(), object()
|
||||
class Logger:
|
||||
@property
|
||||
def handle_sync_success_callbacks_for_async_calls(self):
|
||||
generation = len(calls)
|
||||
def callback(*args):
|
||||
assert args == (response, start, end)
|
||||
calls.append(generation)
|
||||
return callback
|
||||
logger = Logger()
|
||||
"#
|
||||
),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let logger: PythonLogger = locals
|
||||
.get_item("logger")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract()
|
||||
.unwrap();
|
||||
let response = Some(locals.get_item("response").unwrap().unwrap().unbind());
|
||||
let start = locals.get_item("start").unwrap().unwrap().unbind();
|
||||
let end = Some(locals.get_item("end").unwrap().unwrap().unbind());
|
||||
for _ in 0..2 {
|
||||
logger
|
||||
.sync_success_for_async_call(py, &response, &start, &end)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
locals
|
||||
.get_item("calls")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract::<Vec<usize>>()
|
||||
.unwrap(),
|
||||
[0, 1]
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
168
litellm-rust/crates/python-bridge/src/lifecycle/compat.rs
Normal file
168
litellm-rust/crates/python-bridge/src/lifecycle/compat.rs
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
use litellm_core::call_lifecycle::{
|
||||
CallbackFamily, Delivery, ReleaseGate, SuccessFacts, plan_success,
|
||||
};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use super::bindings::PythonLogger;
|
||||
use super::{PythonCallState, dispatch, missing_state};
|
||||
|
||||
pub(super) fn dispatch_success(
|
||||
py: Python<'_>,
|
||||
state: &PythonCallState,
|
||||
logger: &PythonLogger,
|
||||
) -> PyResult<()> {
|
||||
let facts = SuccessFacts {
|
||||
asynchronous: state.asynchronous,
|
||||
internal: state.internal,
|
||||
fallbacks: !state
|
||||
.kwargs
|
||||
.bind(py)
|
||||
.get_item("fallbacks")?
|
||||
.is_none_or(|value| value.is_none()),
|
||||
deferred: logger.defers_async_logging(py),
|
||||
sync_target_kinds: sync_kinds(py, logger)?,
|
||||
};
|
||||
let leaves = dispatch::leaves(py)?;
|
||||
let object = logger.object(py);
|
||||
for selected in plan_success(&facts) {
|
||||
match (selected.family, selected.delivery, selected.gate) {
|
||||
(CallbackFamily::SyncSuccess, Delivery::Worker, _) => {
|
||||
let handler = object.getattr("success_handler")?;
|
||||
let bound = py.import("functools")?.getattr("partial")?.call1((
|
||||
handler,
|
||||
&state.response,
|
||||
&state.start,
|
||||
&state.end,
|
||||
))?;
|
||||
leaves.getattr("submit_worker")?.call1((bound,))?;
|
||||
}
|
||||
(CallbackFamily::AsyncSuccess, Delivery::Background, ReleaseGate::Immediate) => {
|
||||
let coroutine = object.call_method1(
|
||||
"async_success_handler",
|
||||
(&state.response, &state.start, &state.end),
|
||||
)?;
|
||||
let enqueue = leaves.getattr("enqueue_background")?.call1((&coroutine,));
|
||||
if enqueue.is_err()
|
||||
&& let Err(error) = coroutine.call_method0("close")
|
||||
{
|
||||
error.write_unraisable(py, Some(&coroutine));
|
||||
}
|
||||
enqueue?;
|
||||
}
|
||||
(CallbackFamily::AsyncSuccess, Delivery::Background, ReleaseGate::Deferred) => {
|
||||
let pending = Py::new(
|
||||
py,
|
||||
SuppliedDeferred {
|
||||
logger: Some(logger.clone_ref(py)),
|
||||
response: state.response.as_ref().map(|value| value.clone_ref(py)),
|
||||
start: state.start.clone_ref(py),
|
||||
end: state.end.as_ref().map(|value| value.clone_ref(py)),
|
||||
},
|
||||
)?;
|
||||
object.setattr("_native_pending_logging", pending)?;
|
||||
}
|
||||
_ => return Err(missing_state()),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sync_kinds(
|
||||
py: Python<'_>,
|
||||
logger: &PythonLogger,
|
||||
) -> PyResult<Vec<litellm_core::call_lifecycle::CallbackKind>> {
|
||||
let (targets, ids) = dispatch::family_targets(py, logger, CallbackFamily::SyncSuccess)?;
|
||||
Ok(targets.kinds(&ids))
|
||||
}
|
||||
|
||||
pub(super) fn dispatch_failure(
|
||||
py: Python<'_>,
|
||||
state: &PythonCallState,
|
||||
family: CallbackFamily,
|
||||
) -> PyResult<Option<Py<PyAny>>> {
|
||||
let logger = state.logger()?.object(py);
|
||||
let error = state.error.as_ref().ok_or_else(missing_state)?;
|
||||
let trace = py
|
||||
.import("traceback")?
|
||||
.getattr("format_exception")?
|
||||
.call1((error,))?;
|
||||
let trace = pyo3::types::PyString::new(py, "").call_method1("join", (trace,))?;
|
||||
match family {
|
||||
CallbackFamily::SyncFailure => {
|
||||
logger.call_method1("failure_handler", (error, trace, &state.start, &state.end))?;
|
||||
Ok(None)
|
||||
}
|
||||
CallbackFamily::AsyncFailure => Ok(Some(
|
||||
logger
|
||||
.call_method1(
|
||||
"async_failure_handler",
|
||||
(error, trace, &state.start, &state.end),
|
||||
)?
|
||||
.unbind(),
|
||||
)),
|
||||
_ => Err(missing_state()),
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass]
|
||||
struct SuppliedDeferred {
|
||||
logger: Option<PythonLogger>,
|
||||
response: Option<Py<PyAny>>,
|
||||
start: Py<PyAny>,
|
||||
end: Option<Py<PyAny>>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl SuppliedDeferred {
|
||||
fn __call__(slf: &Bound<'_, Self>, py: Python<'_>) -> PyResult<()> {
|
||||
let logger = slf.borrow_mut().logger.take();
|
||||
let Some(logger) = logger else {
|
||||
return Ok(());
|
||||
};
|
||||
let (response, start, end) = {
|
||||
let this = slf.borrow();
|
||||
(
|
||||
this.response.as_ref().map(|value| value.clone_ref(py)),
|
||||
this.start.clone_ref(py),
|
||||
this.end.as_ref().map(|value| value.clone_ref(py)),
|
||||
)
|
||||
};
|
||||
let object = logger.object(py);
|
||||
let coroutine = object.call_method1("async_success_handler", (response, start, end))?;
|
||||
let enqueue = dispatch::leaves(py)?
|
||||
.getattr("enqueue_background")?
|
||||
.call1((&coroutine,));
|
||||
if enqueue.is_err()
|
||||
&& let Err(error) = coroutine.call_method0("close")
|
||||
{
|
||||
error.write_unraisable(py, Some(&coroutine));
|
||||
}
|
||||
match enqueue {
|
||||
Err(error) if error.is_instance_of::<pyo3::exceptions::PyException>(py) => {
|
||||
error.write_unraisable(py, Some(object));
|
||||
Ok(())
|
||||
}
|
||||
result => result.map(|_| ()),
|
||||
}
|
||||
}
|
||||
|
||||
fn __traverse__(&self, visit: pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> {
|
||||
if let Some(logger) = &self.logger {
|
||||
logger.traverse(&visit)?;
|
||||
}
|
||||
visit.call(&self.response)?;
|
||||
visit.call(&self.start)?;
|
||||
visit.call(&self.end)
|
||||
}
|
||||
|
||||
fn close(slf: &Bound<'_, Self>) {
|
||||
let mut this = slf.borrow_mut();
|
||||
this.logger = None;
|
||||
this.response = None;
|
||||
this.end = None;
|
||||
}
|
||||
|
||||
fn __clear__(slf: &Bound<'_, Self>) {
|
||||
Self::close(slf);
|
||||
}
|
||||
}
|
||||
1000
litellm-rust/crates/python-bridge/src/lifecycle/dispatch.rs
Normal file
1000
litellm-rust/crates/python-bridge/src/lifecycle/dispatch.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,41 +1,40 @@
|
|||
use std::sync::Arc;
|
||||
use std::task::Poll;
|
||||
|
||||
use futures_util::future::{AbortHandle, Abortable};
|
||||
#[cfg(test)]
|
||||
use litellm_core::call_lifecycle::host::HostCallFuture;
|
||||
use litellm_core::call_lifecycle::host::{
|
||||
HostCall as NativeCall, HostCallStep as NativeCallStep, HostFailure, HostPhase, HostStep,
|
||||
};
|
||||
use pyo3::exceptions::{PyBaseException, PyException, PyRuntimeError};
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyTuple};
|
||||
|
||||
use futures_util::future::{AbortHandle, Abortable};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::execution::{poll_async_value, run_async_value, run_sync_value};
|
||||
use litellm_core::call_lifecycle::host::{
|
||||
HostCall as NativeCall, HostCallStep as NativeCallStep, HostFailure, HostPhase, HostStep,
|
||||
};
|
||||
use litellm_core::call_lifecycle::{
|
||||
CallbackFamily, Delivery, Dispatch, ReleaseGate, SuccessFacts, plan_failure, plan_success,
|
||||
};
|
||||
|
||||
mod arguments;
|
||||
mod bindings;
|
||||
mod compat;
|
||||
mod dispatch;
|
||||
mod handle;
|
||||
mod preparation;
|
||||
mod setup;
|
||||
|
||||
use crate::execution::{poll_async_value, run_async_value, run_sync_value};
|
||||
pub(crate) use arguments::{BoundArguments, Signature};
|
||||
use bindings::DeploymentHooks;
|
||||
pub(crate) use bindings::PythonLogger;
|
||||
use handle::{Execution, ExecutionBody, ExecutionStep};
|
||||
|
||||
pub(crate) enum OperationClass {
|
||||
Phase(HostPhase),
|
||||
Route,
|
||||
}
|
||||
|
||||
pub(crate) trait PythonRoute: Send + Sync {
|
||||
type Call: NativeCall + 'static;
|
||||
|
||||
fn state(&self) -> &PythonCallState;
|
||||
fn state_mut(&mut self) -> &mut PythonCallState;
|
||||
fn classify(operation: &<Self::Call as NativeCall>::Operation) -> OperationClass;
|
||||
fn phase(operation: &<Self::Call as NativeCall>::Operation) -> Option<HostPhase>;
|
||||
fn lifecycle_result() -> <Self::Call as NativeCall>::Result;
|
||||
fn map_error(error: <Self::Call as NativeCall>::Error) -> PyErr;
|
||||
fn host_error(message: String) -> <Self::Call as NativeCall>::Error;
|
||||
|
|
@ -98,6 +97,18 @@ pub(crate) fn run_call<R: PythonRoute + 'static>(
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) fn debug_setup(
|
||||
py: Python<'_>,
|
||||
call_type: &'static str,
|
||||
args: &Py<PyTuple>,
|
||||
kwargs: &Py<PyDict>,
|
||||
start: &Py<PyAny>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
|
||||
let result = setup::setup(py, call_type, args, kwargs, start, asynchronous)?;
|
||||
Ok((result.logger.object(py).clone().unbind(), result.kwargs))
|
||||
}
|
||||
|
||||
pub(crate) fn missing_state() -> PyErr {
|
||||
pyo3::exceptions::PyRuntimeError::new_err("missing native call state")
|
||||
}
|
||||
|
|
@ -167,12 +178,12 @@ impl<R: PythonRoute> PythonLifecycle<R> {
|
|||
HostFailure::Cancelled(native)
|
||||
};
|
||||
let state = self.route.state_mut();
|
||||
if state.error.is_none() || (cancelled && phase != Some(HostPhase::DeploymentFailure)) {
|
||||
state.retain_error(py, error);
|
||||
}
|
||||
if state.end.is_none() {
|
||||
state.end = now(py).ok();
|
||||
}
|
||||
state.retain_first_error(
|
||||
py,
|
||||
error,
|
||||
cancelled && phase != Some(HostPhase::DeploymentFailure),
|
||||
);
|
||||
let _ = state.finish(py);
|
||||
failure
|
||||
}
|
||||
|
||||
|
|
@ -215,10 +226,7 @@ impl<R: PythonRoute> PythonLifecycle<R> {
|
|||
}
|
||||
HostStep::Ready(NativeCallStep::Host(operation)) => operation,
|
||||
};
|
||||
let phase = match R::classify(&operation) {
|
||||
OperationClass::Phase(phase) => Some(phase),
|
||||
OperationClass::Route => None,
|
||||
};
|
||||
let phase = R::phase(&operation);
|
||||
let result = match phase {
|
||||
Some(phase) => match self.route.state_mut().invoke(py, phase) {
|
||||
Ok(HostStep::Suspend(awaitable)) => {
|
||||
|
|
@ -295,6 +303,7 @@ pub(crate) struct PythonCallState {
|
|||
pub error: Option<Py<PyBaseException>>,
|
||||
pub asynchronous: bool,
|
||||
pub internal: bool,
|
||||
pub supplied: bool,
|
||||
pub call_type: &'static str,
|
||||
}
|
||||
|
||||
|
|
@ -314,42 +323,34 @@ impl PythonCallState {
|
|||
match phase {
|
||||
HostPhase::Setup => self.setup(py)?,
|
||||
HostPhase::DeploymentPreCall => {
|
||||
if !DeploymentHooks::needed(py)? {
|
||||
return Ok(HostStep::Ready(self.kwargs.clone_ref(py).into_any()));
|
||||
}
|
||||
return Ok(HostStep::Suspend(DeploymentHooks::before_call(
|
||||
let copied = self.kwargs.bind(py).copy()?.into_any().unbind();
|
||||
return Ok(HostStep::Suspend(self.deployment(
|
||||
py,
|
||||
&self.kwargs,
|
||||
self.call_type,
|
||||
CallbackFamily::DeploymentPreCall,
|
||||
copied,
|
||||
)?));
|
||||
}
|
||||
HostPhase::Prepare => self.prepare(py)?,
|
||||
HostPhase::DeploymentPostCall => {
|
||||
if !DeploymentHooks::needed(py)? {
|
||||
return self
|
||||
.response
|
||||
.as_ref()
|
||||
.map(|value| HostStep::Ready(value.clone_ref(py)))
|
||||
.ok_or_else(missing_state);
|
||||
}
|
||||
return Ok(HostStep::Suspend(DeploymentHooks::after_success(
|
||||
let response = self
|
||||
.response
|
||||
.as_ref()
|
||||
.map(|value| value.clone_ref(py))
|
||||
.unwrap_or_else(|| py.None());
|
||||
return Ok(HostStep::Suspend(self.deployment(
|
||||
py,
|
||||
&self.kwargs,
|
||||
&self.response,
|
||||
self.call_type,
|
||||
CallbackFamily::DeploymentPostCall,
|
||||
response,
|
||||
)?));
|
||||
}
|
||||
HostPhase::Finalize => self.finalize(py)?,
|
||||
HostPhase::Success => self.dispatch_success(py)?,
|
||||
HostPhase::DeploymentFailure => {
|
||||
if let Some(error) = &self.error
|
||||
&& DeploymentHooks::needed(py)?
|
||||
{
|
||||
return Ok(HostStep::Suspend(DeploymentHooks::after_failure(
|
||||
if self.error.is_some() {
|
||||
return Ok(HostStep::Suspend(self.deployment(
|
||||
py,
|
||||
&self.kwargs,
|
||||
error,
|
||||
self.call_type,
|
||||
CallbackFamily::DeploymentFailure,
|
||||
py.None(),
|
||||
)?));
|
||||
}
|
||||
}
|
||||
|
|
@ -368,6 +369,24 @@ impl PythonCallState {
|
|||
Ok(HostStep::Ready(py.None()))
|
||||
}
|
||||
|
||||
fn deployment(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
family: CallbackFamily,
|
||||
current: Py<PyAny>,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let body = dispatch::DeploymentBody::start(
|
||||
py,
|
||||
self.logger()?,
|
||||
family,
|
||||
self.call_type,
|
||||
&self.kwargs,
|
||||
current,
|
||||
self.error.as_ref(),
|
||||
)?;
|
||||
dispatch::deployment_coroutine(py, body)
|
||||
}
|
||||
|
||||
fn accept(&mut self, py: Python<'_>, phase: HostPhase, value: Py<PyAny>) -> PyResult<()> {
|
||||
match phase {
|
||||
HostPhase::DeploymentPreCall => {
|
||||
|
|
@ -396,6 +415,7 @@ impl PythonCallState {
|
|||
error: None,
|
||||
asynchronous,
|
||||
internal: false,
|
||||
supplied: false,
|
||||
call_type,
|
||||
})
|
||||
}
|
||||
|
|
@ -409,7 +429,7 @@ impl PythonCallState {
|
|||
pub fn setup(&mut self, py: Python<'_>) -> PyResult<()> {
|
||||
self.start = now(py)?;
|
||||
self.internal = bindings::is_internal_call(py)?;
|
||||
let result = bindings::setup(
|
||||
let result = setup::setup(
|
||||
py,
|
||||
self.call_type,
|
||||
&self.args,
|
||||
|
|
@ -417,8 +437,9 @@ impl PythonCallState {
|
|||
&self.start,
|
||||
self.asynchronous,
|
||||
)?;
|
||||
self.logger = Some(result.logger()?);
|
||||
self.kwargs = result.kwargs()?;
|
||||
self.logger = Some(result.logger);
|
||||
self.kwargs = result.kwargs;
|
||||
self.supplied = result.supplied;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -438,6 +459,16 @@ impl PythonCallState {
|
|||
)
|
||||
}
|
||||
|
||||
pub fn dispatch_request(&self, py: Python<'_>, family: CallbackFamily) -> PyResult<()> {
|
||||
dispatch::dispatch_request(
|
||||
py,
|
||||
dispatch::RequestJob {
|
||||
logger: self.logger()?,
|
||||
family,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
|
||||
match self.try_dispatch_success(py) {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
|
|
@ -448,51 +479,76 @@ impl PythonCallState {
|
|||
}
|
||||
}
|
||||
|
||||
fn job(&self, py: Python<'_>, selected: Dispatch) -> PyResult<dispatch::Job> {
|
||||
let logger = self.logger()?;
|
||||
let (targets, ids) = dispatch::family_targets(py, logger, selected.family)?;
|
||||
let sync_request = dispatch::leaves(py)?
|
||||
.getattr("is_sync_request")?
|
||||
.call1((logger.object(py),))?
|
||||
.extract()?;
|
||||
Ok(dispatch::Job {
|
||||
logger: logger.clone_ref(py),
|
||||
targets,
|
||||
ids,
|
||||
dispatch: selected,
|
||||
sync_request,
|
||||
response: self.response.as_ref().map(|value| value.clone_ref(py)),
|
||||
error: self.error.as_ref().map(|value| value.clone_ref(py)),
|
||||
start: self.start.clone_ref(py),
|
||||
end: self
|
||||
.end
|
||||
.as_ref()
|
||||
.map(|value| value.clone_ref(py))
|
||||
.unwrap_or_else(|| py.None()),
|
||||
stream: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn try_dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
|
||||
let logger = self.logger()?;
|
||||
let pending = || PendingSuccess {
|
||||
logger: logger.clone_ref(py),
|
||||
response: self.response.as_ref().map(|value| value.clone_ref(py)),
|
||||
start: self.start.clone_ref(py),
|
||||
end: self.end.as_ref().map(|value| value.clone_ref(py)),
|
||||
if self.supplied {
|
||||
return compat::dispatch_success(py, self, logger);
|
||||
}
|
||||
let (sync_targets, sync_ids) =
|
||||
dispatch::family_targets(py, logger, CallbackFamily::SyncSuccess)?;
|
||||
let facts = SuccessFacts {
|
||||
asynchronous: self.asynchronous,
|
||||
internal: self.internal,
|
||||
fallbacks: !self
|
||||
.kwargs
|
||||
.bind(py)
|
||||
.get_item("fallbacks")?
|
||||
.is_none_or(|value| value.is_none()),
|
||||
deferred: logger.defers_async_logging(py),
|
||||
sync_target_kinds: sync_targets.kinds(&sync_ids),
|
||||
};
|
||||
if !self.asynchronous {
|
||||
if !logger.callbacks_needed(py, "sync_success")? {
|
||||
return logger.success_bookkeeping(
|
||||
py,
|
||||
&self.response,
|
||||
&self.start,
|
||||
&self.end,
|
||||
false,
|
||||
);
|
||||
}
|
||||
pending().sync(py)
|
||||
} else {
|
||||
if !self.internal
|
||||
&& self
|
||||
.kwargs
|
||||
.bind(py)
|
||||
.get_item("fallbacks")?
|
||||
.is_none_or(|value| value.is_none())
|
||||
{
|
||||
if !logger.callbacks_needed(py, "async_success")? {
|
||||
logger.success_bookkeeping(py, &self.response, &self.start, &self.end, true)?;
|
||||
} else if logger.defers_async_logging(py) {
|
||||
for selected in plan_success(&facts) {
|
||||
let runner = dispatch::Runner::start(py, self.job(py, selected)?)?;
|
||||
match (selected.delivery, selected.gate) {
|
||||
(Delivery::Worker, _) => {
|
||||
let job = Py::new(py, dispatch::WorkerJob::new(runner))?;
|
||||
dispatch::leaves(py)?
|
||||
.getattr("submit_worker")?
|
||||
.call1((job,))?;
|
||||
}
|
||||
(Delivery::Background, ReleaseGate::Immediate) => {
|
||||
DeferredSuccess::release(py, runner)?;
|
||||
}
|
||||
(Delivery::Background, ReleaseGate::Deferred) => {
|
||||
logger.defer_success(
|
||||
py,
|
||||
Py::new(
|
||||
py,
|
||||
PendingLogging {
|
||||
pending: Some(pending()),
|
||||
DeferredSuccess {
|
||||
runner: Some(runner),
|
||||
},
|
||||
)?,
|
||||
)?;
|
||||
} else {
|
||||
pending().asynchronous(py)?;
|
||||
}
|
||||
(Delivery::Inline | Delivery::Await, _) => return Err(missing_state()),
|
||||
}
|
||||
logger.sync_success_for_async_call(py, &self.response, &self.start, &self.end)
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn dispatch_failure(
|
||||
|
|
@ -500,19 +556,39 @@ impl PythonCallState {
|
|||
py: Python<'_>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Option<Py<PyAny>>> {
|
||||
if self.logger.is_none() || (self.asynchronous && self.internal) {
|
||||
if self.logger.is_none() || self.error.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(error) = &self.error else {
|
||||
let phase = if asynchronous {
|
||||
HostPhase::AsyncFailure
|
||||
} else {
|
||||
HostPhase::Failure
|
||||
};
|
||||
let Some(selected) = plan_failure(phase, self.asynchronous, self.internal) else {
|
||||
return Ok(None);
|
||||
};
|
||||
self.logger()?
|
||||
.failure(py, error, &self.start, &self.end, asynchronous)
|
||||
if self.supplied {
|
||||
return compat::dispatch_failure(py, self, selected.family);
|
||||
}
|
||||
let mut runner = dispatch::Runner::start(py, self.job(py, selected)?)?;
|
||||
match selected.delivery {
|
||||
Delivery::Inline => match runner.resume(py, None)? {
|
||||
dispatch::Step::Done => Ok(None),
|
||||
dispatch::Step::Await(_) => Err(missing_state()),
|
||||
},
|
||||
Delivery::Await => Ok(Some(dispatch::coroutine(py, runner)?)),
|
||||
Delivery::Worker | Delivery::Background => Err(missing_state()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cleanup(&mut self, py: Python<'_>) {
|
||||
if let Some(logger) = self.logger.take()
|
||||
&& let Err(error) = logger.restore_context(py)
|
||||
&& let Err(error) = dispatch::leaves(py).and_then(|leaves| {
|
||||
leaves
|
||||
.getattr("restore_correlation_context")?
|
||||
.call1((logger.object(py),))
|
||||
.map(|_| ())
|
||||
})
|
||||
{
|
||||
error.write_unraisable(py, None);
|
||||
}
|
||||
|
|
@ -522,6 +598,19 @@ impl PythonCallState {
|
|||
self.error = Some(error.into_value(py));
|
||||
}
|
||||
|
||||
pub fn retain_first_error(&mut self, py: Python<'_>, error: PyErr, replace: bool) {
|
||||
if self.error.is_none() || replace {
|
||||
self.retain_error(py, error);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finish(&mut self, py: Python<'_>) -> PyResult<()> {
|
||||
if self.end.is_none() {
|
||||
self.end = Some(now(py)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
visit.call(&self.args)?;
|
||||
visit.call(&self.kwargs)?;
|
||||
|
|
@ -535,74 +624,96 @@ impl PythonCallState {
|
|||
}
|
||||
}
|
||||
|
||||
struct PendingSuccess {
|
||||
logger: PythonLogger,
|
||||
response: Option<Py<PyAny>>,
|
||||
start: Py<PyAny>,
|
||||
end: Option<Py<PyAny>>,
|
||||
}
|
||||
|
||||
impl PendingSuccess {
|
||||
fn sync(&self, py: Python<'_>) -> PyResult<()> {
|
||||
self.logger
|
||||
.submit_success(py, &self.response, &self.start, &self.end)
|
||||
}
|
||||
|
||||
fn asynchronous(&self, py: Python<'_>) -> PyResult<()> {
|
||||
self.logger
|
||||
.enqueue_success(py, &self.response, &self.start, &self.end)
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass]
|
||||
struct PendingLogging {
|
||||
pending: Option<PendingSuccess>,
|
||||
struct DeferredSuccess {
|
||||
runner: Option<dispatch::Runner>,
|
||||
}
|
||||
|
||||
impl DeferredSuccess {
|
||||
fn release(py: Python<'_>, runner: dispatch::Runner) -> PyResult<()> {
|
||||
let coroutine = dispatch::coroutine(py, runner)?;
|
||||
let enqueue = dispatch::leaves(py)?
|
||||
.getattr("enqueue_background")?
|
||||
.call1((&coroutine,));
|
||||
if enqueue.is_err()
|
||||
&& let Err(error) = coroutine.call_method0(py, "close")
|
||||
{
|
||||
error.write_unraisable(py, Some(coroutine.bind(py)));
|
||||
}
|
||||
enqueue.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PendingLogging {
|
||||
fn release(slf: &Bound<'_, Self>, py: Python<'_>, success: bool) -> PyResult<()> {
|
||||
let pending = slf.borrow_mut().pending.take();
|
||||
if let Some(pending) = pending
|
||||
&& success
|
||||
{
|
||||
match pending.asynchronous(py) {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
error.write_unraisable(py, Some(pending.logger.object(py)));
|
||||
}
|
||||
result => return result,
|
||||
impl DeferredSuccess {
|
||||
fn __call__(slf: &Bound<'_, Self>, py: Python<'_>) -> PyResult<()> {
|
||||
let runner = slf.borrow_mut().runner.take();
|
||||
let Some(runner) = runner else {
|
||||
return Ok(());
|
||||
};
|
||||
let logger = runner.logger(py).clone().unbind();
|
||||
match Self::release(py, runner) {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
error.write_unraisable(py, Some(logger.bind(py)));
|
||||
Ok(())
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn __traverse__(&self, visit: pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> {
|
||||
if let Some(pending) = &self.pending {
|
||||
pending.logger.traverse(&visit)?;
|
||||
visit.call(&pending.response)?;
|
||||
visit.call(&pending.start)?;
|
||||
visit.call(&pending.end)?;
|
||||
match &self.runner {
|
||||
Some(runner) => runner.traverse(&visit),
|
||||
None => Ok(()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn close(slf: &Bound<'_, Self>) {
|
||||
let runner = slf.borrow_mut().runner.take();
|
||||
drop(runner);
|
||||
}
|
||||
|
||||
fn __clear__(slf: &Bound<'_, Self>) {
|
||||
let pending = slf.borrow_mut().pending.take();
|
||||
drop(pending);
|
||||
Self::close(slf);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use pyo3::types::PyDict;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use litellm_core::call_lifecycle::host::HostCallFuture;
|
||||
|
||||
static PYTHON_GLOBALS: Mutex<()> = Mutex::new(());
|
||||
|
||||
fn install_logging_worker(py: Python<'_>, worker: &Bound<'_, PyAny>) -> PyResult<()> {
|
||||
py.import("litellm.litellm_core_utils.logging_worker")?
|
||||
.setattr("GLOBAL_LOGGING_WORKER", worker)
|
||||
fn load_lifecycle_module(py: Python<'_>) -> Bound<'_, PyModule> {
|
||||
py.run(
|
||||
pyo3::ffi::c_str!(
|
||||
r#"
|
||||
import sys
|
||||
import types
|
||||
for name in ("litellm", "litellm.rust_bridge"):
|
||||
sys.modules.setdefault(name, types.ModuleType(name))
|
||||
"#
|
||||
),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let source = std::ffi::CString::new(include_str!(
|
||||
"../../../../../litellm/rust_bridge/lifecycle.py"
|
||||
))
|
||||
.unwrap();
|
||||
PyModule::from_code(
|
||||
py,
|
||||
&source,
|
||||
pyo3::ffi::c_str!("lifecycle.py"),
|
||||
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
struct RetainingHost {
|
||||
|
|
@ -715,8 +826,8 @@ mod tests {
|
|||
&mut self.0
|
||||
}
|
||||
|
||||
fn classify(_: &()) -> OperationClass {
|
||||
OperationClass::Route
|
||||
fn phase(_: &()) -> Option<HostPhase> {
|
||||
None
|
||||
}
|
||||
|
||||
fn lifecycle_result() {}
|
||||
|
|
@ -774,17 +885,7 @@ mod tests {
|
|||
.unwrap_or_else(|error| error.into_inner());
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let source = std::ffi::CString::new(include_str!(
|
||||
"../../../../../litellm/rust_bridge/lifecycle.py"
|
||||
))
|
||||
.unwrap();
|
||||
PyModule::from_code(
|
||||
py,
|
||||
&source,
|
||||
pyo3::ffi::c_str!("lifecycle.py"),
|
||||
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
|
||||
)
|
||||
.unwrap();
|
||||
load_lifecycle_module(py);
|
||||
let route = SyntheticRoute(
|
||||
PythonCallState::new(
|
||||
py,
|
||||
|
|
@ -820,17 +921,7 @@ mod tests {
|
|||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
py.import("asyncio").unwrap();
|
||||
let source = std::ffi::CString::new(include_str!(
|
||||
"../../../../../litellm/rust_bridge/lifecycle.py"
|
||||
))
|
||||
.unwrap();
|
||||
let module = PyModule::from_code(
|
||||
py,
|
||||
&source,
|
||||
pyo3::ffi::c_str!("lifecycle.py"),
|
||||
pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"),
|
||||
)
|
||||
.unwrap();
|
||||
let module = load_lifecycle_module(py);
|
||||
let locals = PyDict::new(py);
|
||||
locals
|
||||
.set_item("drive", module.getattr("drive").unwrap())
|
||||
|
|
@ -938,64 +1029,11 @@ assert reference() is None
|
|||
error: None,
|
||||
asynchronous,
|
||||
internal: false,
|
||||
supplied: false,
|
||||
call_type: "test",
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_dispatch_reports_ordinary_failures_without_replacing_response() {
|
||||
let _guard = PYTHON_GLOBALS
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
pyo3::ffi::c_str!(
|
||||
r#"
|
||||
import sys
|
||||
|
||||
response = object()
|
||||
failure = ValueError('terminal diagnostic')
|
||||
diagnostics = []
|
||||
old_hook = sys.unraisablehook
|
||||
sys.unraisablehook = lambda event: diagnostics.append(event.exc_value)
|
||||
|
||||
class Logger:
|
||||
def handle_sync_success_callbacks_for_async_calls(self, *args):
|
||||
raise failure
|
||||
|
||||
logger = Logger()
|
||||
"#
|
||||
),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let response = locals.get_item("response").unwrap().unwrap().unbind();
|
||||
let mut lifecycle_state = state(
|
||||
py,
|
||||
locals.get_item("logger").unwrap().unwrap().unbind(),
|
||||
response.clone_ref(py),
|
||||
true,
|
||||
);
|
||||
lifecycle_state.internal = true;
|
||||
lifecycle_state.dispatch_success(py).unwrap();
|
||||
assert!(lifecycle_state.response.as_ref().unwrap().is(&response));
|
||||
py.run(
|
||||
pyo3::ffi::c_str!(
|
||||
r#"
|
||||
assert diagnostics == [failure]
|
||||
sys.unraisablehook = old_hook
|
||||
"#
|
||||
),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retained_failure_preserves_exception_identity() {
|
||||
Python::initialize();
|
||||
|
|
@ -1011,139 +1049,6 @@ sys.unraisablehook = old_hook
|
|||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deferred_release_uses_release_context_and_allows_reentry_once() {
|
||||
let _guard = PYTHON_GLOBALS
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
pyo3::ffi::c_str!(
|
||||
r#"
|
||||
import sys
|
||||
import types
|
||||
from contextvars import ContextVar
|
||||
|
||||
litellm = types.ModuleType('litellm')
|
||||
core_utils = types.ModuleType('litellm.litellm_core_utils')
|
||||
logging_worker = types.ModuleType('litellm.litellm_core_utils.logging_worker')
|
||||
litellm.litellm_core_utils = core_utils
|
||||
core_utils.logging_worker = logging_worker
|
||||
sys.modules['litellm'] = litellm
|
||||
sys.modules['litellm.litellm_core_utils'] = core_utils
|
||||
sys.modules['litellm.litellm_core_utils.logging_worker'] = logging_worker
|
||||
|
||||
marker = ContextVar('marker', default='unset')
|
||||
observed = []
|
||||
|
||||
class Coroutine:
|
||||
def close(self):
|
||||
observed.append('closed')
|
||||
|
||||
class Worker:
|
||||
def ensure_initialized_and_enqueue(self, coroutine):
|
||||
observed.append(marker.get())
|
||||
pending.release(True)
|
||||
coroutine.close()
|
||||
|
||||
class Logger:
|
||||
def async_success_handler(self, *args):
|
||||
observed.append('created')
|
||||
return Coroutine()
|
||||
|
||||
worker = Worker()
|
||||
logger = Logger()
|
||||
"#
|
||||
),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
install_logging_worker(py, &locals.get_item("worker").unwrap().unwrap()).unwrap();
|
||||
let pending = Py::new(
|
||||
py,
|
||||
PendingLogging {
|
||||
pending: Some(PendingSuccess {
|
||||
logger: locals
|
||||
.get_item("logger")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract()
|
||||
.unwrap(),
|
||||
response: Some(py.None()),
|
||||
start: py.None(),
|
||||
end: Some(py.None()),
|
||||
}),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
locals.set_item("pending", &pending).unwrap();
|
||||
py.run(
|
||||
pyo3::ffi::c_str!(
|
||||
r#"
|
||||
marker.set('release')
|
||||
pending.release(True)
|
||||
pending.release(True)
|
||||
assert observed == ['created', 'release', 'closed']
|
||||
"#
|
||||
),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deferred_logging_collects_cycles_through_typed_logger() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
pyo3::ffi::c_str!("class Logger: pass\nlogger = Logger()"),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let pending = Py::new(
|
||||
py,
|
||||
PendingLogging {
|
||||
pending: Some(PendingSuccess {
|
||||
logger: locals
|
||||
.get_item("logger")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract()
|
||||
.unwrap(),
|
||||
response: None,
|
||||
start: py.None(),
|
||||
end: None,
|
||||
}),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
locals.set_item("pending", pending).unwrap();
|
||||
py.run(
|
||||
pyo3::ffi::c_str!(
|
||||
r#"
|
||||
import gc
|
||||
import weakref
|
||||
logger.pending = pending
|
||||
reference = weakref.ref(logger)
|
||||
del logger, pending
|
||||
gc.collect()
|
||||
assert reference() is None
|
||||
"#
|
||||
),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coroutine_collects_cycles_retained_by_bridge_host() {
|
||||
Python::initialize();
|
||||
|
|
|
|||
337
litellm-rust/crates/python-bridge/src/lifecycle/setup.rs
Normal file
337
litellm-rust/crates/python-bridge/src/lifecycle/setup.rs
Normal file
|
|
@ -0,0 +1,337 @@
|
|||
use litellm_core::call_lifecycle::CallbackId;
|
||||
use litellm_core::call_lifecycle::registration::{
|
||||
Candidate, DynamicSuccessSlot, Entry, NamedEvent, Registration, RegistrationFacts, Registry,
|
||||
RegistryMutation, classify_dynamic_success, plan_registration,
|
||||
};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyList, PyString, PyTuple};
|
||||
|
||||
use super::bindings::PythonLogger;
|
||||
|
||||
const SETUP_MODULE: &str = "litellm.rust_bridge.setup";
|
||||
|
||||
struct Targets<'py> {
|
||||
objects: Vec<Bound<'py, PyAny>>,
|
||||
}
|
||||
|
||||
impl<'py> Targets<'py> {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
objects: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn intern(&mut self, object: &Bound<'py, PyAny>) -> CallbackId {
|
||||
if let Some(index) = self
|
||||
.objects
|
||||
.iter()
|
||||
.position(|existing| existing.is(object) || existing.eq(object).unwrap_or(false))
|
||||
{
|
||||
return CallbackId(index as u64);
|
||||
}
|
||||
self.objects.push(object.clone());
|
||||
CallbackId((self.objects.len() - 1) as u64)
|
||||
}
|
||||
|
||||
fn get(&self, id: CallbackId) -> PyResult<&Bound<'py, PyAny>> {
|
||||
self.objects
|
||||
.get(id.0 as usize)
|
||||
.ok_or_else(super::missing_state)
|
||||
}
|
||||
}
|
||||
|
||||
fn registration<'py>(
|
||||
setup: &Bound<'py, PyModule>,
|
||||
object: &Bound<'py, PyAny>,
|
||||
) -> PyResult<Registration> {
|
||||
if let Ok(name) = object.cast::<PyString>() {
|
||||
let name = name.to_str()?;
|
||||
let known = setup.getattr("is_known_name")?.call1((name,))?.extract()?;
|
||||
return Ok(Registration::Named {
|
||||
known,
|
||||
async_only: matches!(name, "dynamodb" | "openmeter"),
|
||||
});
|
||||
}
|
||||
let asynchronous = setup
|
||||
.getattr("is_async_callable")?
|
||||
.call1((object,))?
|
||||
.extract()?;
|
||||
Ok(Registration::Object { asynchronous })
|
||||
}
|
||||
|
||||
fn entries<'py>(
|
||||
setup: &Bound<'py, PyModule>,
|
||||
targets: &mut Targets<'py>,
|
||||
list: &Bound<'py, PyAny>,
|
||||
) -> PyResult<Vec<Entry>> {
|
||||
list.try_iter()?
|
||||
.map(|object| {
|
||||
let object = object?;
|
||||
Ok(Entry {
|
||||
id: targets.intern(&object),
|
||||
registration: registration(setup, &object)?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn registry_name(registry: Registry) -> &'static str {
|
||||
match registry {
|
||||
Registry::Input => "input",
|
||||
Registry::AsyncInput => "async_input",
|
||||
Registry::Success => "success",
|
||||
Registry::AsyncSuccess => "async_success",
|
||||
Registry::Failure => "failure",
|
||||
Registry::AsyncFailure => "async_failure",
|
||||
}
|
||||
}
|
||||
|
||||
fn read_registry<'py>(setup: &Bound<'py, PyModule>, name: &str) -> PyResult<Bound<'py, PyAny>> {
|
||||
setup.getattr("registry")?.call1((name,))
|
||||
}
|
||||
|
||||
fn read_candidates<'py>(
|
||||
setup: &Bound<'py, PyModule>,
|
||||
targets: &mut Targets<'py>,
|
||||
dynamic: Option<Bound<'py, PyAny>>,
|
||||
) -> PyResult<Vec<Candidate>> {
|
||||
let mut candidates = Vec::new();
|
||||
let global = read_registry(setup, "callbacks")?;
|
||||
let sources = std::iter::once(global).chain(dynamic);
|
||||
for source in sources {
|
||||
for object in source.try_iter()? {
|
||||
let object = object?;
|
||||
let candidate = if object.is_instance_of::<PyString>() {
|
||||
let resolved = setup
|
||||
.getattr("resolve_named_integration")?
|
||||
.call1((&object,))?;
|
||||
if resolved.is_none() {
|
||||
Candidate {
|
||||
resolved: None,
|
||||
duplicate_type: false,
|
||||
}
|
||||
} else {
|
||||
let duplicate_type = setup
|
||||
.getattr("async_success_registry_has_type")?
|
||||
.call1((&resolved,))?
|
||||
.extract()?;
|
||||
Candidate {
|
||||
resolved: Some(Entry {
|
||||
id: targets.intern(&resolved),
|
||||
registration: registration(setup, &resolved)?,
|
||||
}),
|
||||
duplicate_type,
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Candidate {
|
||||
resolved: Some(Entry {
|
||||
id: targets.intern(&object),
|
||||
registration: registration(setup, &object)?,
|
||||
}),
|
||||
duplicate_type: false,
|
||||
}
|
||||
};
|
||||
candidates.push(candidate);
|
||||
}
|
||||
}
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
fn apply<'py>(
|
||||
setup: &Bound<'py, PyModule>,
|
||||
targets: &Targets<'py>,
|
||||
mutations: &[RegistryMutation],
|
||||
function_id: Option<&Bound<'py, PyAny>>,
|
||||
) -> PyResult<()> {
|
||||
for mutation in mutations {
|
||||
match mutation {
|
||||
RegistryMutation::Append(registry, id) => {
|
||||
setup
|
||||
.getattr("append_registry")?
|
||||
.call1((registry_name(*registry), targets.get(*id)?))?;
|
||||
}
|
||||
RegistryMutation::Remove(registry, id) => {
|
||||
setup
|
||||
.getattr("remove_registry")?
|
||||
.call1((registry_name(*registry), targets.get(*id)?))?;
|
||||
}
|
||||
RegistryMutation::ExpandNamed(event, id) => {
|
||||
let event = match event {
|
||||
NamedEvent::Success => "success",
|
||||
NamedEvent::Failure => "failure",
|
||||
};
|
||||
setup
|
||||
.getattr("expand_named")?
|
||||
.call1((targets.get(*id)?, event))?;
|
||||
}
|
||||
RegistryMutation::Bootstrap => {
|
||||
setup.getattr("bootstrap")?.call1((function_id,))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct DynamicLists<'py> {
|
||||
success: Option<Bound<'py, PyList>>,
|
||||
async_success: Option<Bound<'py, PyList>>,
|
||||
failure: Option<Bound<'py, PyList>>,
|
||||
}
|
||||
|
||||
fn split_dynamic<'py>(
|
||||
py: Python<'py>,
|
||||
setup: &Bound<'py, PyModule>,
|
||||
targets: &mut Targets<'py>,
|
||||
kwargs: &Bound<'py, PyDict>,
|
||||
) -> PyResult<DynamicLists<'py>> {
|
||||
let success = match kwargs.get_item("success_callback")? {
|
||||
Some(value) if value.is_instance_of::<PyList>() => {
|
||||
let list = value.cast_into::<PyList>()?;
|
||||
let sync = PyList::empty(py);
|
||||
let asynchronous = PyList::empty(py);
|
||||
for object in list.iter() {
|
||||
let entry = Entry {
|
||||
id: targets.intern(&object),
|
||||
registration: registration(setup, &object)?,
|
||||
};
|
||||
let named_async = object
|
||||
.cast::<PyString>()
|
||||
.ok()
|
||||
.and_then(|name| {
|
||||
name.to_str()
|
||||
.ok()
|
||||
.map(|name| matches!(name, "dynamodb" | "s3"))
|
||||
})
|
||||
.unwrap_or(false);
|
||||
match classify_dynamic_success(entry, named_async) {
|
||||
DynamicSuccessSlot::Sync => sync.append(&object)?,
|
||||
DynamicSuccessSlot::Async => asynchronous.append(&object)?,
|
||||
}
|
||||
}
|
||||
kwargs.del_item("success_callback")?;
|
||||
Some((sync, (!asynchronous.is_empty()).then_some(asynchronous)))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let failure = match kwargs.get_item("failure_callback")? {
|
||||
Some(value) if value.is_instance_of::<PyList>() => {
|
||||
kwargs.del_item("failure_callback")?;
|
||||
Some(value.cast_into::<PyList>()?)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let (success, async_success) = match success {
|
||||
Some((sync, asynchronous)) => (Some(sync), asynchronous),
|
||||
None => (None, None),
|
||||
};
|
||||
Ok(DynamicLists {
|
||||
success,
|
||||
async_success,
|
||||
failure,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) struct Setup {
|
||||
pub logger: PythonLogger,
|
||||
pub kwargs: Py<PyDict>,
|
||||
pub supplied: bool,
|
||||
}
|
||||
|
||||
pub(super) fn setup(
|
||||
py: Python<'_>,
|
||||
call_type: &str,
|
||||
args: &Py<PyTuple>,
|
||||
kwargs: &Py<PyDict>,
|
||||
start: &Py<PyAny>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Setup> {
|
||||
let setup = py.import(SETUP_MODULE)?;
|
||||
let kwargs = kwargs.bind(py).copy()?;
|
||||
if !kwargs.contains("litellm_call_id")? {
|
||||
let call_id = py.import("uuid")?.call_method0("uuid4")?.str()?;
|
||||
kwargs.set_item("litellm_call_id", call_id)?;
|
||||
}
|
||||
if let Some(supplied) = kwargs.get_item("litellm_logging_obj")? {
|
||||
let logging_class = py
|
||||
.import("litellm.litellm_core_utils.litellm_logging")?
|
||||
.getattr("Logging")?;
|
||||
if supplied.is_instance(&logging_class)? {
|
||||
return Ok(Setup {
|
||||
logger: supplied.extract()?,
|
||||
kwargs: kwargs.unbind(),
|
||||
supplied: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
setup.getattr("prepare_environment")?.call0()?;
|
||||
let guardrails = setup.getattr("applied_guardrails")?.call1((&kwargs,))?;
|
||||
let function_id = kwargs.get_item("id")?;
|
||||
|
||||
let mut targets = Targets::new();
|
||||
let dynamic = match kwargs.get_item("callbacks")? {
|
||||
Some(value) => {
|
||||
kwargs.del_item("callbacks")?;
|
||||
(!value.is_none()).then_some(value)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let candidates = read_candidates(&setup, &mut targets, dynamic)?;
|
||||
let facts = RegistrationFacts {
|
||||
candidates,
|
||||
input: entries(&setup, &mut targets, &read_registry(&setup, "input")?)?,
|
||||
success: entries(&setup, &mut targets, &read_registry(&setup, "success")?)?,
|
||||
failure: entries(&setup, &mut targets, &read_registry(&setup, "failure")?)?,
|
||||
async_success: entries(
|
||||
&setup,
|
||||
&mut targets,
|
||||
&read_registry(&setup, "async_success")?,
|
||||
)?
|
||||
.into_iter()
|
||||
.map(|entry| entry.id)
|
||||
.collect(),
|
||||
async_failure: entries(
|
||||
&setup,
|
||||
&mut targets,
|
||||
&read_registry(&setup, "async_failure")?,
|
||||
)?
|
||||
.into_iter()
|
||||
.map(|entry| entry.id)
|
||||
.collect(),
|
||||
bootstrap_pending: setup.getattr("bootstrap_pending")?.call0()?.extract()?,
|
||||
};
|
||||
apply(
|
||||
&setup,
|
||||
&targets,
|
||||
&plan_registration(&facts),
|
||||
function_id.as_ref(),
|
||||
)?;
|
||||
|
||||
let dynamic = split_dynamic(py, &setup, &mut targets, &kwargs)?;
|
||||
setup.getattr("breadcrumb")?.call1((&kwargs,))?;
|
||||
if let Some(logger_fn) = kwargs.get_item("logger_fn")? {
|
||||
setup.getattr("logger_fn")?.call1((logger_fn,))?;
|
||||
}
|
||||
let model = match args.bind(py).get_item(0) {
|
||||
Ok(model) => Some(model),
|
||||
Err(_) => kwargs.get_item("model")?,
|
||||
};
|
||||
let build = setup.getattr("build_logging")?;
|
||||
let build_kwargs = PyDict::new(py);
|
||||
build_kwargs.set_item("call_type", call_type)?;
|
||||
build_kwargs.set_item("model", model)?;
|
||||
build_kwargs.set_item("kwargs", &kwargs)?;
|
||||
build_kwargs.set_item("start_time", start)?;
|
||||
build_kwargs.set_item("asynchronous", asynchronous)?;
|
||||
build_kwargs.set_item("dynamic_success", dynamic.success)?;
|
||||
build_kwargs.set_item("dynamic_async_success", dynamic.async_success)?;
|
||||
build_kwargs.set_item("dynamic_failure", dynamic.failure)?;
|
||||
build_kwargs.set_item("guardrails", guardrails)?;
|
||||
let logger = build.call((), Some(&build_kwargs))?;
|
||||
Ok(Setup {
|
||||
logger: logger.extract()?,
|
||||
kwargs: kwargs.unbind(),
|
||||
supplied: false,
|
||||
})
|
||||
}
|
||||
|
|
@ -13,6 +13,11 @@ use litellm_python_interop::from_py_preserving_errors as from_py;
|
|||
use crate::auth::{AZURE_AD_TOKEN_PROVIDER, PythonTokenProvider};
|
||||
use crate::lifecycle::BoundArguments;
|
||||
|
||||
pub(crate) struct Projection<Native, Retained> {
|
||||
pub native: Native,
|
||||
pub retained: Retained,
|
||||
}
|
||||
|
||||
/// Fields every lifecycle route reads from its bound `*args, **kwargs` before
|
||||
/// asking core to build the typed request. Route-specific inputs (for example
|
||||
/// the OCR `document`) are read separately by the route.
|
||||
|
|
|
|||
|
|
@ -3,158 +3,94 @@ use pyo3::types::PyDict;
|
|||
use serde_json::Value;
|
||||
|
||||
use litellm_core::ocr::LiteLLMOcrResponse;
|
||||
use litellm_core::ocr::hooks::OcrPreCallRequest;
|
||||
use litellm_core::ocr::hooks::OcrDuringCallRequest;
|
||||
use litellm_python_interop::to_py_preserving_errors as to_py;
|
||||
|
||||
use crate::lifecycle::PythonLogger;
|
||||
use litellm_core::call_lifecycle::CallbackFamily;
|
||||
|
||||
pub(super) struct OcrLoggingFields {
|
||||
model: String,
|
||||
custom_llm_provider: String,
|
||||
optional_params: Value,
|
||||
use super::host::PythonPayload;
|
||||
use crate::lifecycle::{PythonCallState, PythonLogger};
|
||||
|
||||
pub(super) fn update_logging(
|
||||
py: Python<'_>,
|
||||
logger: &PythonLogger,
|
||||
kwargs: &Py<PyDict>,
|
||||
request: &OcrDuringCallRequest,
|
||||
secret_fields: &[&str],
|
||||
) -> PyResult<()> {
|
||||
py.import("litellm.rust_bridge.ocr")?
|
||||
.getattr("update_logging")?
|
||||
.call1((
|
||||
logger.object(py),
|
||||
kwargs,
|
||||
&request.model,
|
||||
&request.custom_llm_provider,
|
||||
to_py(py, &request.optional_params)?,
|
||||
secret_fields,
|
||||
&request.url,
|
||||
))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl From<&OcrPreCallRequest> for OcrLoggingFields {
|
||||
fn from(request: &OcrPreCallRequest) -> Self {
|
||||
Self {
|
||||
model: request.model.clone(),
|
||||
custom_llm_provider: request.custom_llm_provider.clone(),
|
||||
optional_params: request.optional_params.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonLogger {
|
||||
pub(super) fn update_ocr(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
kwargs: &Py<PyDict>,
|
||||
pre_call: &OcrLoggingFields,
|
||||
secret_fields: &[&str],
|
||||
url: &str,
|
||||
) -> PyResult<()> {
|
||||
let update = PyDict::new(py);
|
||||
update.set_item("kwargs", redact(py, kwargs.bind(py), secret_fields)?)?;
|
||||
update.set_item("model", &pre_call.model)?;
|
||||
update.set_item(
|
||||
"optional_params",
|
||||
redact(
|
||||
py,
|
||||
&to_py(py, &pre_call.optional_params)?
|
||||
.into_bound(py)
|
||||
.cast_into::<PyDict>()?,
|
||||
secret_fields,
|
||||
)?,
|
||||
)?;
|
||||
let params = PyDict::new(py);
|
||||
params.set_item(
|
||||
"litellm_call_id",
|
||||
kwargs.bind(py).get_item("litellm_call_id")?,
|
||||
)?;
|
||||
params.set_item("api_base", url)?;
|
||||
for name in ["logger_fn", "litellm_request_debug"] {
|
||||
if let Some(value) = kwargs.bind(py).get_item(name)? {
|
||||
params.set_item(name, value)?;
|
||||
}
|
||||
}
|
||||
for name in custom_pricing_fields(py)? {
|
||||
if let Some(value) = kwargs.bind(py).get_item(&name)?
|
||||
&& !value.is_none()
|
||||
{
|
||||
params.set_item(name, value)?;
|
||||
}
|
||||
}
|
||||
update.set_item("litellm_params", params)?;
|
||||
update.set_item("custom_llm_provider", &pre_call.custom_llm_provider)?;
|
||||
self.object(py)
|
||||
.call_method("update_from_kwargs", (), Some(&update))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn pre_ocr(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
api_key: Option<&str>,
|
||||
body: &Bound<'_, PyDict>,
|
||||
headers: &Bound<'_, PyDict>,
|
||||
url: &str,
|
||||
) -> PyResult<()> {
|
||||
let additional = PyDict::new(py);
|
||||
additional.set_item("complete_input_dict", body)?;
|
||||
additional.set_item("headers", headers)?;
|
||||
additional.set_item("api_base", url)?;
|
||||
pub(super) fn pre_call(
|
||||
py: Python<'_>,
|
||||
state: &PythonCallState,
|
||||
request: &OcrDuringCallRequest,
|
||||
payload: &PythonPayload,
|
||||
) -> PyResult<()> {
|
||||
let logger = state.logger()?;
|
||||
if state.supplied {
|
||||
let additional_args = PyDict::new(py);
|
||||
additional_args.set_item("complete_input_dict", &payload.body)?;
|
||||
additional_args.set_item("headers", &payload.headers)?;
|
||||
additional_args.set_item("api_base", &request.url)?;
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("input", "OCR document processing")?;
|
||||
kwargs.set_item("api_key", api_key)?;
|
||||
kwargs.set_item("additional_args", &additional)?;
|
||||
if self.callbacks_needed(py, "input")? {
|
||||
self.object(py).call_method("pre_call", (), Some(&kwargs))?;
|
||||
} else {
|
||||
self.object(py)
|
||||
.call_method("_pre_call", (), Some(&kwargs))?;
|
||||
self.object(py).call_method0("record_api_call_start_time")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn post_ocr(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
original_response: &Value,
|
||||
body: Option<&Py<PyDict>>,
|
||||
headers: Option<&Py<PyDict>>,
|
||||
) -> PyResult<()> {
|
||||
let additional = PyDict::new(py);
|
||||
additional.set_item("complete_input_dict", body)?;
|
||||
additional.set_item("headers", headers)?;
|
||||
if self.callbacks_needed(py, "input")? {
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("original_response", to_py(py, original_response)?)?;
|
||||
kwargs.set_item("additional_args", &additional)?;
|
||||
self.object(py)
|
||||
.call_method("post_call", (), Some(&kwargs))?;
|
||||
} else {
|
||||
let response = py
|
||||
.import("json")?
|
||||
.call_method1("dumps", (to_py(py, original_response)?,))?;
|
||||
self.object(py).call_method1(
|
||||
"record_post_call",
|
||||
(response, py.None(), py.None(), additional),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
kwargs.set_item("api_key", request.api_key.as_deref())?;
|
||||
kwargs.set_item("additional_args", additional_args)?;
|
||||
logger
|
||||
.object(py)
|
||||
.call_method("pre_call", (), Some(&kwargs))?;
|
||||
return Ok(());
|
||||
}
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("api_key", request.api_key.as_deref())?;
|
||||
kwargs.set_item("body", &payload.body)?;
|
||||
kwargs.set_item("headers", &payload.headers)?;
|
||||
kwargs.set_item("url", &request.url)?;
|
||||
py.import("litellm.rust_bridge.leaves")?
|
||||
.getattr("record_pre_call")?
|
||||
.call((logger.object(py),), Some(&kwargs))?;
|
||||
state.dispatch_request(py, CallbackFamily::RequestPreCall)
|
||||
}
|
||||
|
||||
fn custom_pricing_fields(py: Python<'_>) -> PyResult<Vec<String>> {
|
||||
py.import("litellm.types.utils")?
|
||||
.getattr("CustomPricingLiteLLMParams")?
|
||||
.getattr("model_fields")?
|
||||
.cast_into::<PyDict>()?
|
||||
.keys()
|
||||
.iter()
|
||||
.map(|name| name.extract::<String>())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn redact(
|
||||
pub(super) fn post_call(
|
||||
py: Python<'_>,
|
||||
params: &Bound<'_, PyDict>,
|
||||
secret_fields: &[&str],
|
||||
) -> PyResult<Py<PyDict>> {
|
||||
let redacted = PyDict::new(py);
|
||||
for (name, value) in params {
|
||||
let name = name.extract::<String>()?;
|
||||
if name == "proxy_server_request" {
|
||||
continue;
|
||||
}
|
||||
if secret_fields.contains(&name.as_str()) {
|
||||
redacted.set_item(name, "****")?;
|
||||
} else {
|
||||
redacted.set_item(name, value)?;
|
||||
}
|
||||
state: &PythonCallState,
|
||||
original_response: &Value,
|
||||
payload: &PythonPayload,
|
||||
) -> PyResult<()> {
|
||||
let logger = state.logger()?;
|
||||
if state.supplied {
|
||||
let additional_args = PyDict::new(py);
|
||||
additional_args.set_item("complete_input_dict", &payload.body)?;
|
||||
additional_args.set_item("headers", &payload.headers)?;
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("original_response", to_py(py, original_response)?)?;
|
||||
kwargs.set_item("additional_args", additional_args)?;
|
||||
logger
|
||||
.object(py)
|
||||
.call_method("post_call", (), Some(&kwargs))?;
|
||||
return Ok(());
|
||||
}
|
||||
Ok(redacted.unbind())
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("original_response", to_py(py, original_response)?)?;
|
||||
kwargs.set_item("body", &payload.body)?;
|
||||
kwargs.set_item("headers", &payload.headers)?;
|
||||
py.import("litellm.rust_bridge.leaves")?
|
||||
.getattr("record_post_call")?
|
||||
.call((logger.object(py),), Some(&kwargs))?;
|
||||
state.dispatch_request(py, CallbackFamily::RequestPostCall)
|
||||
}
|
||||
|
||||
pub(super) fn response(py: Python<'_>, response: &LiteLLMOcrResponse) -> PyResult<Py<PyAny>> {
|
||||
|
|
|
|||
|
|
@ -36,11 +36,11 @@ fn extract_bytes(value: &Bound<'_, PyAny>) -> PyResult<Bytes> {
|
|||
return Ok(Bytes::from_owner(value.extract::<PyBackedBytes>()?));
|
||||
}
|
||||
// Bytes subclasses may retain GC edges that a native Bytes owner cannot traverse.
|
||||
Ok(Bytes::copy_from_slice(value.extract::<PyBackedBytes>()?.as_ref()))
|
||||
Ok(Bytes::copy_from_slice(
|
||||
value.extract::<PyBackedBytes>()?.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
|
||||
pub(super) struct FileDocumentInput {
|
||||
pub input: OcrDocumentInput,
|
||||
pub reader: Option<PythonFileReader>,
|
||||
|
|
@ -56,9 +56,11 @@ impl FromPyObject<'_, '_> for FileDocumentInput {
|
|||
Err(error) if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py) => None,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let missing = || PyValueError::new_err(
|
||||
"document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes",
|
||||
);
|
||||
let missing = || {
|
||||
PyValueError::new_err(
|
||||
"document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes",
|
||||
)
|
||||
};
|
||||
let file = document.get_item("file").map_err(|error| {
|
||||
if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py) {
|
||||
missing()
|
||||
|
|
@ -93,7 +95,9 @@ impl FromPyObject<'_, '_> for FileDocumentInput {
|
|||
reader: None,
|
||||
});
|
||||
}
|
||||
let reader = file.getattr_opt("read")?.filter(|value| value.is_callable());
|
||||
let reader = file
|
||||
.getattr_opt("read")?
|
||||
.filter(|value| value.is_callable());
|
||||
let Some(reader) = reader else {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"Unsupported file input type: {}. Expected pathlib.Path, bytes, or a file-like object.",
|
||||
|
|
@ -107,7 +111,10 @@ impl FromPyObject<'_, '_> for FileDocumentInput {
|
|||
.transpose()?;
|
||||
Ok(Self {
|
||||
input: OcrDocumentInput::HostReader { mime_type },
|
||||
reader: Some(PythonFileReader { reader: reader.unbind(), name }),
|
||||
reader: Some(PythonFileReader {
|
||||
reader: reader.unbind(),
|
||||
name,
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -127,12 +134,21 @@ mod tests {
|
|||
let error = document.extract::<FileDocumentInput>().err().unwrap();
|
||||
assert!(error.is_instance_of::<PyValueError>(py));
|
||||
}
|
||||
for expression in [c"{'file': b'abc', 'mime_type': None}", c"{'file': b'abc', 'mime_type': 7}"] {
|
||||
let error = py.eval(expression, None, None).unwrap().extract::<FileDocumentInput>().err().unwrap();
|
||||
for expression in [
|
||||
c"{'file': b'abc', 'mime_type': None}",
|
||||
c"{'file': b'abc', 'mime_type': 7}",
|
||||
] {
|
||||
let error = py
|
||||
.eval(expression, None, None)
|
||||
.unwrap()
|
||||
.extract::<FileDocumentInput>()
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(error.is_instance_of::<PyTypeError>(py));
|
||||
}
|
||||
let locals = PyDict::new(py);
|
||||
py.run(c"from pathlib import Path
|
||||
py.run(
|
||||
c"from pathlib import Path
|
||||
failure = KeyError('reader failed')
|
||||
class Reader:
|
||||
def __init__(self):
|
||||
|
|
@ -143,12 +159,31 @@ class Reader:
|
|||
reader = Reader()
|
||||
document = {'file': reader}
|
||||
path_document = {'file': Path('/nonexistent/ocr-projection-test.pdf')}
|
||||
", Some(&locals), Some(&locals)).unwrap();
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let document = locals.get_item("document").unwrap().unwrap();
|
||||
let input: FileDocumentInput = document.extract().unwrap();
|
||||
assert_eq!(locals.get_item("reader").unwrap().unwrap().getattr("reads").unwrap().extract::<usize>().unwrap(), 0);
|
||||
assert_eq!(
|
||||
locals
|
||||
.get_item("reader")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.getattr("reads")
|
||||
.unwrap()
|
||||
.extract::<usize>()
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert!(input.reader.is_some());
|
||||
let path: FileDocumentInput = locals.get_item("path_document").unwrap().unwrap().extract().unwrap();
|
||||
let path: FileDocumentInput = locals
|
||||
.get_item("path_document")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract()
|
||||
.unwrap();
|
||||
assert!(matches!(path.input, OcrDocumentInput::Path { .. }));
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -103,9 +103,13 @@ pub(super) fn public_exception(
|
|||
std::io::ErrorKind::PermissionDenied => 13,
|
||||
_ => 5,
|
||||
});
|
||||
return Ok(PyErr::from_value(py.import("builtins")?.getattr("OSError")?.call1((
|
||||
errno, source.to_string(), path,
|
||||
))?));
|
||||
return Ok(PyErr::from_value(
|
||||
py.import("builtins")?.getattr("OSError")?.call1((
|
||||
errno,
|
||||
source.to_string(),
|
||||
path.into_pyobject(py)?.call_method0("__fspath__")?,
|
||||
))?,
|
||||
));
|
||||
}
|
||||
raise_public(py, classify(error), model, provider, None)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use pyo3::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
use litellm_auth::ResolvedCredential;
|
||||
use litellm_core::call_lifecycle::host::HostPhase;
|
||||
use litellm_core::ocr::hooks::{OcrDuringCallRequest, OcrPostCallRequest};
|
||||
use litellm_core::ocr::{OcrCall, OcrHostOperation, OcrHostResult};
|
||||
use litellm_core::ocr::{OcrCall, OcrHostOperation, OcrHostResult, OcrProjectedRequest};
|
||||
use litellm_python_interop::{
|
||||
from_py_preserving_errors as from_py, to_py_preserving_errors as to_py,
|
||||
};
|
||||
|
|
@ -13,75 +13,100 @@ use litellm_python_interop::{
|
|||
use super::document::PythonFileReader;
|
||||
use super::errors::to_pyerr as ocr_error_to_pyerr;
|
||||
use super::project::project;
|
||||
use super::{ASYNC_SIGNATURE, SIGNATURE, callbacks, errors};
|
||||
use super::{callbacks, errors};
|
||||
use crate::auth::PythonTokenProvider;
|
||||
use crate::lifecycle::{OperationClass, PythonCallState, PythonRoute, missing_state, now};
|
||||
use crate::lifecycle::{PythonCallState, PythonRoute, Signature, missing_state};
|
||||
use crate::marshal::Projection;
|
||||
|
||||
pub(super) struct PythonOcrHost {
|
||||
state: PythonCallState,
|
||||
projected: Option<ProjectedOcrHost>,
|
||||
signature: &'static Signature,
|
||||
retained: Option<OcrRetained>,
|
||||
}
|
||||
|
||||
struct ProjectedOcrHost {
|
||||
model: String,
|
||||
provider: &'static str,
|
||||
secret_fields: Vec<&'static str>,
|
||||
azure_ad_token_provider: Option<PythonTokenProvider>,
|
||||
pre_call: Option<callbacks::OcrLoggingFields>,
|
||||
reader: Option<PythonFileReader>,
|
||||
reader_failed: bool,
|
||||
body: Option<Py<PyDict>>,
|
||||
headers: Option<Py<PyDict>>,
|
||||
pub(super) struct OcrRetained {
|
||||
pub model: String,
|
||||
pub provider: &'static str,
|
||||
pub secret_fields: Vec<&'static str>,
|
||||
pub azure_ad_token_provider: Option<PythonTokenProvider>,
|
||||
pub payload: Option<PythonPayload>,
|
||||
pub reader: Option<PythonFileReader>,
|
||||
}
|
||||
|
||||
pub(super) struct PythonPayload {
|
||||
pub body: Py<PyDict>,
|
||||
pub headers: Py<PyDict>,
|
||||
}
|
||||
|
||||
impl PythonPayload {
|
||||
fn from_request(py: Python<'_>, request: &OcrDuringCallRequest) -> PyResult<Self> {
|
||||
let body = to_py(py, &request.body)?
|
||||
.into_bound(py)
|
||||
.cast_into::<PyDict>()?;
|
||||
let headers = PyDict::new(py);
|
||||
for (name, value) in &request.headers {
|
||||
headers.set_item(name, value)?;
|
||||
}
|
||||
Ok(Self {
|
||||
body: body.unbind(),
|
||||
headers: headers.unbind(),
|
||||
})
|
||||
}
|
||||
|
||||
fn write_back(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
mut request: OcrDuringCallRequest,
|
||||
) -> PyResult<OcrDuringCallRequest> {
|
||||
request.body = from_py(self.body.bind(py))?;
|
||||
request.headers = self
|
||||
.headers
|
||||
.bind(py)
|
||||
.iter()
|
||||
.map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
visit.call(&self.body)?;
|
||||
visit.call(&self.headers)
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonOcrHost {
|
||||
pub(super) fn new(state: PythonCallState) -> Self {
|
||||
pub(super) fn new(state: PythonCallState, signature: &'static Signature) -> Self {
|
||||
Self {
|
||||
state,
|
||||
projected: None,
|
||||
signature,
|
||||
retained: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn projected(&self) -> PyResult<&ProjectedOcrHost> {
|
||||
self.projected.as_ref().ok_or_else(missing_state)
|
||||
fn retained(&self) -> PyResult<&OcrRetained> {
|
||||
self.retained.as_ref().ok_or_else(missing_state)
|
||||
}
|
||||
|
||||
fn projected_mut(&mut self) -> PyResult<&mut ProjectedOcrHost> {
|
||||
self.projected.as_mut().ok_or_else(missing_state)
|
||||
fn retained_mut(&mut self) -> PyResult<&mut OcrRetained> {
|
||||
self.retained.as_mut().ok_or_else(missing_state)
|
||||
}
|
||||
|
||||
fn project(&mut self, py: Python<'_>) -> PyResult<OcrHostResult> {
|
||||
let signature = if self.state.asynchronous {
|
||||
&ASYNC_SIGNATURE
|
||||
} else {
|
||||
&SIGNATURE
|
||||
};
|
||||
let arguments = signature.bind(self.state.args.bind(py), self.state.kwargs.bind(py))?;
|
||||
let projected = project(py, &arguments)?;
|
||||
let has_token_provider = projected.azure_ad_token_provider.is_some();
|
||||
self.projected = Some(ProjectedOcrHost {
|
||||
model: projected.request.model.clone(),
|
||||
provider: projected.request.provider_name(),
|
||||
secret_fields: projected.secret_fields,
|
||||
azure_ad_token_provider: projected.azure_ad_token_provider,
|
||||
pre_call: None,
|
||||
reader: projected.reader,
|
||||
reader_failed: false,
|
||||
body: None,
|
||||
headers: None,
|
||||
});
|
||||
Ok(OcrHostResult::Request(Ok((
|
||||
Box::new(
|
||||
projected
|
||||
.request
|
||||
.with_host_hooks(Arc::new(BridgeOcrHooks), None),
|
||||
),
|
||||
has_token_provider,
|
||||
))))
|
||||
let arguments = self
|
||||
.signature
|
||||
.bind(self.state.args.bind(py), self.state.kwargs.bind(py))?;
|
||||
let Projection { native, retained } = project(py, &arguments)?;
|
||||
let host_token_provider = retained.azure_ad_token_provider.is_some();
|
||||
self.retained = Some(retained);
|
||||
Ok(OcrHostResult::Request(Ok(OcrProjectedRequest {
|
||||
request: Box::new(native),
|
||||
intercepts_requests: true,
|
||||
host_token_provider,
|
||||
})))
|
||||
}
|
||||
|
||||
fn acquire_azure_ad_token(&self, py: Python<'_>) -> PyResult<ResolvedCredential> {
|
||||
self.projected()?
|
||||
self.retained()?
|
||||
.azure_ad_token_provider
|
||||
.as_ref()
|
||||
.ok_or_else(missing_state)?
|
||||
|
|
@ -91,34 +116,21 @@ impl PythonOcrHost {
|
|||
fn during_call(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
mut request: OcrDuringCallRequest,
|
||||
request: OcrDuringCallRequest,
|
||||
) -> PyResult<OcrDuringCallRequest> {
|
||||
let projected = self.projected()?;
|
||||
let pre_call = projected.pre_call.as_ref().ok_or_else(missing_state)?;
|
||||
let retained = self.retained()?;
|
||||
let logger = self.state.logger()?;
|
||||
logger.update_ocr(
|
||||
callbacks::update_logging(
|
||||
py,
|
||||
logger,
|
||||
&self.state.kwargs,
|
||||
pre_call,
|
||||
&projected.secret_fields,
|
||||
&request.url,
|
||||
&request,
|
||||
&retained.secret_fields,
|
||||
)?;
|
||||
let body = to_py(py, &request.body)?
|
||||
.into_bound(py)
|
||||
.cast_into::<PyDict>()?;
|
||||
let headers = PyDict::new(py);
|
||||
for (name, value) in &request.headers {
|
||||
headers.set_item(name, value)?;
|
||||
}
|
||||
logger.pre_ocr(py, request.api_key.as_deref(), &body, &headers, &request.url)?;
|
||||
request.body = from_py(&body)?;
|
||||
request.headers = headers
|
||||
.iter()
|
||||
.map(|(name, value)| Ok((name.extract::<String>()?, value.extract::<String>()?)))
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
let projected = self.projected_mut()?;
|
||||
projected.body = Some(body.unbind());
|
||||
projected.headers = Some(headers.unbind());
|
||||
let payload = PythonPayload::from_request(py, &request)?;
|
||||
callbacks::pre_call(py, &self.state, &request, &payload)?;
|
||||
let request = payload.write_back(py, request)?;
|
||||
self.retained_mut()?.payload = Some(payload);
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
|
|
@ -127,26 +139,23 @@ impl PythonOcrHost {
|
|||
py: Python<'_>,
|
||||
request: OcrPostCallRequest,
|
||||
) -> PyResult<OcrPostCallRequest> {
|
||||
let projected = self.projected()?;
|
||||
self.state.logger()?.post_ocr(
|
||||
py,
|
||||
&request.original_response,
|
||||
projected.body.as_ref(),
|
||||
projected.headers.as_ref(),
|
||||
)?;
|
||||
let payload = self
|
||||
.retained()?
|
||||
.payload
|
||||
.as_ref()
|
||||
.ok_or_else(missing_state)?;
|
||||
callbacks::post_call(py, &self.state, &request.original_response, payload)?;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
fn map_failure(&mut self, py: Python<'_>, error: litellm_core::ocr::Error) -> PyResult<()> {
|
||||
if self.state.end.is_none() {
|
||||
self.state.end = Some(now(py)?);
|
||||
}
|
||||
let (model, provider) = match &self.projected {
|
||||
Some(projected) => (projected.model.as_str(), projected.provider),
|
||||
self.state.finish(py)?;
|
||||
let (model, provider) = match &self.retained {
|
||||
Some(retained) => (retained.model.as_str(), retained.provider),
|
||||
None => ("", ""),
|
||||
};
|
||||
let mapped = match self.state.error.take() {
|
||||
Some(host_error) if self.projected.as_ref().is_some_and(|host| host.reader_failed) => {
|
||||
Some(host_error) if matches!(error, litellm_core::ocr::Error::HostDocumentRead) => {
|
||||
PyErr::from_value(host_error.into_bound(py).into_any())
|
||||
}
|
||||
Some(host_error) => errors::public_host_exception(py, &host_error, model, provider)?,
|
||||
|
|
@ -168,10 +177,8 @@ impl PythonRoute for PythonOcrHost {
|
|||
&mut self.state
|
||||
}
|
||||
|
||||
fn classify(operation: &OcrHostOperation) -> OperationClass {
|
||||
operation
|
||||
.phase()
|
||||
.map_or(OperationClass::Route, OperationClass::Phase)
|
||||
fn phase(operation: &OcrHostOperation) -> Option<HostPhase> {
|
||||
operation.phase()
|
||||
}
|
||||
|
||||
fn lifecycle_result() -> OcrHostResult {
|
||||
|
|
@ -190,18 +197,24 @@ impl PythonRoute for PythonOcrHost {
|
|||
Ok(match operation {
|
||||
OcrHostOperation::ProjectRequest => self.project(py)?,
|
||||
OcrHostOperation::ReadDocument => {
|
||||
let reader = self.projected_mut()?.reader.take().ok_or_else(missing_state)?;
|
||||
let result = reader.read(py);
|
||||
self.projected_mut()?.reader_failed = result.is_err();
|
||||
OcrHostResult::Document(Ok(result?))
|
||||
let reader = self
|
||||
.retained_mut()?
|
||||
.reader
|
||||
.take()
|
||||
.ok_or_else(missing_state)?;
|
||||
match reader.read(py) {
|
||||
Ok(content) => OcrHostResult::Document(Ok(content)),
|
||||
Err(error) if error.is_instance_of::<pyo3::exceptions::PyException>(py) => {
|
||||
self.state.retain_first_error(py, error, false);
|
||||
OcrHostResult::Document(Err(litellm_core::ocr::Error::HostDocumentRead))
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
OcrHostOperation::AcquireAzureAdToken => {
|
||||
OcrHostResult::AzureAdToken(Ok(self.acquire_azure_ad_token(py)?))
|
||||
}
|
||||
OcrHostOperation::PreCall(request) => {
|
||||
self.projected_mut()?.pre_call = Some((&request).into());
|
||||
OcrHostResult::PreCall(Ok(request))
|
||||
}
|
||||
OcrHostOperation::PreCall(request) => OcrHostResult::PreCall(Ok(request)),
|
||||
OcrHostOperation::DuringCall(request) => {
|
||||
OcrHostResult::DuringCall(Ok(self.during_call(py, request)?))
|
||||
}
|
||||
|
|
@ -209,7 +222,7 @@ impl PythonRoute for PythonOcrHost {
|
|||
OcrHostResult::PostCall(Ok(self.post_call(py, request)?))
|
||||
}
|
||||
OcrHostOperation::ConstructResponse(response) => {
|
||||
self.state.end = Some(now(py)?);
|
||||
self.state.finish(py)?;
|
||||
self.state.response = Some(callbacks::response(py, response.as_ref())?);
|
||||
OcrHostResult::Lifecycle(Ok(()))
|
||||
}
|
||||
|
|
@ -224,28 +237,22 @@ impl PythonRoute for PythonOcrHost {
|
|||
}
|
||||
|
||||
fn cleanup(&mut self) {
|
||||
self.projected = None;
|
||||
self.retained = None;
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &pyo3::gc::PyVisit<'_>) -> Result<(), pyo3::gc::PyTraverseError> {
|
||||
let Some(projected) = &self.projected else {
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
let Some(retained) = &self.retained else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(provider) = &projected.azure_ad_token_provider {
|
||||
if let Some(provider) = &retained.azure_ad_token_provider {
|
||||
provider.traverse(visit)?;
|
||||
}
|
||||
if let Some(reader) = &projected.reader {
|
||||
if let Some(reader) = &retained.reader {
|
||||
reader.traverse(visit)?;
|
||||
}
|
||||
visit.call(&projected.body)?;
|
||||
visit.call(&projected.headers)
|
||||
}
|
||||
}
|
||||
|
||||
struct BridgeOcrHooks;
|
||||
|
||||
impl litellm_core::ocr::hooks::OcrHooks for BridgeOcrHooks {
|
||||
fn intercepts_requests(&self) -> bool {
|
||||
true
|
||||
if let Some(payload) = &retained.payload {
|
||||
payload.traverse(visit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,13 +62,16 @@ fn call(
|
|||
..OcrAdmission::all()
|
||||
},
|
||||
))?;
|
||||
let host = PythonOcrHost::new(PythonCallState::new(
|
||||
py,
|
||||
args.unbind(),
|
||||
kwargs.copy()?.unbind(),
|
||||
asynchronous,
|
||||
signature.name,
|
||||
)?);
|
||||
let host = PythonOcrHost::new(
|
||||
PythonCallState::new(
|
||||
py,
|
||||
args.unbind(),
|
||||
kwargs.copy()?.unbind(),
|
||||
asynchronous,
|
||||
signature.name,
|
||||
)?,
|
||||
signature,
|
||||
);
|
||||
run_call(py, call, host)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,54 +10,38 @@ use litellm_core::ocr::{
|
|||
};
|
||||
use litellm_python_interop::from_py_preserving_errors as from_py;
|
||||
|
||||
use super::document::FileDocumentInput;
|
||||
use super::errors::to_pyerr as ocr_error_to_pyerr;
|
||||
use super::document::{FileDocumentInput, PythonFileReader};
|
||||
use crate::auth::PythonTokenProvider;
|
||||
use super::host::OcrRetained;
|
||||
use crate::lifecycle::BoundArguments;
|
||||
use crate::marshal::BoundRouteInputs;
|
||||
use crate::marshal::{BoundRouteInputs, Projection};
|
||||
|
||||
/// Positional parameters of `ocr()` that are never projected into
|
||||
/// `optional_params`.
|
||||
const BOUND_FIELDS: &[&str] = &["model", "document", "timeout", "input_sources"];
|
||||
|
||||
pub(super) struct ProjectedOcrCall {
|
||||
pub request: LiteLLMOcrRequest,
|
||||
pub azure_ad_token_provider: Option<PythonTokenProvider>,
|
||||
pub secret_fields: Vec<&'static str>,
|
||||
pub reader: Option<PythonFileReader>,
|
||||
}
|
||||
|
||||
enum ProjectedDocument {
|
||||
File(FileDocumentInput),
|
||||
Url(serde_json::Value),
|
||||
}
|
||||
|
||||
impl ProjectedDocument {
|
||||
fn into_native(self) -> Result<FileDocumentInput, litellm_core::ocr::Error> {
|
||||
match self {
|
||||
Self::File(file) => Ok(file),
|
||||
Self::Url(value) => Ok(FileDocumentInput {
|
||||
input: OcrDocument::try_from(value)?.into(),
|
||||
reader: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn project_document(document: &Bound<'_, PyAny>) -> PyResult<ProjectedDocument> {
|
||||
fn project_document(
|
||||
document: &Bound<'_, PyAny>,
|
||||
) -> PyResult<Result<FileDocumentInput, litellm_core::ocr::Error>> {
|
||||
let kind: String = document.get_item("type")?.extract()?;
|
||||
if kind != "file" {
|
||||
return Ok(ProjectedDocument::Url(from_py(document)?));
|
||||
let value: serde_json::Value = from_py(document)?;
|
||||
return Ok(
|
||||
OcrDocument::try_from(value).map(|document| FileDocumentInput {
|
||||
input: document.into(),
|
||||
reader: None,
|
||||
}),
|
||||
);
|
||||
}
|
||||
document.extract().map(ProjectedDocument::File)
|
||||
document.extract().map(Ok)
|
||||
}
|
||||
|
||||
/// Pure core assembly; every failure here is a typed `ocr::Error`.
|
||||
fn build_request(
|
||||
inputs: BoundRouteInputs,
|
||||
document: ProjectedDocument,
|
||||
) -> Result<ProjectedOcrCall, litellm_core::ocr::Error> {
|
||||
let document = document.into_native()?;
|
||||
document: Result<FileDocumentInput, litellm_core::ocr::Error>,
|
||||
) -> Result<Projection<LiteLLMOcrRequest, OcrRetained>, litellm_core::ocr::Error> {
|
||||
let document = document?;
|
||||
let BoundRouteInputs {
|
||||
model,
|
||||
custom_llm_provider,
|
||||
|
|
@ -83,18 +67,23 @@ fn build_request(
|
|||
input_sources,
|
||||
},
|
||||
)?;
|
||||
Ok(ProjectedOcrCall {
|
||||
request,
|
||||
azure_ad_token_provider,
|
||||
secret_fields,
|
||||
reader: document.reader,
|
||||
Ok(Projection {
|
||||
retained: OcrRetained {
|
||||
model: request.model.clone(),
|
||||
provider: request.provider_name(),
|
||||
azure_ad_token_provider,
|
||||
secret_fields,
|
||||
reader: document.reader,
|
||||
payload: None,
|
||||
},
|
||||
native: request,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn project(
|
||||
py: Python<'_>,
|
||||
arguments: &BoundArguments<'_>,
|
||||
) -> PyResult<ProjectedOcrCall> {
|
||||
) -> PyResult<Projection<LiteLLMOcrRequest, OcrRetained>> {
|
||||
let model: String = arguments.extract("model")?;
|
||||
let custom_llm_provider: Option<String> = arguments.optional("custom_llm_provider")?;
|
||||
let document = project_document(&arguments.required("document")?)?;
|
||||
|
|
@ -142,7 +131,7 @@ mod tests {
|
|||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
project_document(&file).unwrap().into_native().unwrap().input,
|
||||
project_document(&file).unwrap().unwrap().input,
|
||||
litellm_core::ocr::OcrDocumentInput::Bytes { bytes, mime_type, .. }
|
||||
if bytes == b"%PDF-1.4"[..] && mime_type.as_deref() == Some("application/pdf")
|
||||
));
|
||||
|
|
@ -155,7 +144,7 @@ mod tests {
|
|||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
project_document(&original).unwrap().into_native().unwrap().input,
|
||||
project_document(&original).unwrap().unwrap().input,
|
||||
litellm_core::ocr::OcrDocumentInput::Document(OcrDocument::DocumentUrl { document_url, .. })
|
||||
if document_url == "https://example.com/a.pdf"
|
||||
));
|
||||
|
|
@ -169,7 +158,7 @@ mod tests {
|
|||
let document = py
|
||||
.eval(c"{'type': 'mystery', 'mystery': 'x'}", None, None)
|
||||
.unwrap();
|
||||
let error = project_document(&document).unwrap().into_native().err().unwrap();
|
||||
let error = project_document(&document).unwrap().err().unwrap();
|
||||
assert!(error.to_string().contains("document"));
|
||||
});
|
||||
}
|
||||
|
|
@ -181,14 +170,16 @@ mod tests {
|
|||
let missing = py.eval(c"{}", None, None).unwrap();
|
||||
assert!(
|
||||
project_document(&missing)
|
||||
.err().unwrap()
|
||||
.err()
|
||||
.unwrap()
|
||||
.is_instance_of::<PyKeyError>(py)
|
||||
);
|
||||
|
||||
let non_string = py.eval(c"{'type': 1}", None, None).unwrap();
|
||||
assert!(
|
||||
project_document(&non_string)
|
||||
.err().unwrap()
|
||||
.err()
|
||||
.unwrap()
|
||||
.is_instance_of::<PyTypeError>(py)
|
||||
);
|
||||
|
||||
|
|
@ -202,8 +193,9 @@ class Document:
|
|||
document = Document()
|
||||
",
|
||||
);
|
||||
let error =
|
||||
project_document(&locals.get_item("document").unwrap().unwrap()).err().unwrap();
|
||||
let error = project_document(&locals.get_item("document").unwrap().unwrap())
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
|
|
@ -232,8 +224,11 @@ document = Document()
|
|||
",
|
||||
);
|
||||
let document = locals.get_item("document").unwrap().unwrap();
|
||||
let projected = project_document(&document).unwrap().into_native().unwrap();
|
||||
assert!(matches!(projected.input, litellm_core::ocr::OcrDocumentInput::Bytes { .. }));
|
||||
let projected = project_document(&document).unwrap().unwrap();
|
||||
assert!(matches!(
|
||||
projected.input,
|
||||
litellm_core::ocr::OcrDocumentInput::Bytes { .. }
|
||||
));
|
||||
let reads: Vec<String> = document.getattr("reads").unwrap().extract().unwrap();
|
||||
assert_eq!(reads, ["type", "mime_type", "file"]);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -573,7 +573,6 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.streaming_chunks: list[Any] = [] # for generating complete stream response
|
||||
self.sync_streaming_chunks: list[Any] = [] # for generating complete stream response
|
||||
self.log_raw_request_response = log_raw_request_response
|
||||
self._native_callback_fast_path: bool = False
|
||||
|
||||
# Initialize dynamic callbacks
|
||||
self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: pr
|
|||
)
|
||||
return await fallback(*args, **kwargs)
|
||||
|
||||
|
||||
def _decline_types() -> tuple[type[BaseException], ...]:
|
||||
exception_types: Final = native_exception_types()
|
||||
return (exception_types[0],) if exception_types is not None else ()
|
||||
|
|
|
|||
|
|
@ -3307,7 +3307,10 @@ class ProxyBaseLLMRequestProcessing:
|
|||
pending: Final = getattr(logging_obj, "_native_pending_logging", None)
|
||||
if pending is not None:
|
||||
logging_obj._native_pending_logging = None # rebind-ok: consume the native OCR release signal once
|
||||
pending.release(not exception_raised)
|
||||
if exception_raised:
|
||||
pending.close()
|
||||
else:
|
||||
pending()
|
||||
_enqueue_fn: Final = getattr(logging_obj, "_enqueue_deferred_logging", None)
|
||||
if _enqueue_fn is None:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -29,10 +29,12 @@ def _build_document_from_upload(
|
|||
filename: str | None,
|
||||
content_type: str | None,
|
||||
) -> dict[str, str]:
|
||||
mime_type: Final = content_type.split(";")[0].strip() if content_type else None
|
||||
if not mime_type or mime_type == "application/octet-stream":
|
||||
if filename:
|
||||
mime_type = get_mime_type(filename)
|
||||
supplied_mime: Final = content_type.split(";")[0].strip() if content_type else None
|
||||
mime_type: Final = (
|
||||
get_mime_type(filename)
|
||||
if filename and (not supplied_mime or supplied_mime == "application/octet-stream")
|
||||
else supplied_mime
|
||||
)
|
||||
|
||||
return convert_file_document_to_url_document(
|
||||
{ # mutable-ok: OCR file document TypedDict handed to the converter
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import datetime
|
||||
from asyncio import Future
|
||||
from collections.abc import Coroutine, Mapping
|
||||
from typing import Never, final
|
||||
|
|
@ -115,11 +116,20 @@ class TokenCounter:
|
|||
def acount_request(self, body: bytes) -> Future[dict[str, object]]: ...
|
||||
|
||||
def gil_stats() -> dict[str, int]: ...
|
||||
def _debug_setup(
|
||||
call_type: str,
|
||||
args: tuple[object, ...],
|
||||
kwargs: dict[str, object],
|
||||
start: datetime.datetime,
|
||||
asynchronous: bool,
|
||||
) -> tuple[object, dict[str, object]]: ...
|
||||
|
||||
__all__ = [
|
||||
"ResponsesWebSocketConnection",
|
||||
"RustBridgeDeclined",
|
||||
"RustUpstreamError",
|
||||
"TokenCounter",
|
||||
"_debug_setup",
|
||||
"achat_completions",
|
||||
"amessages",
|
||||
"aocr",
|
||||
|
|
|
|||
|
|
@ -47,3 +47,8 @@ def native_exception_types() -> tuple[type[BaseException], type[BaseException]]
|
|||
if not isinstance(declined, type) or not isinstance(upstream, type):
|
||||
return None
|
||||
return declined, upstream
|
||||
|
||||
|
||||
def native_decline_types() -> tuple[type[BaseException], ...]:
|
||||
exceptions: Final = native_exception_types()
|
||||
return (exceptions[0],) if exceptions is not None else ()
|
||||
|
|
|
|||
913
litellm/rust_bridge/leaves.py
Normal file
913
litellm/rust_bridge/leaves.py
Normal file
|
|
@ -0,0 +1,913 @@
|
|||
"""Leaf helpers invoked by the native callback dispatcher.
|
||||
|
||||
Rust selects every target, delivery and sequence. Nothing here chooses a callback,
|
||||
reads a registry, or decides whether an event fires. Each function performs one
|
||||
integration-specific or interop-specific action on values Rust hands it.
|
||||
|
||||
Labeled leaf helpers pending native migration (see rust-callback-inventory.md):
|
||||
`prepare_success_logging`, `prepare_failure_logging`, `dispatch_named_success`,
|
||||
`dispatch_named_failure`, `dispatch_callable`. They are deleted when cost and
|
||||
payload construction move into core and when each string integration becomes a
|
||||
CustomLogger.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import traceback
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
Literal,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
cast, # noqa: TID251 # narrows the untyped legacy Logging and CustomLogger surfaces once
|
||||
)
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
TerminalFamily: TypeAlias = Literal["sync_success", "async_success", "sync_failure", "async_failure"]
|
||||
Family: TypeAlias = Literal["request", "deployment", TerminalFamily]
|
||||
Details: TypeAlias = dict[
|
||||
str, object
|
||||
] # mutable-ok: model_call_details is the shared mutable envelope callbacks write to
|
||||
Timestamp: TypeAlias = datetime.datetime
|
||||
LegacyCall: TypeAlias = Callable[..., object]
|
||||
|
||||
|
||||
class LoggerView(Protocol):
|
||||
model: str | None
|
||||
messages: object
|
||||
call_type: str
|
||||
start_time: Timestamp
|
||||
litellm_call_id: str
|
||||
completion_start_time: Timestamp | None
|
||||
model_call_details: Details
|
||||
log_raw_request_response: bool
|
||||
standard_callback_dynamic_params: StandardCallbackDynamicParams
|
||||
standard_built_in_tools_params: object
|
||||
|
||||
def record_api_call_start_time(self) -> None: ...
|
||||
|
||||
def record_post_call(
|
||||
self, original_response: object, input: object, api_key: object, additional_args: Details
|
||||
) -> None: ...
|
||||
|
||||
def should_run_callback(self, callback: object, litellm_params: Details, event_hook: str) -> bool: ...
|
||||
|
||||
def _pre_call(self, input: str, api_key: str | None, model: str | None, additional_args: Details) -> None: ...
|
||||
|
||||
def _print_llm_call_debugging_log(
|
||||
self, api_base: str, headers: Mapping[str, str], additional_args: Details
|
||||
) -> None: ...
|
||||
|
||||
def _get_request_curl_command(
|
||||
self, api_base: str, headers: Mapping[str, str] | None, additional_args: Details, data: object
|
||||
) -> str: ...
|
||||
|
||||
def _get_masked_api_base(self, api_base: str) -> str: ...
|
||||
|
||||
def _get_raw_request_body(self, data: object) -> Details: ...
|
||||
|
||||
def _get_masked_headers(self, headers: Mapping[str, str]) -> Details: ...
|
||||
|
||||
def _response_cost_calculator(self, result: object) -> float | None: ...
|
||||
|
||||
def _build_standard_logging_payload(
|
||||
self, init_response_obj: object, start_time: Timestamp, end_time: Timestamp
|
||||
) -> object: ...
|
||||
|
||||
def _handle_callback_failure(self, callback: object) -> None: ...
|
||||
|
||||
|
||||
class IntegrationView(Protocol):
|
||||
def log_pre_api_call(self, model: str | None, messages: object, kwargs: Details) -> None: ...
|
||||
|
||||
def log_post_api_call(
|
||||
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp | None
|
||||
) -> None: ...
|
||||
|
||||
def log_success_event(
|
||||
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp
|
||||
) -> None: ...
|
||||
|
||||
def log_failure_event(
|
||||
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp
|
||||
) -> None: ...
|
||||
|
||||
def async_log_success_event(
|
||||
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp
|
||||
) -> Awaitable[None]: ...
|
||||
|
||||
def async_log_failure_event(
|
||||
self, kwargs: Details, response_obj: object, start_time: Timestamp, end_time: Timestamp
|
||||
) -> Awaitable[None]: ...
|
||||
|
||||
def logging_hook(self, kwargs: Details, result: object, call_type: str) -> tuple[Details, object]: ...
|
||||
|
||||
def async_logging_hook(
|
||||
self, kwargs: Details, result: object, call_type: str
|
||||
) -> Awaitable[tuple[Details, object]]: ...
|
||||
|
||||
def redact_standard_logging_payload_from_model_call_details(self, model_call_details: Details) -> Details: ...
|
||||
|
||||
def log_input_event(
|
||||
self, model: str | None, messages: object, kwargs: Details, print_verbose: LegacyCall, callback_func: LegacyCall
|
||||
) -> None: ...
|
||||
|
||||
def log_event(
|
||||
self,
|
||||
kwargs: Details,
|
||||
response_obj: object,
|
||||
start_time: Timestamp,
|
||||
end_time: Timestamp,
|
||||
print_verbose: LegacyCall,
|
||||
callback_func: LegacyCall,
|
||||
) -> None: ...
|
||||
|
||||
def async_log_event(
|
||||
self,
|
||||
kwargs: Details,
|
||||
response_obj: object,
|
||||
start_time: Timestamp,
|
||||
end_time: Timestamp,
|
||||
print_verbose: LegacyCall,
|
||||
callback_func: LegacyCall,
|
||||
) -> Awaitable[None]: ...
|
||||
|
||||
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _details_of(value: object) -> Details:
|
||||
return cast(Details, value) # cast-ok: legacy model_call_details nesting is untyped and shared by callbacks
|
||||
|
||||
|
||||
def _call_of(value: object) -> LegacyCall:
|
||||
return cast(LegacyCall, value) # cast-ok: legacy integration entry points are untyped
|
||||
|
||||
|
||||
def _attribute(target: object, name: str) -> object:
|
||||
value: Final[object] = getattr(target, name) # pyright: ignore[reportAny] # legacy attributes are untyped by definition
|
||||
return value
|
||||
|
||||
|
||||
def _seed(details: Details, key: str) -> Details:
|
||||
existing: Final[object] = details.get(key)
|
||||
if isinstance(existing, dict):
|
||||
return _details_of(existing) # pyright: ignore[reportUnknownArgumentType] # isinstance yields dict[Unknown, Unknown]
|
||||
seeded: Final[Details] = {} # mutable-ok: legacy envelope seed shared with callbacks
|
||||
details[key] = seeded # rebind-ok: seeding a nested dict on the shared envelope is the legacy contract
|
||||
return seeded
|
||||
|
||||
|
||||
def _logger(logger: Logging) -> LoggerView:
|
||||
return cast(LoggerView, logger) # cast-ok: legacy Logging is untyped; this protocol names the attributes we read
|
||||
|
||||
|
||||
def _integration(callback: CustomLogger) -> IntegrationView:
|
||||
return cast(IntegrationView, callback) # cast-ok: legacy CustomLogger methods are untyped
|
||||
|
||||
|
||||
def _singleton(name: str) -> object:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
return getattr(litellm_logging, name, None) # pyright: ignore[reportAny] # legacy module globals are rebound at runtime
|
||||
|
||||
|
||||
def _print_verbose() -> LegacyCall:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
return cast(LegacyCall, litellm_logging.print_verbose) # cast-ok: legacy debug printer is untyped
|
||||
|
||||
|
||||
def _redact_string(value: str) -> str:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
redact: Final = _call_of(litellm_logging._redact_string) # pyright: ignore[reportPrivateUsage] # private legacy helper
|
||||
return str(redact(value))
|
||||
|
||||
|
||||
def _redact_result(details: Details, result: object) -> object:
|
||||
from litellm.litellm_core_utils import redact_messages
|
||||
|
||||
redact: Final = cast(LegacyCall, redact_messages.redact_message_input_output_from_logging) # cast-ok: legacy helper
|
||||
return redact(model_call_details=details, result=result)
|
||||
|
||||
|
||||
def record_pre_call(
|
||||
logger: Logging,
|
||||
*,
|
||||
api_key: str | None,
|
||||
body: Details,
|
||||
headers: Mapping[str, str],
|
||||
url: str,
|
||||
) -> None:
|
||||
view: Final = _logger(logger)
|
||||
additional_args: Final[Details] = { # mutable-ok: legacy additional_args is rebound by callbacks in place
|
||||
"complete_input_dict": body,
|
||||
"headers": headers,
|
||||
"api_base": url,
|
||||
}
|
||||
view._pre_call(input="OCR document processing", api_key=api_key, model=None, additional_args=additional_args) # pyright: ignore[reportPrivateUsage] # legacy state writer
|
||||
view._print_llm_call_debugging_log(api_base=url, headers=headers, additional_args=additional_args) # pyright: ignore[reportPrivateUsage] # legacy debug output
|
||||
_capture_raw_request(view, additional_args)
|
||||
_run_logger_fn(logger)
|
||||
view.record_api_call_start_time()
|
||||
|
||||
|
||||
def _capture_raw_request(view: LoggerView, additional_args: Details) -> None:
|
||||
import litellm
|
||||
from litellm.types.utils import RawRequestTypedDict
|
||||
|
||||
if not (view.log_raw_request_response or litellm.log_raw_request_response):
|
||||
return
|
||||
details: Final = view.model_call_details
|
||||
metadata: Final = _seed(_seed(details, "litellm_params"), "metadata")
|
||||
if litellm.turn_off_message_logging:
|
||||
metadata["raw_request"] = "redacted by litellm. 'litellm.turn_off_message_logging=True'"
|
||||
return
|
||||
api_base: Final = str(additional_args.get("api_base") or "")
|
||||
headers: Final = cast(Mapping[str, str], additional_args.get("headers") or _EMPTY) # cast-ok: legacy nested dict
|
||||
body: Final = additional_args.get("complete_input_dict", _EMPTY)
|
||||
try:
|
||||
curl: Final = view._get_request_curl_command( # pyright: ignore[reportPrivateUsage] # legacy debug formatter
|
||||
api_base=api_base, headers=headers, additional_args=additional_args, data=body
|
||||
)
|
||||
metadata["raw_request"] = _redact_string(str(curl))
|
||||
details["raw_request_typed_dict"] = RawRequestTypedDict(
|
||||
raw_request_api_base=view._get_masked_api_base(api_base), # pyright: ignore[reportPrivateUsage] # legacy masking
|
||||
raw_request_body=view._get_raw_request_body(body), # pyright: ignore[reportPrivateUsage] # legacy masking
|
||||
raw_request_headers=view._get_masked_headers(headers), # pyright: ignore[reportPrivateUsage] # legacy masking
|
||||
error=None,
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # raw-request capture is best effort by contract
|
||||
details["raw_request_typed_dict"] = RawRequestTypedDict(error=str(error))
|
||||
metadata["raw_request"] = _redact_string(f"Unable to Log raw request: {error}")
|
||||
|
||||
|
||||
def _run_logger_fn(logger: Logging) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
logger_fn: Final = getattr(logger, "logger_fn", None)
|
||||
if not callable(logger_fn):
|
||||
return
|
||||
try:
|
||||
_call_of(logger_fn)(_logger(logger).model_call_details)
|
||||
except Exception as error: # noqa: BLE001 # user logger_fn failures never fail the request
|
||||
verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", error)
|
||||
|
||||
|
||||
def record_post_call(logger: Logging, *, original_response: object, body: Details, headers: Mapping[str, str]) -> None:
|
||||
view: Final = _logger(logger)
|
||||
serialized: Final = (
|
||||
json.dumps(original_response, default=str) if isinstance(original_response, dict) else original_response
|
||||
)
|
||||
view.record_post_call(
|
||||
original_response=serialized,
|
||||
input=None,
|
||||
api_key=None,
|
||||
additional_args={"complete_input_dict": body, "headers": headers}, # mutable-ok: rebound in place by callbacks
|
||||
)
|
||||
_run_logger_fn(logger)
|
||||
_redact_result(view.model_call_details, serialized)
|
||||
|
||||
|
||||
def log_pre_api_call(logger: Logging, callback: CustomLogger) -> None:
|
||||
view: Final = _logger(logger)
|
||||
_integration(callback).log_pre_api_call(model=view.model, messages=view.messages, kwargs=view.model_call_details)
|
||||
|
||||
|
||||
def log_post_api_call(logger: Logging, callback: CustomLogger) -> None:
|
||||
view: Final = _logger(logger)
|
||||
_integration(callback).log_post_api_call(
|
||||
kwargs=view.model_call_details, response_obj=None, start_time=view.start_time, end_time=None
|
||||
)
|
||||
|
||||
|
||||
def dispatch_named_request(logger: Logging, name: str, event: Literal["pre_api_call", "post_api_call"]) -> None:
|
||||
from litellm.integrations.supabase import Supabase
|
||||
|
||||
view: Final = _logger(logger)
|
||||
details: Final = view.model_call_details
|
||||
match name:
|
||||
case "supabase" if event == "pre_api_call" and isinstance(client := _singleton("supabaseClient"), Supabase):
|
||||
client.input_log_event(
|
||||
model=view.model,
|
||||
messages=view.messages,
|
||||
end_user=details.get("user", "default"),
|
||||
litellm_call_id=details["litellm_call_id"],
|
||||
print_verbose=_print_verbose(),
|
||||
)
|
||||
case "sentry" if callable(add_breadcrumb := _singleton("add_breadcrumb")):
|
||||
add_breadcrumb(category="litellm.llm_call", message=f"Model Call Details {event}: {details}", level="info")
|
||||
case _:
|
||||
return
|
||||
|
||||
|
||||
def dispatch_callable_request(logger: Logging, callback: LegacyCall) -> None:
|
||||
custom: Final = _singleton("customLogger")
|
||||
if not isinstance(custom, CustomLogger):
|
||||
return
|
||||
view: Final = _logger(logger)
|
||||
_integration(custom).log_input_event(
|
||||
model=view.model,
|
||||
messages=view.messages,
|
||||
kwargs=view.model_call_details,
|
||||
print_verbose=_print_verbose(),
|
||||
callback_func=callback,
|
||||
)
|
||||
|
||||
|
||||
def _typed_call_type(call_type: str) -> object:
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
try:
|
||||
return CallTypes(call_type)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _awaitable(value: object) -> Awaitable[object]:
|
||||
return cast(Awaitable[object], value) # cast-ok: legacy async hooks return coroutines
|
||||
|
||||
|
||||
def pre_call_deployment_hook(callback: CustomLogger, kwargs: Details, call_type: str) -> Awaitable[object]:
|
||||
hook: Final = _call_of(_attribute(callback, "async_pre_call_deployment_hook"))
|
||||
return _awaitable(hook(kwargs, _typed_call_type(call_type)))
|
||||
|
||||
|
||||
def post_call_success_deployment_hook(
|
||||
callback: CustomLogger, kwargs: Details, response: object, call_type: str
|
||||
) -> Awaitable[object]:
|
||||
hook: Final = _call_of(_attribute(callback, "async_post_call_success_deployment_hook"))
|
||||
return _awaitable(hook(kwargs, response, _typed_call_type(call_type)))
|
||||
|
||||
|
||||
def failure_deployment_hook_view(
|
||||
kwargs: Details, exception: BaseException
|
||||
) -> tuple[Mapping[str, object], BaseException, int | None]:
|
||||
from litellm import utils
|
||||
|
||||
raw_depth: Final = kwargs.get("fallback_depth")
|
||||
depth: Final = raw_depth if isinstance(raw_depth, int) else None
|
||||
safe_request: Final = MappingProxyType({key: value for key, value in kwargs.items() if key != "attempted_targets"})
|
||||
snapshot: Final = _call_of(utils._snapshot_exception_for_hook)(exception) # pyright: ignore[reportPrivateUsage] # legacy snapshot helper
|
||||
return safe_request, cast(BaseException, snapshot), depth # cast-ok: snapshot is a same-class copy of the exception
|
||||
|
||||
|
||||
def post_call_failure_deployment_hook(
|
||||
callback: CustomLogger,
|
||||
request: Mapping[str, object],
|
||||
exception: BaseException,
|
||||
call_type: str,
|
||||
fallback_depth: int | None,
|
||||
) -> Awaitable[object]:
|
||||
from litellm import utils
|
||||
|
||||
hook: Final = _call_of(_attribute(callback, "async_post_call_failure_deployment_hook"))
|
||||
accepts_depth: Final = _call_of(utils._accepts_fallback_depth_kwarg_for_class)(type(callback)) # pyright: ignore[reportPrivateUsage] # legacy signature probe
|
||||
typed: Final = _typed_call_type(call_type)
|
||||
if accepts_depth:
|
||||
return _awaitable(hook(request, exception, typed, fallback_depth=fallback_depth))
|
||||
return _awaitable(hook(request, exception, typed))
|
||||
|
||||
|
||||
def report_deployment_failure_hook_error(callback: object, error: BaseException) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.debug("async_post_call_failure_deployment_hook error in %s: %s", type(callback).__name__, error)
|
||||
|
||||
|
||||
def report_target_failure(logger: Logging, callback: object, family: Family, error: BaseException) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.error(
|
||||
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while %s logging with %s: %s",
|
||||
family,
|
||||
callback,
|
||||
"".join(traceback.format_exception(error)),
|
||||
)
|
||||
capture: Final = _singleton("capture_exception")
|
||||
if callable(capture) and family in ("request", "sync_success", "sync_failure"):
|
||||
capture(error)
|
||||
if family not in ("request", "deployment", "sync_failure"):
|
||||
_logger(logger)._handle_callback_failure(callback=callback) # pyright: ignore[reportPrivateUsage] # legacy prometheus counter
|
||||
|
||||
|
||||
def prepare_success_logging(logger: Logging, response: object, start_time: Timestamp, end_time: Timestamp) -> object:
|
||||
from litellm.litellm_core_utils.litellm_logging import emit_standard_logging_payload
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
view: Final = _logger(logger)
|
||||
details: Final = view.model_call_details
|
||||
if view.completion_start_time is None:
|
||||
view.completion_start_time = end_time
|
||||
details["completion_start_time"] = end_time
|
||||
details["log_event_type"] = "successful_api_call"
|
||||
details["end_time"] = end_time
|
||||
details["cache_hit"] = None
|
||||
hidden: Final = _details_of(getattr(response, "_hidden_params", None) or {}) # mutable-ok: legacy fallback envelope
|
||||
if hidden and isinstance(details.get("litellm_params"), dict):
|
||||
_seed(_seed(details, "litellm_params"), "metadata")["hidden_params"] = hidden
|
||||
existing: Final = details.get("response_cost")
|
||||
if "response_cost" in hidden:
|
||||
details["response_cost"] = hidden["response_cost"]
|
||||
elif existing is None or existing == 0:
|
||||
details["response_cost"] = view._response_cost_calculator(result=response) # pyright: ignore[reportPrivateUsage] # labeled leaf: native cost pending
|
||||
payload: Final = view._build_standard_logging_payload(response, start_time, end_time) # pyright: ignore[reportPrivateUsage] # labeled leaf: native payload pending
|
||||
details["standard_logging_object"] = payload
|
||||
if payload is not None:
|
||||
emit_standard_logging_payload(
|
||||
cast(StandardLoggingPayload, payload) # cast-ok: legacy builder returns the payload TypedDict
|
||||
) # cast-ok: legacy builder returns the payload TypedDict
|
||||
return _redact_result(details, response)
|
||||
|
||||
|
||||
def prepare_failure_logging(
|
||||
logger: Logging, exception: BaseException, start_time: Timestamp, end_time: Timestamp
|
||||
) -> str:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
formatted: Final = "".join(traceback.format_exception(exception))
|
||||
view: Final = _logger(logger)
|
||||
details: Final = view.model_call_details
|
||||
if details.get("exception") is exception and details.get("standard_logging_object") is not None:
|
||||
return formatted
|
||||
details["log_event_type"] = "failed_api_call"
|
||||
details["exception"] = exception
|
||||
details["traceback_exception"] = _redact_string(formatted)
|
||||
details["end_time"] = end_time
|
||||
details.setdefault("original_response", None)
|
||||
if details.get("combined_usage_object") is None:
|
||||
details["response_cost"] = 0
|
||||
headers: Final = getattr(exception, "headers", None)
|
||||
if isinstance(headers, Mapping):
|
||||
_seed(_seed(details, "litellm_params"), "metadata").update(
|
||||
_details_of(headers) # pyright: ignore[reportUnknownArgumentType] # exception headers carry no element types
|
||||
)
|
||||
build: Final = cast( # cast-ok: labeled leaf: native payload pending
|
||||
LegacyCall, litellm_logging.get_standard_logging_object_payload
|
||||
)
|
||||
details["standard_logging_object"] = build(
|
||||
kwargs=details,
|
||||
init_response_obj=_EMPTY,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=logger,
|
||||
status="failure",
|
||||
error_str=_redact_string(str(exception)),
|
||||
original_exception=exception,
|
||||
standard_built_in_tools_params=view.standard_built_in_tools_params,
|
||||
)
|
||||
return formatted
|
||||
|
||||
|
||||
_EVENT_HOOKS: Final[Mapping[TerminalFamily, str]] = MappingProxyType(
|
||||
{
|
||||
"sync_success": "success_handler",
|
||||
"async_success": "async_success_handler",
|
||||
"sync_failure": "failure_handler",
|
||||
"async_failure": "async_failure_handler",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def should_run_callback(logger: Logging, callback: object, family: TerminalFamily) -> bool:
|
||||
view: Final = _logger(logger)
|
||||
params: Final = _details_of(view.model_call_details.get("litellm_params") or _EMPTY)
|
||||
return view.should_run_callback(callback=callback, litellm_params=params, event_hook=_EVENT_HOOKS[family])
|
||||
|
||||
|
||||
def should_run_guardrail_hook(logger: Logging, callback: object) -> bool:
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
if not isinstance(callback, CustomGuardrail):
|
||||
return True
|
||||
decide: Final = cast(LegacyCall, callback.should_run_guardrail) # cast-ok: legacy guardrail method is untyped
|
||||
return decide(data=_logger(logger).model_call_details, event_type=GuardrailEventHooks.logging_only) is True
|
||||
|
||||
|
||||
def logging_hook(logger: Logging, callback: CustomLogger, result: object) -> object:
|
||||
view: Final = _logger(logger)
|
||||
details, replaced = _integration(callback).logging_hook(
|
||||
kwargs=view.model_call_details, result=result, call_type=view.call_type
|
||||
)
|
||||
view.model_call_details = details
|
||||
return replaced
|
||||
|
||||
|
||||
async def async_logging_hook(logger: Logging, callback: CustomLogger, result: object) -> object:
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils import redact_messages
|
||||
|
||||
view: Final = _logger(logger)
|
||||
redact: Final = cast( # cast-ok: legacy helper
|
||||
LegacyCall, redact_messages.redact_message_input_output_from_custom_logger
|
||||
)
|
||||
redacted: Final = (
|
||||
result
|
||||
if isinstance(callback, CustomGuardrail)
|
||||
else redact(result=result, litellm_logging_obj=logger, custom_logger=callback)
|
||||
)
|
||||
details, replaced = await _integration(callback).async_logging_hook(
|
||||
kwargs=view.model_call_details, result=redacted, call_type=view.call_type
|
||||
)
|
||||
view.model_call_details = details
|
||||
return replaced
|
||||
|
||||
|
||||
def is_sync_request(logger: Logging) -> bool:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LoggingClass
|
||||
|
||||
params: Final = _details_of(_logger(logger).model_call_details.get("litellm_params") or _EMPTY)
|
||||
decide: Final = _call_of(LoggingClass._is_sync_litellm_request) # pyright: ignore[reportPrivateUsage] # legacy predicate
|
||||
return decide(params) is True
|
||||
|
||||
|
||||
def mark_logged(logger: Logging, marker: str) -> None:
|
||||
_logger(logger).model_call_details[marker] = True
|
||||
|
||||
|
||||
def already_logged(logger: Logging, marker: str) -> bool:
|
||||
return _logger(logger).model_call_details.get(marker, False) is True
|
||||
|
||||
|
||||
def log_success_event(
|
||||
logger: Logging, callback: CustomLogger, response: object, start_time: Timestamp, end_time: Timestamp
|
||||
) -> None:
|
||||
_integration(callback).log_success_event(
|
||||
kwargs=_logger(logger).model_call_details, response_obj=response, start_time=start_time, end_time=end_time
|
||||
)
|
||||
|
||||
|
||||
def async_log_success_event(
|
||||
logger: Logging, callback: CustomLogger, response: object, start_time: Timestamp, end_time: Timestamp
|
||||
) -> Awaitable[None]:
|
||||
from litellm.litellm_core_utils import redact_messages
|
||||
|
||||
integration: Final = _integration(callback)
|
||||
details: Final = integration.redact_standard_logging_payload_from_model_call_details(
|
||||
model_call_details=_logger(logger).model_call_details
|
||||
)
|
||||
redact: Final = cast( # cast-ok: legacy helper
|
||||
Callable[..., Details], redact_messages.redact_streaming_responses_for_custom_logger
|
||||
)
|
||||
view: Final = redact(model_call_details=details, custom_logger=callback)
|
||||
return integration.async_log_success_event(
|
||||
kwargs=view, response_obj=response, start_time=start_time, end_time=end_time
|
||||
)
|
||||
|
||||
|
||||
def log_failure_event(logger: Logging, callback: CustomLogger, start_time: Timestamp, end_time: Timestamp) -> None:
|
||||
_integration(callback).log_failure_event(
|
||||
kwargs=_logger(logger).model_call_details, response_obj=None, start_time=start_time, end_time=end_time
|
||||
)
|
||||
|
||||
|
||||
def async_log_failure_event(
|
||||
logger: Logging, callback: CustomLogger, start_time: Timestamp, end_time: Timestamp
|
||||
) -> Awaitable[None]:
|
||||
return _integration(callback).async_log_failure_event(
|
||||
kwargs=_logger(logger).model_call_details, response_obj=None, start_time=start_time, end_time=end_time
|
||||
)
|
||||
|
||||
|
||||
def _custom_logger_singleton() -> IntegrationView:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
existing: Final = _singleton("customLogger")
|
||||
if isinstance(existing, CustomLogger):
|
||||
return _integration(existing)
|
||||
created: Final = CustomLogger()
|
||||
litellm_logging.customLogger = created # pyright: ignore[reportAttributeAccessIssue] # legacy module global
|
||||
return _integration(created)
|
||||
|
||||
|
||||
def dispatch_callable(
|
||||
logger: Logging,
|
||||
callback: LegacyCall,
|
||||
family: TerminalFamily,
|
||||
response: object,
|
||||
start_time: Timestamp,
|
||||
end_time: Timestamp,
|
||||
) -> Awaitable[None] | None:
|
||||
custom: Final = _custom_logger_singleton()
|
||||
details: Final = _logger(logger).model_call_details
|
||||
match family:
|
||||
case "sync_success" | "sync_failure":
|
||||
custom.log_event(
|
||||
kwargs=details,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=_print_verbose(),
|
||||
callback_func=callback,
|
||||
)
|
||||
return None
|
||||
case "async_success" | "async_failure":
|
||||
return custom.async_log_event(
|
||||
kwargs=details,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=_print_verbose(),
|
||||
callback_func=callback,
|
||||
)
|
||||
|
||||
|
||||
def dispatch_named_success(
|
||||
logger: Logging, name: str, response: object, start_time: Timestamp, end_time: Timestamp
|
||||
) -> Awaitable[None] | None:
|
||||
from litellm.integrations.athina import AthinaLogger
|
||||
from litellm.integrations.greenscale import GreenscaleLogger
|
||||
from litellm.integrations.helicone import HeliconeLogger
|
||||
from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger
|
||||
from litellm.integrations.lunary import LunaryLogger
|
||||
from litellm.integrations.openmeter import OpenMeterLogger
|
||||
from litellm.integrations.prompt_layer import PromptLayerLogger
|
||||
from litellm.integrations.s3 import S3Logger
|
||||
from litellm.integrations.supabase import Supabase
|
||||
from litellm.integrations.traceloop import TraceloopLogger
|
||||
from litellm.integrations.weights_biases import WeightsBiasesLogger
|
||||
|
||||
view: Final = _logger(logger)
|
||||
details: Final = view.model_call_details
|
||||
print_verbose: Final = _print_verbose()
|
||||
without_response: Final = { # mutable-ok: legacy integrations receive a private mutable copy
|
||||
key: value for key, value in details.items() if key != "original_response"
|
||||
}
|
||||
match name:
|
||||
case "promptlayer" if isinstance(promptlayer := _singleton("promptLayerLogger"), PromptLayerLogger):
|
||||
promptlayer.log_event(
|
||||
kwargs=details,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "wandb" if isinstance(wandb := _singleton("weightsBiasesLogger"), WeightsBiasesLogger):
|
||||
wandb.log_event(
|
||||
kwargs=details,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "athina" if isinstance(athina := _singleton("athinaLogger"), AthinaLogger):
|
||||
athina.log_event(
|
||||
kwargs=details,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "logfire" if isinstance(logfire := _singleton("logfireLogger"), LogfireLogger):
|
||||
logfire.log_event(
|
||||
kwargs=without_response,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
level=LogfireLevel.INFO,
|
||||
)
|
||||
case "greenscale" if isinstance(greenscale := _singleton("greenscaleLogger"), GreenscaleLogger):
|
||||
greenscale.log_event(
|
||||
kwargs=without_response,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "supabase" if isinstance(supabase := _singleton("supabaseClient"), Supabase):
|
||||
supabase.log_event(
|
||||
model=view.model,
|
||||
messages=view.messages,
|
||||
end_user=details.get("user", "default"),
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
litellm_call_id=details["litellm_call_id"],
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "lunary" if isinstance(lunary := _singleton("lunaryLogger"), LunaryLogger):
|
||||
lunary.log_event(
|
||||
kwargs=details,
|
||||
type="llm",
|
||||
event="end",
|
||||
model=view.model,
|
||||
input=details["input"],
|
||||
user_id=details.get("user", "default"),
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
run_id=view.litellm_call_id,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "helicone" if isinstance(helicone := _singleton("heliconeLogger"), HeliconeLogger):
|
||||
helicone.log_success(
|
||||
model=view.model,
|
||||
messages=view.messages,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
kwargs=details,
|
||||
)
|
||||
case "langfuse":
|
||||
_langfuse(
|
||||
logger, response=response, start_time=start_time, end_time=end_time, level=None, status_message=None
|
||||
)
|
||||
case "traceloop" if isinstance(traceloop := _singleton("traceloopLogger"), TraceloopLogger):
|
||||
traceloop.log_event(
|
||||
kwargs=details,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
user_id=details.get("user", None),
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "s3" if isinstance(s3 := _singleton("s3Logger"), S3Logger):
|
||||
s3.log_event(
|
||||
kwargs=details,
|
||||
response_obj=response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "openmeter" if isinstance(openmeter := _singleton("openMeterLogger"), OpenMeterLogger):
|
||||
return openmeter.async_log_success_event(
|
||||
kwargs=details, response_obj=response, start_time=start_time, end_time=end_time
|
||||
)
|
||||
case "dynamodb":
|
||||
return _dynamodb(details, response, start_time, end_time, print_verbose)
|
||||
case _:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _dynamodb(
|
||||
details: Details, response: object, start_time: Timestamp, end_time: Timestamp, print_verbose: LegacyCall
|
||||
) -> Awaitable[None]:
|
||||
from litellm.integrations.dynamodb import DyanmoDBLogger
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
existing: Final = _singleton("dynamoLogger")
|
||||
dynamo: Final = existing if isinstance(existing, DyanmoDBLogger) else DyanmoDBLogger()
|
||||
litellm_logging.dynamoLogger = dynamo # pyright: ignore[reportAttributeAccessIssue] # legacy module global
|
||||
return dynamo._async_log_event( # pyright: ignore[reportPrivateUsage] # legacy async entry point
|
||||
kwargs=details, response_obj=response, start_time=start_time, end_time=end_time, print_verbose=print_verbose
|
||||
)
|
||||
|
||||
|
||||
def _langfuse(
|
||||
logger: Logging,
|
||||
*,
|
||||
response: object,
|
||||
start_time: Timestamp,
|
||||
end_time: Timestamp,
|
||||
level: str | None,
|
||||
status_message: str | None,
|
||||
) -> None:
|
||||
from litellm.integrations.langfuse.langfuse import LangFuseLogger
|
||||
from litellm.integrations.langfuse.langfuse_handler import LangFuseHandler
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
view: Final = _logger(logger)
|
||||
kwargs: Final = { # mutable-ok: langfuse receives a private mutable copy
|
||||
key: value for key, value in view.model_call_details.items() if key != "original_response"
|
||||
}
|
||||
global_logger: Final = _singleton("langFuseLogger")
|
||||
handler: Final = LangFuseHandler.get_langfuse_logger_for_request(
|
||||
globalLangfuseLogger=global_logger if isinstance(global_logger, LangFuseLogger) else None,
|
||||
standard_callback_dynamic_params=view.standard_callback_dynamic_params,
|
||||
in_memory_dynamic_logger_cache=litellm_logging.in_memory_dynamic_logger_cache,
|
||||
)
|
||||
user: Final = kwargs.get("user")
|
||||
result: Final = handler.log_event_on_langfuse(
|
||||
kwargs=kwargs,
|
||||
response_obj=cast(ModelResponse, response), # cast-ok: OCR responses sit outside the legacy union
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
user_id=user if isinstance(user, str) else None,
|
||||
level="DEFAULT" if level is None else level,
|
||||
status_message=status_message,
|
||||
)
|
||||
trace_id: Final = result.get("trace_id")
|
||||
if isinstance(trace_id, str):
|
||||
litellm_logging.in_memory_trace_id_cache.set_cache(
|
||||
litellm_call_id=view.litellm_call_id, service_name="langfuse", trace_id=trace_id
|
||||
)
|
||||
|
||||
|
||||
def dispatch_named_failure(
|
||||
logger: Logging,
|
||||
name: str,
|
||||
exception: BaseException,
|
||||
formatted: str,
|
||||
start_time: Timestamp,
|
||||
end_time: Timestamp,
|
||||
) -> None:
|
||||
from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger
|
||||
from litellm.integrations.lunary import LunaryLogger
|
||||
from litellm.integrations.supabase import Supabase
|
||||
from litellm.integrations.traceloop import TraceloopLogger
|
||||
|
||||
view: Final = _logger(logger)
|
||||
details: Final = view.model_call_details
|
||||
print_verbose: Final = _print_verbose()
|
||||
match name:
|
||||
case "lunary" if isinstance(lunary := _singleton("lunaryLogger"), LunaryLogger):
|
||||
lunary.log_event(
|
||||
kwargs=details,
|
||||
type="llm",
|
||||
event="error",
|
||||
user_id=details.get("user", "default"),
|
||||
model=view.model,
|
||||
input=details["input"],
|
||||
error=formatted,
|
||||
run_id=view.litellm_call_id,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "sentry" if callable(capture := _singleton("capture_exception")):
|
||||
capture(exception)
|
||||
case "supabase" if isinstance(supabase := _singleton("supabaseClient"), Supabase):
|
||||
supabase.log_event(
|
||||
model=view.model,
|
||||
messages=view.messages,
|
||||
end_user=details.get("user", "default"),
|
||||
response_obj=None,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
litellm_call_id=details["litellm_call_id"],
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case "langfuse":
|
||||
_langfuse(
|
||||
logger,
|
||||
response=None,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
level="ERROR",
|
||||
status_message=str(exception),
|
||||
)
|
||||
case "traceloop" if isinstance(traceloop := _singleton("traceloopLogger"), TraceloopLogger):
|
||||
traceloop.log_event(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
response_obj=None,
|
||||
user_id=details.get("user", None),
|
||||
print_verbose=print_verbose,
|
||||
status_message=str(exception),
|
||||
level="ERROR",
|
||||
kwargs=details,
|
||||
)
|
||||
case "logfire" if isinstance(logfire := _singleton("logfireLogger"), LogfireLogger):
|
||||
logfire.log_event(
|
||||
kwargs={ # mutable-ok: logfire receives a private mutable copy
|
||||
key: value for key, value in details.items() if key != "original_response"
|
||||
}
|
||||
| {"exception": exception}, # mutable-ok: merged into the private copy above
|
||||
response_obj=None,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
level=LogfireLevel.ERROR,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
case _:
|
||||
return
|
||||
|
||||
|
||||
def restore_correlation_context(logger: object) -> None:
|
||||
from litellm import utils
|
||||
|
||||
utils._restore_correlation_context_if_supported(logger) # pyright: ignore[reportPrivateUsage] # legacy interop helper
|
||||
|
||||
|
||||
def submit_worker(job: Callable[[], None]) -> None:
|
||||
import contextvars
|
||||
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
|
||||
context: Final = contextvars.copy_context()
|
||||
_ = executor.submit(context.run, job)
|
||||
|
||||
|
||||
def enqueue_background(coroutine: Awaitable[None]) -> None:
|
||||
import contextvars
|
||||
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
enqueue: Final = _call_of(_attribute(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue"))
|
||||
contextvars.copy_context().run(enqueue, coroutine)
|
||||
|
|
@ -1,19 +1,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
Protocol,
|
||||
cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from typing import Final, Protocol
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -53,50 +42,6 @@ async def drive(execution: Execution) -> object:
|
|||
execution.close()
|
||||
|
||||
|
||||
class MetadataUpdater(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
result: object,
|
||||
logging_obj: Logging,
|
||||
model: str | None,
|
||||
kwargs: dict[str, object],
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallSetup:
|
||||
logger: Logging
|
||||
kwargs: dict[str, object]
|
||||
|
||||
|
||||
def setup(
|
||||
call_type: str,
|
||||
args: tuple[object, ...],
|
||||
kwargs: Mapping[str, object],
|
||||
start_time: datetime.datetime,
|
||||
asynchronous: bool,
|
||||
) -> CallSetup:
|
||||
from litellm import utils
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
arguments: Final = { # mutable-ok: function_setup consumes an owned kwargs dict
|
||||
"litellm_call_id": str(uuid.uuid4()),
|
||||
**kwargs,
|
||||
}
|
||||
supplied: Final = arguments.get("litellm_logging_obj")
|
||||
if isinstance(supplied, Logging):
|
||||
supplied._native_callback_fast_path = False # pyright: ignore[reportPrivateUsage] # supplied loggers retain all dispatch contracts
|
||||
return CallSetup(supplied, arguments)
|
||||
logger, prepared = utils.function_setup(
|
||||
call_type, utils.Rules(), start_time, *args, is_async_call=asynchronous, **arguments
|
||||
)
|
||||
if type(logger) is Logging and call_type in ("ocr", "aocr"):
|
||||
logger._native_callback_fast_path = True # pyright: ignore[reportPrivateUsage] # only bridge-created OCR loggers opt into callback elision
|
||||
return CallSetup(logger, prepared)
|
||||
|
||||
|
||||
def check_limits(kwargs: Mapping[str, object]) -> None:
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit
|
||||
|
|
@ -106,100 +51,3 @@ def check_limits(kwargs: Mapping[str, object]) -> None:
|
|||
raise litellm.BudgetExceededError(current_cost=current_cost, max_budget=litellm.max_budget)
|
||||
if max_retries_per_request_hit(kwargs, litellm.num_retries_per_request):
|
||||
raise RuntimeError("Max retries per request hit!")
|
||||
|
||||
|
||||
def finalize(
|
||||
response: object,
|
||||
logger: Logging,
|
||||
kwargs: dict[str, object],
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.llm_response_utils import response_metadata
|
||||
|
||||
model: Final = kwargs.get("model")
|
||||
update: Final = cast( # cast-ok: legacy metadata function accepts concrete kwargs
|
||||
MetadataUpdater, response_metadata.update_response_metadata
|
||||
)
|
||||
update(response, logger, model if isinstance(model, str) else None, kwargs, start_time, end_time)
|
||||
|
||||
|
||||
def deployment_callbacks_needed() -> bool:
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
return any(isinstance(callback, CustomLogger) for callback in litellm.callbacks)
|
||||
|
||||
|
||||
def callbacks_needed(logger: Logging, phase: str) -> bool:
|
||||
import litellm
|
||||
from litellm._logging import (
|
||||
_is_debugging_on, # pyright: ignore[reportPrivateUsage] # use the same debug gate as Logging
|
||||
)
|
||||
|
||||
if (
|
||||
_is_debugging_on()
|
||||
or getattr(logger, "litellm_request_debug", False)
|
||||
or os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD")
|
||||
):
|
||||
return True
|
||||
input_needed: Final = bool(
|
||||
litellm.input_callback
|
||||
or litellm._async_input_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
|
||||
or logger.dynamic_input_callbacks
|
||||
or callable(getattr(logger, "logger_fn", None))
|
||||
or logger.log_raw_request_response
|
||||
or litellm.log_raw_request_response
|
||||
)
|
||||
match phase:
|
||||
case "input":
|
||||
return input_needed
|
||||
case "sync_success":
|
||||
return bool(litellm.success_callback or logger.dynamic_success_callbacks)
|
||||
case "sync_success_async":
|
||||
return bool(
|
||||
(litellm.success_callback or logger.dynamic_success_callbacks)
|
||||
and logger._should_run_sync_callbacks_for_async_calls() # pyright: ignore[reportPrivateUsage] # preserve async call filtering of sync callbacks
|
||||
)
|
||||
case "async_success":
|
||||
return bool(litellm._async_success_callback or logger.dynamic_async_success_callbacks) # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
|
||||
case "sync_failure":
|
||||
return bool(litellm.failure_callback or logger.dynamic_failure_callbacks)
|
||||
case "async_failure":
|
||||
return bool(litellm._async_failure_callback or logger.dynamic_async_failure_callbacks) # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
|
||||
case "payload":
|
||||
return bool(
|
||||
input_needed
|
||||
or litellm.success_callback
|
||||
or litellm.failure_callback
|
||||
or litellm._async_success_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
|
||||
or litellm._async_failure_callback # pyright: ignore[reportPrivateUsage] # live async registries have no public accessor
|
||||
or logger.dynamic_success_callbacks
|
||||
or logger.dynamic_async_success_callbacks
|
||||
or logger.dynamic_failure_callbacks
|
||||
or logger.dynamic_async_failure_callbacks
|
||||
)
|
||||
case _:
|
||||
return True
|
||||
|
||||
|
||||
def success_bookkeeping(
|
||||
logger: Logging, response: object, start: datetime.datetime, end: datetime.datetime, asynchronous: bool
|
||||
) -> None:
|
||||
phase: Final = "async_success" if asynchronous else "sync_success"
|
||||
if logger.should_run_logging(phase):
|
||||
logger._success_handler_helper_fn( # pyright: ignore[reportPrivateUsage] # retain success bookkeeping without constructing a callback payload
|
||||
result=response, start_time=start, end_time=end, build_logging_payload=False
|
||||
)
|
||||
logger.has_run_logging(phase)
|
||||
|
||||
|
||||
def failure_bookkeeping(
|
||||
logger: Logging, error: BaseException, start: datetime.datetime, end: datetime.datetime, asynchronous: bool
|
||||
) -> None:
|
||||
phase: Final = "async_failure" if asynchronous else "sync_failure"
|
||||
if logger.should_run_logging(phase):
|
||||
logger._failure_handler_helper_fn( # pyright: ignore[reportPrivateUsage] # retain failure accounting without formatting an unused traceback or payload
|
||||
error, "", start, end, build_logging_payload=False
|
||||
)
|
||||
logger.has_run_logging(phase)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from collections.abc import Callable, Coroutine, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, cast # noqa: TID251 # validates dynamically loaded native callables
|
||||
|
||||
|
|
@ -8,6 +8,56 @@ from pydantic import TypeAdapter
|
|||
|
||||
from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse
|
||||
from litellm.rust_bridge.bindings import NativeBinding
|
||||
from litellm.types.utils import CustomPricingLiteLLMParams
|
||||
|
||||
|
||||
class OcrLoggingProtocol(Protocol):
|
||||
def update_from_kwargs(
|
||||
self,
|
||||
*,
|
||||
kwargs: dict[str, object],
|
||||
model: str,
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
custom_llm_provider: str,
|
||||
) -> object: ...
|
||||
|
||||
|
||||
def _redact(params: Mapping[str, object], secret_fields: Sequence[str]) -> dict[str, object]:
|
||||
return { # mutable-ok: update_from_kwargs takes dict
|
||||
name: "****" if name in secret_fields else value
|
||||
for name, value in params.items()
|
||||
if name != "proxy_server_request"
|
||||
}
|
||||
|
||||
|
||||
def update_logging(
|
||||
logger: OcrLoggingProtocol,
|
||||
kwargs: Mapping[str, object],
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
secret_fields: Sequence[str],
|
||||
url: str,
|
||||
) -> None:
|
||||
logger.update_from_kwargs(
|
||||
kwargs=_redact(kwargs, secret_fields),
|
||||
model=model,
|
||||
optional_params=_redact(optional_params, secret_fields),
|
||||
litellm_params={ # mutable-ok: update_from_kwargs takes dict
|
||||
"litellm_call_id": kwargs.get("litellm_call_id"),
|
||||
"api_base": url,
|
||||
**{
|
||||
name: kwargs[name] for name in ("logger_fn", "litellm_request_debug") if name in kwargs
|
||||
}, # mutable-ok: splat into the dict above
|
||||
**{ # mutable-ok: splat into the dict above
|
||||
name: kwargs[name]
|
||||
for name in CustomPricingLiteLLMParams.model_fields
|
||||
if name in kwargs and kwargs[name] is not None
|
||||
},
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
|
||||
class RustOcr(Protocol):
|
||||
|
|
|
|||
288
litellm/rust_bridge/setup.py
Normal file
288
litellm/rust_bridge/setup.py
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
from collections.abc import Callable, Mapping, MutableSequence, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
Literal,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
cast, # noqa: TID251 # narrows legacy untyped registries at the boundary
|
||||
)
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
CallbackTarget: TypeAlias = str | Callable[..., object] | CustomLogger
|
||||
RegistryName: TypeAlias = Literal[
|
||||
"input", "async_input", "success", "async_success", "failure", "async_failure", "callbacks"
|
||||
]
|
||||
NamedEvent: TypeAlias = Literal["success", "failure"]
|
||||
Kwargs: TypeAlias = dict[str, object] # mutable-ok: the retained call kwargs are the dict callbacks receive and mutate
|
||||
|
||||
_REGISTRY_ATTRIBUTES: Final[Mapping[RegistryName, str]] = MappingProxyType(
|
||||
{
|
||||
"input": "input_callback",
|
||||
"async_input": "_async_input_callback",
|
||||
"success": "success_callback",
|
||||
"async_success": "_async_success_callback",
|
||||
"failure": "failure_callback",
|
||||
"async_failure": "_async_failure_callback",
|
||||
"callbacks": "callbacks",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _CallbackManager(Protocol):
|
||||
def add_litellm_success_callback(self, callback: CallbackTarget) -> None: ...
|
||||
|
||||
def add_litellm_failure_callback(self, callback: CallbackTarget) -> None: ...
|
||||
|
||||
def add_litellm_async_success_callback(self, callback: CallbackTarget) -> None: ...
|
||||
|
||||
def add_litellm_async_failure_callback(self, callback: CallbackTarget) -> None: ...
|
||||
|
||||
|
||||
class _LoggingFactory(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
model: str | None,
|
||||
messages: object,
|
||||
stream: bool,
|
||||
litellm_call_id: str,
|
||||
litellm_trace_id: str | None,
|
||||
function_id: str,
|
||||
call_type: str,
|
||||
start_time: datetime.datetime,
|
||||
dynamic_success_callbacks: Sequence[CallbackTarget] | None,
|
||||
dynamic_failure_callbacks: Sequence[CallbackTarget] | None,
|
||||
dynamic_async_success_callbacks: Sequence[CallbackTarget] | None,
|
||||
dynamic_async_failure_callbacks: Sequence[CallbackTarget] | None,
|
||||
kwargs: Kwargs,
|
||||
applied_guardrails: Sequence[str],
|
||||
supports_correlation_logging: bool,
|
||||
) -> Logging: ...
|
||||
|
||||
|
||||
class _EnvironmentUpdater(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
model: str | None,
|
||||
user: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
stream_options: object,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
def _legacy(value: object) -> Callable[..., object]:
|
||||
return cast(Callable[..., object], value) # cast-ok: legacy module-level helpers are untyped
|
||||
|
||||
|
||||
def _mapping(value: object) -> Mapping[str, object] | None:
|
||||
if not isinstance(value, Mapping):
|
||||
return None
|
||||
return cast(Mapping[str, object], value) # cast-ok: isinstance narrows only to Mapping[Unknown, Unknown]
|
||||
|
||||
|
||||
def registry(
|
||||
name: RegistryName,
|
||||
) -> MutableSequence[CallbackTarget]: # mutable-ok: the public litellm registries are mutated by contract
|
||||
import litellm
|
||||
|
||||
return cast( # cast-ok: legacy module-level lists are untyped
|
||||
MutableSequence[CallbackTarget], getattr(litellm, _REGISTRY_ATTRIBUTES[name])
|
||||
)
|
||||
|
||||
|
||||
def is_async_callable(callback: object) -> bool:
|
||||
from litellm.litellm_core_utils.cached_imports import get_coroutine_checker
|
||||
|
||||
return get_coroutine_checker().is_async_callable(callback)
|
||||
|
||||
|
||||
def is_known_name(callback: str) -> bool:
|
||||
import litellm
|
||||
|
||||
known: Final = cast(Sequence[str], litellm._known_custom_logger_compatible_callbacks) # pyright: ignore[reportPrivateUsage] # cast-ok: registry list has no public typed accessor
|
||||
return callback in known
|
||||
|
||||
|
||||
def resolve_named_integration(callback: str) -> CustomLogger | None:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
factory: Final = _legacy(litellm_logging._init_custom_logger_compatible_class) # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # legacy factory
|
||||
resolved: Final = factory(callback, internal_usage_cache=None, llm_router=None)
|
||||
return resolved if isinstance(resolved, CustomLogger) else None
|
||||
|
||||
|
||||
def async_success_registry_has_type(callback: object) -> bool:
|
||||
return any(type(existing) is type(callback) for existing in registry("async_success"))
|
||||
|
||||
|
||||
def bootstrap_pending() -> bool:
|
||||
from litellm import utils
|
||||
|
||||
return not utils.callback_list
|
||||
|
||||
|
||||
def bootstrap(function_id: str | None) -> None:
|
||||
from litellm import utils
|
||||
from litellm.litellm_core_utils import cached_imports
|
||||
|
||||
combined: Final = _bootstrap_list()
|
||||
utils.callback_list = combined # rebind-ok: legacy module global consumed by set_callbacks
|
||||
set_callbacks: Final = _legacy(cached_imports.get_set_callbacks()) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # cached import is untyped
|
||||
set_callbacks(callback_list=combined, function_id=function_id)
|
||||
|
||||
|
||||
def _bootstrap_list() -> list[str]: # mutable-ok: set_callbacks and the legacy module global expect a list
|
||||
names: Final = frozenset({*registry("input"), *registry("success"), *registry("failure")})
|
||||
consumed: Final = cast(Sequence[str], names) # cast-ok: legacy list annotation is narrower than its contents
|
||||
return list(consumed) # mutable-ok: consumed by set_callbacks
|
||||
|
||||
|
||||
def expand_named(callback: str, event: NamedEvent) -> None:
|
||||
from litellm import utils
|
||||
|
||||
utils._add_custom_logger_callback_to_specific_event(callback, event) # pyright: ignore[reportPrivateUsage] # legacy expansion helper
|
||||
|
||||
|
||||
def append_registry(name: RegistryName, callback: CallbackTarget) -> None:
|
||||
import litellm
|
||||
|
||||
manager: Final = cast(_CallbackManager, litellm.logging_callback_manager) # cast-ok: legacy manager is untyped
|
||||
match name:
|
||||
case "input" | "async_input":
|
||||
registry(name).append(callback)
|
||||
case "success":
|
||||
manager.add_litellm_success_callback(callback)
|
||||
case "async_success":
|
||||
manager.add_litellm_async_success_callback(callback)
|
||||
case "failure":
|
||||
manager.add_litellm_failure_callback(callback)
|
||||
case "async_failure":
|
||||
manager.add_litellm_async_failure_callback(callback)
|
||||
case "callbacks":
|
||||
raise KeyError(name)
|
||||
|
||||
|
||||
def remove_registry(name: RegistryName, callback: CallbackTarget) -> None:
|
||||
target: Final = registry(name)
|
||||
for index in range(len(target) - 1, -1, -1):
|
||||
if target[index] is callback or target[index] == callback:
|
||||
del target[index]
|
||||
return
|
||||
|
||||
|
||||
def logger_fn(callback: object) -> None:
|
||||
from litellm import utils
|
||||
|
||||
utils.user_logger_fn = callback # rebind-ok: legacy module global read by pre_call
|
||||
|
||||
|
||||
def breadcrumb(kwargs: Mapping[str, object]) -> None:
|
||||
from litellm import utils
|
||||
|
||||
if utils.add_breadcrumb is None:
|
||||
return
|
||||
import litellm
|
||||
from litellm.litellm_core_utils import core_helpers
|
||||
|
||||
deep_copy: Final = _legacy(core_helpers.safe_deep_copy) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # legacy helper
|
||||
copied: Final = _copied(deep_copy, kwargs)
|
||||
hidden: Final = frozenset(("messages", "input", "prompt")) if litellm.turn_off_message_logging else frozenset[str]()
|
||||
details: Final = MappingProxyType({key: value for key, value in copied.items() if key not in hidden})
|
||||
_legacy(utils.add_breadcrumb)(category="litellm.llm_call", message=f"Keyword Args: {details}", level="info")
|
||||
|
||||
|
||||
def _copied(deep_copy: Callable[..., object], kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
try:
|
||||
copied: Final = deep_copy(dict(kwargs)) # mutable-ok: safe_deep_copy expects a dict
|
||||
except Exception: # noqa: BLE001 # legacy breadcrumb falls back to the live mapping
|
||||
return kwargs
|
||||
return _mapping(copied) or kwargs
|
||||
|
||||
|
||||
def prepare_environment() -> None:
|
||||
from litellm import utils
|
||||
|
||||
utils.custom_llm_setup()
|
||||
|
||||
|
||||
def applied_guardrails(kwargs: Mapping[str, object]) -> Sequence[str]:
|
||||
from litellm.utils import get_applied_guardrails
|
||||
|
||||
return tuple(get_applied_guardrails(dict(kwargs))) # mutable-ok: legacy helper expects a dict
|
||||
|
||||
|
||||
def _litellm_params(kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
metadata: Final = kwargs.get("metadata")
|
||||
litellm_metadata: Final = kwargs.get("litellm_metadata")
|
||||
base: Final = MappingProxyType({"api_base": ""})
|
||||
with_metadata: Final = MappingProxyType({**base, "metadata": metadata}) if "metadata" in kwargs else base
|
||||
typed_metadata: Final = _mapping(litellm_metadata)
|
||||
if typed_metadata is None:
|
||||
return with_metadata
|
||||
with_litellm_metadata: Final = MappingProxyType({**with_metadata, "litellm_metadata": typed_metadata})
|
||||
if metadata:
|
||||
return with_litellm_metadata
|
||||
copied_metadata: Final = dict(typed_metadata) # mutable-ok: callbacks may mutate this copy
|
||||
return MappingProxyType({**with_litellm_metadata, "metadata": copied_metadata})
|
||||
|
||||
|
||||
def _owned(
|
||||
callbacks: Sequence[CallbackTarget] | None,
|
||||
) -> list[CallbackTarget] | None: # mutable-ok: Logging appends to these lists
|
||||
return list(callbacks) if callbacks is not None else None # mutable-ok: Logging appends to these lists
|
||||
|
||||
|
||||
def build_logging(
|
||||
*,
|
||||
call_type: str,
|
||||
model: str | None,
|
||||
kwargs: Kwargs,
|
||||
start_time: datetime.datetime,
|
||||
asynchronous: bool,
|
||||
dynamic_success: Sequence[CallbackTarget] | None,
|
||||
dynamic_async_success: Sequence[CallbackTarget] | None,
|
||||
dynamic_failure: Sequence[CallbackTarget] | None,
|
||||
guardrails: Sequence[str],
|
||||
) -> Logging:
|
||||
from litellm.litellm_core_utils.cached_imports import get_litellm_logging_class
|
||||
|
||||
function_id: Final = kwargs.get("id")
|
||||
trace_id: Final = kwargs.get("litellm_trace_id")
|
||||
factory: Final = cast(_LoggingFactory, get_litellm_logging_class()) # cast-ok: legacy constructor is untyped
|
||||
logger: Final = factory(
|
||||
model=model,
|
||||
messages="default-message-value",
|
||||
stream=False,
|
||||
litellm_call_id=str(kwargs["litellm_call_id"]),
|
||||
litellm_trace_id=trace_id if isinstance(trace_id, str) else None,
|
||||
function_id=function_id if isinstance(function_id, str) else "",
|
||||
call_type=call_type,
|
||||
start_time=start_time,
|
||||
dynamic_success_callbacks=_owned(dynamic_success),
|
||||
dynamic_failure_callbacks=_owned(dynamic_failure),
|
||||
dynamic_async_success_callbacks=_owned(dynamic_async_success),
|
||||
dynamic_async_failure_callbacks=None,
|
||||
kwargs=kwargs,
|
||||
applied_guardrails=list(guardrails), # mutable-ok: legacy constructor annotation is list
|
||||
supports_correlation_logging=asynchronous,
|
||||
)
|
||||
update: Final = cast(_EnvironmentUpdater, logger.update_environment_variables) # cast-ok: legacy method is untyped
|
||||
update(
|
||||
model=model,
|
||||
user="",
|
||||
optional_params=MappingProxyType({}),
|
||||
litellm_params=_litellm_params(kwargs),
|
||||
stream_options=kwargs.get("stream_options"),
|
||||
)
|
||||
return logger
|
||||
|
|
@ -461,10 +461,16 @@ def test_native_pending_logging_is_released_only_for_ocr(call_type: str, excepti
|
|||
)
|
||||
|
||||
if call_type in ("ocr", "aocr"):
|
||||
pending.release.assert_called_once_with(not exception_raised)
|
||||
if exception_raised:
|
||||
pending.close.assert_called_once_with()
|
||||
pending.assert_not_called()
|
||||
else:
|
||||
pending.assert_called_once_with()
|
||||
pending.close.assert_not_called()
|
||||
assert logger._native_pending_logging is None
|
||||
else:
|
||||
pending.release.assert_not_called()
|
||||
pending.assert_not_called()
|
||||
pending.close.assert_not_called()
|
||||
assert logger._native_pending_logging is pending
|
||||
if exception_raised:
|
||||
enqueue.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -33,3 +33,17 @@ def test_binding_validates_native_attribute(
|
|||
binding: Final = bindings.NativeBinding("route", validate=lambda item: item if isinstance(item, int) else None)
|
||||
|
||||
assert binding.load() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("available", [False, True])
|
||||
def test_decline_accessor_only_catches_admission_declines(monkeypatch: pytest.MonkeyPatch, available: bool) -> None:
|
||||
class Declined(Exception):
|
||||
pass
|
||||
|
||||
class Upstream(Exception):
|
||||
pass
|
||||
|
||||
native: Final = SimpleNamespace(RustBridgeDeclined=Declined, RustUpstreamError=Upstream) if available else None
|
||||
monkeypatch.setattr(bindings, "get_native_bridge", lambda: native)
|
||||
assert bindings.native_decline_types() == ((Declined,) if available else ())
|
||||
assert not isinstance(Upstream("already dispatched"), bindings.native_decline_types())
|
||||
|
|
|
|||
282
tests/test_litellm/rust_bridge/test_leaves.py
Normal file
282
tests/test_litellm/rust_bridge/test_leaves.py
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
import asyncio
|
||||
import datetime
|
||||
import inspect
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.integrations.athina import AthinaLogger
|
||||
from litellm.integrations.dynamodb import DyanmoDBLogger
|
||||
from litellm.integrations.greenscale import GreenscaleLogger
|
||||
from litellm.integrations.helicone import HeliconeLogger
|
||||
from litellm.integrations.langfuse.langfuse import LangFuseLogger
|
||||
from litellm.integrations.logfire_logger import LogfireLevel, LogfireLogger
|
||||
from litellm.integrations.lunary import LunaryLogger
|
||||
from litellm.integrations.openmeter import OpenMeterLogger
|
||||
from litellm.integrations.prompt_layer import PromptLayerLogger
|
||||
from litellm.integrations.s3 import S3Logger
|
||||
from litellm.integrations.supabase import Supabase
|
||||
from litellm.integrations.traceloop import TraceloopLogger
|
||||
from litellm.integrations.weights_biases import WeightsBiasesLogger
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
from litellm.rust_bridge import leaves
|
||||
|
||||
START: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
|
||||
END: Final = START + datetime.timedelta(seconds=1)
|
||||
DETAILS: Final[dict[str, object]] = {
|
||||
"litellm_call_id": "call-id",
|
||||
"input": "document",
|
||||
"user": "user-1",
|
||||
"original_response": "raw",
|
||||
"litellm_params": {},
|
||||
}
|
||||
|
||||
|
||||
def _logger() -> Mock:
|
||||
return Mock(
|
||||
model="mistral/mistral-ocr-latest",
|
||||
messages="document",
|
||||
litellm_call_id="call-id",
|
||||
model_call_details=dict(DETAILS),
|
||||
standard_callback_dynamic_params={},
|
||||
)
|
||||
|
||||
|
||||
def _stub(cls: type, method: str) -> tuple[object, Mock]:
|
||||
instance: Final[object] = object.__new__(cls)
|
||||
recorder: Final = Mock(name=f"{cls.__name__}.{method}")
|
||||
setattr(instance, method, recorder)
|
||||
return instance, recorder
|
||||
|
||||
|
||||
def _assert_binds(cls: type, method: str, recorder: Mock, instance: object) -> None:
|
||||
recorder.assert_called_once()
|
||||
target: Final[object] = getattr(cls, method)
|
||||
assert callable(target)
|
||||
signature: Final = inspect.signature(target)
|
||||
bound: Final = signature.bind(instance, *recorder.call_args.args, **recorder.call_args.kwargs)
|
||||
required: Final = frozenset(
|
||||
name for name, parameter in signature.parameters.items() if parameter.default is inspect.Parameter.empty
|
||||
)
|
||||
assert required <= set(bound.arguments)
|
||||
|
||||
|
||||
SUCCESS_CASES: Final[tuple[tuple[str, str, type, str], ...]] = (
|
||||
("promptlayer", "promptLayerLogger", PromptLayerLogger, "log_event"),
|
||||
("wandb", "weightsBiasesLogger", WeightsBiasesLogger, "log_event"),
|
||||
("athina", "athinaLogger", AthinaLogger, "log_event"),
|
||||
("logfire", "logfireLogger", LogfireLogger, "log_event"),
|
||||
("greenscale", "greenscaleLogger", GreenscaleLogger, "log_event"),
|
||||
("supabase", "supabaseClient", Supabase, "log_event"),
|
||||
("lunary", "lunaryLogger", LunaryLogger, "log_event"),
|
||||
("helicone", "heliconeLogger", HeliconeLogger, "log_success"),
|
||||
("traceloop", "traceloopLogger", TraceloopLogger, "log_event"),
|
||||
("s3", "s3Logger", S3Logger, "log_event"),
|
||||
)
|
||||
|
||||
FAILURE_CASES: Final[tuple[tuple[str, str, type, str], ...]] = (
|
||||
("lunary", "lunaryLogger", LunaryLogger, "log_event"),
|
||||
("supabase", "supabaseClient", Supabase, "log_event"),
|
||||
("traceloop", "traceloopLogger", TraceloopLogger, "log_event"),
|
||||
("logfire", "logfireLogger", LogfireLogger, "log_event"),
|
||||
)
|
||||
|
||||
|
||||
def _install(monkeypatch: pytest.MonkeyPatch, global_name: str, value: object) -> None:
|
||||
monkeypatch.setattr(litellm_logging, global_name, value, raising=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("name", "global_name", "cls", "method"), SUCCESS_CASES)
|
||||
def test_named_success_calls_integration_with_its_real_signature(
|
||||
monkeypatch: pytest.MonkeyPatch, name: str, global_name: str, cls: type, method: str
|
||||
) -> None:
|
||||
instance, recorder = _stub(cls, method)
|
||||
_install(monkeypatch, global_name, instance)
|
||||
response: Final = object()
|
||||
|
||||
assert leaves.dispatch_named_success(_logger(), name, response, START, END) is None
|
||||
|
||||
_assert_binds(cls, method, recorder, instance)
|
||||
kwargs: Final = recorder.call_args.kwargs
|
||||
assert kwargs["start_time"] is START and kwargs["end_time"] is END
|
||||
assert kwargs["response_obj"] is response
|
||||
assert callable(kwargs["print_verbose"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("name", "global_name", "cls", "method"), FAILURE_CASES)
|
||||
def test_named_failure_calls_integration_with_its_real_signature(
|
||||
monkeypatch: pytest.MonkeyPatch, name: str, global_name: str, cls: type, method: str
|
||||
) -> None:
|
||||
instance, recorder = _stub(cls, method)
|
||||
_install(monkeypatch, global_name, instance)
|
||||
error: Final = RuntimeError("boom")
|
||||
|
||||
leaves.dispatch_named_failure(_logger(), name, error, "formatted", START, END)
|
||||
|
||||
_assert_binds(cls, method, recorder, instance)
|
||||
kwargs: Final = recorder.call_args.kwargs
|
||||
assert kwargs["start_time"] is START and kwargs["end_time"] is END
|
||||
assert kwargs.get("response_obj") is None
|
||||
|
||||
|
||||
def test_logfire_receives_enum_levels_and_exception_on_failure(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
instance, recorder = _stub(LogfireLogger, "log_event")
|
||||
_install(monkeypatch, "logfireLogger", instance)
|
||||
error: Final = RuntimeError("boom")
|
||||
|
||||
leaves.dispatch_named_success(_logger(), "logfire", object(), START, END)
|
||||
assert recorder.call_args.kwargs["level"] is LogfireLevel.INFO
|
||||
assert "original_response" not in recorder.call_args.kwargs["kwargs"]
|
||||
|
||||
leaves.dispatch_named_failure(_logger(), "logfire", error, "formatted", START, END)
|
||||
assert recorder.call_args.kwargs["level"] is LogfireLevel.ERROR
|
||||
assert recorder.call_args.kwargs["kwargs"]["exception"] is error
|
||||
assert "original_response" not in recorder.call_args.kwargs["kwargs"]
|
||||
|
||||
|
||||
def test_lunary_marks_success_and_error_events(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
instance, recorder = _stub(LunaryLogger, "log_event")
|
||||
_install(monkeypatch, "lunaryLogger", instance)
|
||||
|
||||
leaves.dispatch_named_success(_logger(), "lunary", object(), START, END)
|
||||
assert recorder.call_args.kwargs["event"] == "end"
|
||||
assert recorder.call_args.kwargs["run_id"] == "call-id"
|
||||
|
||||
leaves.dispatch_named_failure(_logger(), "lunary", RuntimeError("boom"), "formatted", START, END)
|
||||
assert recorder.call_args.kwargs["event"] == "error"
|
||||
assert recorder.call_args.kwargs["error"] == "formatted"
|
||||
|
||||
|
||||
def test_supabase_request_hook_only_fires_pre_call(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
instance, recorder = _stub(Supabase, "input_log_event")
|
||||
_install(monkeypatch, "supabaseClient", instance)
|
||||
|
||||
leaves.dispatch_named_request(_logger(), "supabase", "post_api_call")
|
||||
recorder.assert_not_called()
|
||||
|
||||
leaves.dispatch_named_request(_logger(), "supabase", "pre_api_call")
|
||||
_assert_binds(Supabase, "input_log_event", recorder, instance)
|
||||
assert recorder.call_args.kwargs["end_user"] == "user-1"
|
||||
|
||||
|
||||
def test_sentry_hooks_are_called_when_configured(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
breadcrumb: Final = Mock()
|
||||
capture: Final = Mock()
|
||||
_install(monkeypatch, "add_breadcrumb", breadcrumb)
|
||||
_install(monkeypatch, "capture_exception", capture)
|
||||
error: Final = RuntimeError("boom")
|
||||
|
||||
leaves.dispatch_named_request(_logger(), "sentry", "pre_api_call")
|
||||
assert breadcrumb.call_args.kwargs["category"] == "litellm.llm_call"
|
||||
|
||||
leaves.dispatch_named_failure(_logger(), "sentry", error, "formatted", START, END)
|
||||
capture.assert_called_once_with(error)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("name", "global_name"), [(case[0], case[1]) for case in SUCCESS_CASES])
|
||||
def test_unconfigured_or_foreign_singleton_is_skipped(
|
||||
monkeypatch: pytest.MonkeyPatch, name: str, global_name: str
|
||||
) -> None:
|
||||
_install(monkeypatch, global_name, None)
|
||||
assert leaves.dispatch_named_success(_logger(), name, object(), START, END) is None
|
||||
|
||||
foreign: Final = Mock()
|
||||
_install(monkeypatch, global_name, foreign)
|
||||
assert leaves.dispatch_named_success(_logger(), name, object(), START, END) is None
|
||||
assert not foreign.method_calls
|
||||
|
||||
|
||||
def _run(awaitable: object) -> None:
|
||||
async def consume() -> None:
|
||||
assert inspect.isawaitable(awaitable)
|
||||
_ = await awaitable
|
||||
|
||||
asyncio.run(consume())
|
||||
|
||||
|
||||
def test_openmeter_returns_the_async_coroutine(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
instance: Final = object.__new__(OpenMeterLogger)
|
||||
recorder: Final = Mock()
|
||||
|
||||
async def async_log_success_event(**kwargs: object) -> None:
|
||||
recorder(**kwargs)
|
||||
|
||||
monkeypatch.setattr(instance, "async_log_success_event", async_log_success_event)
|
||||
_install(monkeypatch, "openMeterLogger", instance)
|
||||
response: Final = object()
|
||||
|
||||
_run(leaves.dispatch_named_success(_logger(), "openmeter", response, START, END))
|
||||
|
||||
_assert_binds(OpenMeterLogger, "async_log_success_event", recorder, instance)
|
||||
assert recorder.call_args.kwargs["response_obj"] is response
|
||||
|
||||
|
||||
def test_dynamodb_reuses_the_existing_singleton(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
instance: Final = object.__new__(DyanmoDBLogger)
|
||||
recorder: Final = Mock()
|
||||
|
||||
async def _async_log_event(**kwargs: object) -> None:
|
||||
recorder(**kwargs)
|
||||
|
||||
monkeypatch.setattr(instance, "_async_log_event", _async_log_event)
|
||||
_install(monkeypatch, "dynamoLogger", instance)
|
||||
|
||||
_run(leaves.dispatch_named_success(_logger(), "dynamodb", object(), START, END))
|
||||
|
||||
_assert_binds(DyanmoDBLogger, "_async_log_event", recorder, instance)
|
||||
assert litellm_logging.dynamoLogger is instance
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def langfuse_handler(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[LangFuseLogger, Mock, Mock]]:
|
||||
handler: Final = object.__new__(LangFuseLogger)
|
||||
recorder: Final = Mock(return_value={"trace_id": "trace-1"})
|
||||
monkeypatch.setattr(handler, "log_event_on_langfuse", recorder)
|
||||
_install(monkeypatch, "langFuseLogger", handler)
|
||||
cache: Final = Mock()
|
||||
_install(monkeypatch, "in_memory_trace_id_cache", cache)
|
||||
yield handler, recorder, cache
|
||||
|
||||
|
||||
def test_langfuse_success_logs_and_caches_trace_id(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None:
|
||||
handler, recorder, cache = langfuse_handler
|
||||
response: Final = object()
|
||||
|
||||
leaves.dispatch_named_success(_logger(), "langfuse", response, START, END)
|
||||
|
||||
_assert_binds(LangFuseLogger, "log_event_on_langfuse", recorder, handler)
|
||||
kwargs: Final = recorder.call_args.kwargs
|
||||
assert kwargs["response_obj"] is response
|
||||
assert kwargs["user_id"] == "user-1"
|
||||
assert kwargs["level"] == "DEFAULT" and kwargs["status_message"] is None
|
||||
assert "original_response" not in kwargs["kwargs"]
|
||||
cache.set_cache.assert_called_once_with(litellm_call_id="call-id", service_name="langfuse", trace_id="trace-1")
|
||||
|
||||
|
||||
def test_langfuse_failure_logs_error_level(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None:
|
||||
handler, recorder, _ = langfuse_handler
|
||||
|
||||
leaves.dispatch_named_failure(_logger(), "langfuse", RuntimeError("boom"), "formatted", START, END)
|
||||
|
||||
_assert_binds(LangFuseLogger, "log_event_on_langfuse", recorder, handler)
|
||||
kwargs: Final = recorder.call_args.kwargs
|
||||
assert kwargs["response_obj"] is None
|
||||
assert kwargs["level"] == "ERROR" and kwargs["status_message"] == "boom"
|
||||
|
||||
|
||||
def test_langfuse_skips_cache_without_trace_id(langfuse_handler: tuple[LangFuseLogger, Mock, Mock]) -> None:
|
||||
_, recorder, cache = langfuse_handler
|
||||
recorder.return_value = {}
|
||||
|
||||
leaves.dispatch_named_success(_logger(), "langfuse", object(), START, END)
|
||||
|
||||
cache.set_cache.assert_not_called()
|
||||
|
||||
|
||||
def test_signature_binding_rejects_drift() -> None:
|
||||
instance, recorder = _stub(Supabase, "log_event")
|
||||
recorder(model="m", end_userr="typo", response_obj=None, start_time=START, end_time=END)
|
||||
with pytest.raises(TypeError):
|
||||
_assert_binds(Supabase, "log_event", recorder, instance)
|
||||
|
|
@ -8,7 +8,8 @@ import litellm
|
|||
from litellm.llms.base_llm.ocr.transformation import OCRResponse
|
||||
from litellm.ocr import legacy
|
||||
from litellm.rust_bridge import bindings, configuration
|
||||
from litellm.rust_bridge.ocr import NATIVE_AOCR, NATIVE_OCR
|
||||
from litellm.rust_bridge.leaves import record_post_call, record_pre_call
|
||||
from litellm.rust_bridge.ocr import NATIVE_AOCR, NATIVE_OCR, update_logging
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -21,6 +22,73 @@ def isolated_ocr_configuration(monkeypatch: pytest.MonkeyPatch) -> Generator[Non
|
|||
configuration.reset_rust_configuration()
|
||||
|
||||
|
||||
def test_logging_redacts_views_and_preserves_opaque_arguments_and_pricing() -> None:
|
||||
logger: Final = Mock()
|
||||
opaque: Final = object()
|
||||
logger_fn: Final = object()
|
||||
kwargs: Final = {
|
||||
"vertex_credentials": "secret",
|
||||
"proxy_server_request": opaque,
|
||||
"metadata": opaque,
|
||||
"logger_fn": logger_fn,
|
||||
"litellm_request_debug": False,
|
||||
"litellm_call_id": "call-id",
|
||||
"input_cost_per_token": 0,
|
||||
"output_cost_per_token": None,
|
||||
}
|
||||
optional: Final = {"vertex_credentials": "secret", "pages": [1], "proxy_server_request": opaque}
|
||||
update_logging(logger, kwargs, "model", "vertex_ai", optional, ("vertex_credentials",), "https://provider")
|
||||
logger.update_from_kwargs.assert_called_once_with(
|
||||
kwargs={
|
||||
"vertex_credentials": "****",
|
||||
"metadata": opaque,
|
||||
"logger_fn": logger_fn,
|
||||
"litellm_request_debug": False,
|
||||
"litellm_call_id": "call-id",
|
||||
"input_cost_per_token": 0,
|
||||
"output_cost_per_token": None,
|
||||
},
|
||||
model="model",
|
||||
custom_llm_provider="vertex_ai",
|
||||
optional_params={"vertex_credentials": "****", "pages": [1]},
|
||||
litellm_params={
|
||||
"litellm_call_id": "call-id",
|
||||
"api_base": "https://provider",
|
||||
"logger_fn": logger_fn,
|
||||
"litellm_request_debug": False,
|
||||
"input_cost_per_token": 0,
|
||||
},
|
||||
)
|
||||
assert kwargs["vertex_credentials"] == optional["vertex_credentials"] == "secret"
|
||||
assert kwargs["proxy_server_request"] is optional["proxy_server_request"] is opaque
|
||||
assert logger.update_from_kwargs.call_args.kwargs["kwargs"]["metadata"] is opaque
|
||||
assert logger.update_from_kwargs.call_args.kwargs["optional_params"]["pages"] is optional["pages"]
|
||||
|
||||
|
||||
def test_logging_callbacks_receive_captured_payload_roots_and_propagate_errors() -> None:
|
||||
logger: Final = Mock(log_raw_request_response=False, logger_fn=None, model_call_details={})
|
||||
body: Final[dict[str, object]] = {"document": "original"}
|
||||
headers: Final = {"authorization": "key"}
|
||||
response: Final = object()
|
||||
record_pre_call(logger, api_key="key", body=body, headers=headers, url="https://provider")
|
||||
record_post_call(logger, original_response=response, body=body, headers=headers)
|
||||
logger._pre_call.assert_called_once_with(
|
||||
input="OCR document processing",
|
||||
api_key="key",
|
||||
model=None,
|
||||
additional_args={"complete_input_dict": body, "headers": headers, "api_base": "https://provider"},
|
||||
)
|
||||
for callback in (logger._pre_call, logger.record_post_call):
|
||||
assert callback.call_args.kwargs["additional_args"]["complete_input_dict"] is body
|
||||
assert callback.call_args.kwargs["additional_args"]["headers"] is headers
|
||||
assert logger.record_post_call.call_args.kwargs["original_response"] is response
|
||||
failure: Final = RuntimeError("callback failed")
|
||||
failing_logger: Final = Mock(_pre_call=Mock(side_effect=failure))
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
record_pre_call(failing_logger, api_key=None, body=body, headers=headers, url="https://provider")
|
||||
assert caught.value is failure
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
async def test_unavailable_native_uses_legacy(monkeypatch: pytest.MonkeyPatch, asynchronous: bool) -> None:
|
||||
|
|
|
|||
169
tests/test_litellm/rust_bridge/test_setup.py
Normal file
169
tests/test_litellm/rust_bridge/test_setup.py
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
import datetime
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import utils
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.rust_bridge.loader import native_bridge_available
|
||||
|
||||
pytestmark = pytest.mark.skipif(not native_bridge_available(), reason="requires the Rust extension")
|
||||
|
||||
REGISTRIES: Final = (
|
||||
"input_callback",
|
||||
"_async_input_callback",
|
||||
"success_callback",
|
||||
"_async_success_callback",
|
||||
"failure_callback",
|
||||
"_async_failure_callback",
|
||||
"callbacks",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_registries(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
for name in REGISTRIES:
|
||||
monkeypatch.setattr(litellm, name, [])
|
||||
monkeypatch.setattr(utils, "callback_list", [])
|
||||
monkeypatch.setattr(utils, "user_logger_fn", None)
|
||||
monkeypatch.setenv("OPENMETER_API_KEY", "test")
|
||||
monkeypatch.setenv("OPENMETER_API_ENDPOINT", "http://127.0.0.1:9")
|
||||
yield
|
||||
|
||||
|
||||
class SyncLogger(CustomLogger):
|
||||
pass
|
||||
|
||||
|
||||
class AsyncOnly(CustomLogger):
|
||||
pass
|
||||
|
||||
|
||||
def sync_fn(*args: object, **kwargs: object) -> None:
|
||||
del args, kwargs
|
||||
|
||||
|
||||
async def async_fn(*args: object, **kwargs: object) -> None:
|
||||
del args, kwargs
|
||||
|
||||
|
||||
def snapshot() -> dict[str, list[object]]:
|
||||
return {name: list(getattr(litellm, name)) for name in REGISTRIES}
|
||||
|
||||
|
||||
def run_legacy(kwargs: dict[str, object]) -> tuple[Logging, dict[str, object]]:
|
||||
logger, prepared = utils.function_setup(
|
||||
"ocr",
|
||||
utils.Rules(),
|
||||
datetime.datetime.now(),
|
||||
is_async_call=False,
|
||||
**{"litellm_call_id": "legacy", **kwargs},
|
||||
)
|
||||
assert isinstance(logger, Logging)
|
||||
return logger, prepared
|
||||
|
||||
|
||||
def run_native(kwargs: dict[str, object]) -> tuple[Logging, dict[str, object]]:
|
||||
from litellm.rust_bridge import _native
|
||||
|
||||
logger, prepared = _native._debug_setup(
|
||||
"ocr", (), {"litellm_call_id": "native", **kwargs}, datetime.datetime.now(), False
|
||||
)
|
||||
assert isinstance(logger, Logging)
|
||||
return logger, prepared
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"globals_before, kwargs",
|
||||
[
|
||||
({}, {}),
|
||||
({"callbacks": [SyncLogger()]}, {}),
|
||||
({"callbacks": [async_fn]}, {}),
|
||||
({}, {"callbacks": [SyncLogger(), sync_fn]}),
|
||||
({"success_callback": [async_fn, "openmeter", sync_fn]}, {}),
|
||||
({"failure_callback": [async_fn, sync_fn]}, {}),
|
||||
({"input_callback": [async_fn, sync_fn]}, {}),
|
||||
({}, {"success_callback": [sync_fn, async_fn, "s3", "dynamodb"], "failure_callback": [sync_fn]}),
|
||||
({"callbacks": [SyncLogger()]}, {"callbacks": [SyncLogger()], "success_callback": [async_fn]}),
|
||||
],
|
||||
ids=[
|
||||
"empty",
|
||||
"global-custom-logger",
|
||||
"global-async-callable",
|
||||
"dynamic-callbacks",
|
||||
"success-safety-net",
|
||||
"failure-safety-net",
|
||||
"input-safety-net",
|
||||
"per-call-success-failure-split",
|
||||
"mixed",
|
||||
],
|
||||
)
|
||||
def test_native_setup_registry_side_effects_match_function_setup(
|
||||
clean_registries: None, globals_before: dict[str, list[object]], kwargs: dict[str, object]
|
||||
) -> None:
|
||||
for name, values in globals_before.items():
|
||||
getattr(litellm, name).extend(values)
|
||||
legacy_logger, legacy_kwargs = run_legacy(
|
||||
{key: list(value) if isinstance(value, list) else value for key, value in kwargs.items()}
|
||||
)
|
||||
legacy_snapshot: Final = snapshot()
|
||||
legacy_bootstrap: Final = list(utils.callback_list or [])
|
||||
|
||||
for name in REGISTRIES:
|
||||
getattr(litellm, name).clear()
|
||||
utils.callback_list = []
|
||||
for name, values in globals_before.items():
|
||||
getattr(litellm, name).extend(values)
|
||||
native_logger, native_kwargs = run_native(
|
||||
{key: list(value) if isinstance(value, list) else value for key, value in kwargs.items()}
|
||||
)
|
||||
|
||||
assert snapshot() == legacy_snapshot
|
||||
assert sorted(map(repr, utils.callback_list or [])) == sorted(map(repr, legacy_bootstrap))
|
||||
assert set(native_kwargs) - {"litellm_call_id"} == set(legacy_kwargs) - {"litellm_call_id"}
|
||||
for attribute in (
|
||||
"dynamic_success_callbacks",
|
||||
"dynamic_async_success_callbacks",
|
||||
"dynamic_failure_callbacks",
|
||||
"dynamic_async_failure_callbacks",
|
||||
"call_type",
|
||||
"stream",
|
||||
"model",
|
||||
):
|
||||
assert getattr(native_logger, attribute) == getattr(legacy_logger, attribute), attribute
|
||||
|
||||
|
||||
def test_native_setup_honours_caller_supplied_logging_object(clean_registries: None) -> None:
|
||||
class Supplied(Logging):
|
||||
pass
|
||||
|
||||
supplied: Final = Supplied(
|
||||
model="mistral/mistral-ocr-latest",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="ocr",
|
||||
start_time=datetime.datetime.now(),
|
||||
litellm_call_id="supplied",
|
||||
function_id="",
|
||||
)
|
||||
litellm.callbacks.append(SyncLogger())
|
||||
logger, kwargs = run_native({"litellm_logging_obj": supplied, "callbacks": [SyncLogger()]})
|
||||
assert logger is supplied
|
||||
assert kwargs["callbacks"] is not None
|
||||
assert litellm.success_callback == []
|
||||
|
||||
|
||||
def test_native_setup_records_logger_fn_and_metadata(clean_registries: None) -> None:
|
||||
def logger_fn(details: object) -> None:
|
||||
del details
|
||||
|
||||
logger, kwargs = run_native(
|
||||
{"model": "mistral/mistral-ocr-latest", "logger_fn": logger_fn, "metadata": {"source": "test"}}
|
||||
)
|
||||
assert utils.user_logger_fn is logger_fn
|
||||
assert logger.litellm_params["metadata"] == {"source": "test"}
|
||||
assert logger.model == "mistral/mistral-ocr-latest"
|
||||
assert kwargs["metadata"] == {"source": "test"}
|
||||
|
|
@ -10,6 +10,7 @@ import litellm
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRResponse
|
||||
from tests.test_litellm_rust.support.callback_recorder import RecordingLogger
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
|
||||
from tests.test_litellm_rust.support.requests import (
|
||||
OCR_DOCUMENT,
|
||||
OCR_RESPONSE,
|
||||
|
|
@ -18,7 +19,6 @@ from tests.test_litellm_rust.support.requests import (
|
|||
request_body,
|
||||
request_headers,
|
||||
)
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
|
||||
|
||||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
||||
|
|
@ -119,69 +119,6 @@ def test_native_ocr_pre_call_header_rebinding_does_not_replace_execution_root(oc
|
|||
assert "x-rebound" not in ocr_server.requests[0].headers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_native_ocr_pre_call_nested_document_edit_updates_caller_callback_and_provider_references(
|
||||
ocr_server: RecordingServer, asynchronous: bool
|
||||
) -> None:
|
||||
original: Final = dict(OCR_DOCUMENT)
|
||||
replacement_url: Final = "data:application/pdf;base64,ZGVm"
|
||||
retained: Final = []
|
||||
aliases: Final = []
|
||||
|
||||
class Retain(CustomLogger):
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
aliases.append(request_body(kwargs)["document"] is original)
|
||||
retained.append(request_body(kwargs)["document"])
|
||||
|
||||
class Edit(CustomLogger):
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
original["document_url"] = replacement_url
|
||||
|
||||
arguments: Final = {
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"document": original,
|
||||
"api_key": "test-key",
|
||||
"api_base": ocr_server.base_url,
|
||||
"callbacks": [Retain(), Edit()],
|
||||
}
|
||||
response: Final = (
|
||||
await call_native_aocr(ocr_server, **arguments)
|
||||
if asynchronous
|
||||
else call_native_ocr(ocr_server, **arguments)
|
||||
)
|
||||
|
||||
assert aliases == [True]
|
||||
assert retained[0]["document_url"] == replacement_url
|
||||
assert original["document_url"] == replacement_url
|
||||
assert ocr_server.requests[0].body["document"]["document_url"] == replacement_url
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
|
||||
|
||||
def test_native_ocr_pre_call_document_replacement_does_not_mutate_original_document(
|
||||
ocr_server: RecordingServer,
|
||||
) -> None:
|
||||
original: Final = dict(OCR_DOCUMENT)
|
||||
replacement: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,ZGVm"}
|
||||
retained: Final = []
|
||||
|
||||
class RetainAndReplace(CustomLogger):
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
body = request_body(kwargs)
|
||||
retained.append(body["document"])
|
||||
body["document"] = replacement
|
||||
|
||||
call_native_ocr(
|
||||
ocr_server,
|
||||
document=original,
|
||||
callbacks=[RetainAndReplace()],
|
||||
)
|
||||
|
||||
assert retained[0] is original
|
||||
assert original["document_url"] == OCR_DOCUMENT["document_url"]
|
||||
assert ocr_server.requests[0].body["document"] == replacement
|
||||
|
||||
|
||||
def test_native_ocr_pre_call_body_rebinding_is_visible_to_callbacks_but_not_provider(
|
||||
ocr_server: RecordingServer,
|
||||
) -> None:
|
||||
|
|
@ -372,6 +309,7 @@ async def test_native_azure_ocr_resolves_token_before_pre_call_on_caller_context
|
|||
asynchronous: bool,
|
||||
) -> None:
|
||||
from contextvars import ContextVar
|
||||
|
||||
context: Final = ContextVar("azure-token-context", default="missing")
|
||||
context.set("caller")
|
||||
caller_thread: Final = threading.current_thread()
|
||||
|
|
@ -400,9 +338,7 @@ async def test_native_azure_ocr_resolves_token_before_pre_call_on_caller_context
|
|||
"callbacks": [Edit()],
|
||||
}
|
||||
response: Final = (
|
||||
await call_native_aocr(ocr_server, **arguments)
|
||||
if asynchronous
|
||||
else call_native_ocr(ocr_server, **arguments)
|
||||
await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments)
|
||||
)
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
assert observations == ["token", "pre_call"]
|
||||
|
|
@ -431,9 +367,7 @@ async def test_native_azure_ocr_token_provider_can_make_nested_native_ocr_call(
|
|||
"azure_ad_token_provider": provider,
|
||||
}
|
||||
response: Final = (
|
||||
await call_native_aocr(ocr_server, **arguments)
|
||||
if asynchronous
|
||||
else call_native_ocr(ocr_server, **arguments)
|
||||
await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments)
|
||||
)
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
assert calls == ["token"]
|
||||
|
|
@ -488,7 +422,9 @@ async def test_native_azure_ocr_releases_token_provider_after_terminal_outcome(
|
|||
) -> None:
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
from tests.test_litellm_rust.support.callback_recorder import drain_logging
|
||||
|
||||
class Provider:
|
||||
def __call__(self) -> str:
|
||||
if outcome == "failure":
|
||||
|
|
@ -534,3 +470,274 @@ async def test_native_azure_ocr_releases_token_provider_after_terminal_outcome(
|
|||
await asyncio.sleep(0)
|
||||
gc.collect()
|
||||
assert reference() is None
|
||||
|
||||
|
||||
FORBIDDEN_ORCHESTRATION: Final = (
|
||||
"pre_call",
|
||||
"post_call",
|
||||
"success_handler",
|
||||
"async_success_handler",
|
||||
"failure_handler",
|
||||
"async_failure_handler",
|
||||
"_success_handler_body",
|
||||
"_async_success_handler_body",
|
||||
"_failure_handler_body",
|
||||
"_async_failure_handler_body",
|
||||
"dispatch_success_handlers",
|
||||
"dispatch_failure_handlers",
|
||||
"handle_sync_success_callbacks_for_async_calls",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def legacy_orchestration_disabled(monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||||
from litellm import utils
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
reached: Final[list[str]] = []
|
||||
|
||||
def forbid(name: str):
|
||||
def method(self, *args, **kwargs):
|
||||
reached.append(name)
|
||||
raise AssertionError(f"legacy orchestration reached: {name}")
|
||||
|
||||
return method
|
||||
|
||||
for name in FORBIDDEN_ORCHESTRATION:
|
||||
monkeypatch.setattr(Logging, name, forbid(name))
|
||||
|
||||
def forbid_utils(name: str):
|
||||
async def hook(*args, **kwargs):
|
||||
reached.append(name)
|
||||
raise AssertionError(f"legacy orchestration reached: {name}")
|
||||
|
||||
def setup(*args, **kwargs):
|
||||
reached.append(name)
|
||||
raise AssertionError(f"legacy orchestration reached: {name}")
|
||||
|
||||
return setup if name == "function_setup" else hook
|
||||
|
||||
for name in (
|
||||
"function_setup",
|
||||
"async_pre_call_deployment_hook",
|
||||
"async_post_call_success_deployment_hook",
|
||||
"async_post_call_failure_deployment_hook",
|
||||
):
|
||||
monkeypatch.setattr(utils, name, forbid_utils(name))
|
||||
return reached
|
||||
|
||||
|
||||
class DeploymentRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.hooks: list[str] = []
|
||||
|
||||
async def async_pre_call_deployment_hook(self, kwargs, call_type):
|
||||
self.hooks.append(f"pre:{call_type.value}")
|
||||
return {**kwargs, "metadata": {**(kwargs.get("metadata") or {}), "deployment": "seen"}}
|
||||
|
||||
async def async_post_call_success_deployment_hook(self, request_data, response, call_type):
|
||||
self.hooks.append(f"post:{request_data['metadata']['deployment']}")
|
||||
return response
|
||||
|
||||
async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
|
||||
self.hooks.append(f"failure:{type(exception).__name__}:{fallback_depth}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_native_ocr_success_runs_integrations_without_legacy_orchestration(
|
||||
ocr_server: RecordingServer, legacy_orchestration_disabled: list[str], asynchronous: bool
|
||||
) -> None:
|
||||
recorder: Final = RecordingLogger()
|
||||
deployment: Final = DeploymentRecorder()
|
||||
litellm.callbacks.append(deployment)
|
||||
arguments: Final = {"callbacks": [recorder]}
|
||||
response: Final = (
|
||||
await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments)
|
||||
)
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
success_event: Final = "async_log_success_event" if asynchronous else "log_success_event"
|
||||
events: Final = await recorder.wait_for_async(success_event)
|
||||
assert legacy_orchestration_disabled == []
|
||||
assert deployment.hooks == (["pre:aocr", "post:seen"] if asynchronous else [])
|
||||
assert recorder.names.count("log_pre_api_call") == 1
|
||||
assert events[0].kwargs["standard_logging_object"]["status"] == "success"
|
||||
assert events[0].kwargs["response_cost"] is not None
|
||||
assert events[0].kwargs["litellm_params"]["api_base"] == f"{ocr_server.base_url}/v1/ocr"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_native_ocr_failure_runs_integrations_without_legacy_orchestration(
|
||||
ocr_server: RecordingServer, legacy_orchestration_disabled: list[str], asynchronous: bool
|
||||
) -> None:
|
||||
ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500))
|
||||
recorder: Final = RecordingLogger()
|
||||
deployment: Final = DeploymentRecorder()
|
||||
litellm.callbacks.append(deployment)
|
||||
arguments: Final = {"callbacks": [recorder]}
|
||||
with pytest.raises(litellm.InternalServerError) as caught:
|
||||
await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments)
|
||||
assert legacy_orchestration_disabled == []
|
||||
assert deployment.hooks == (["pre:aocr", "failure:InternalServerError:None"] if asynchronous else [])
|
||||
failures: Final = tuple(event for event in recorder.events if event.name.endswith("log_failure_event"))
|
||||
assert [event.name for event in failures] == (
|
||||
["log_failure_event", "async_log_failure_event"] if asynchronous else ["log_failure_event"]
|
||||
)
|
||||
assert all(event.kwargs["exception"] is caught.value for event in failures)
|
||||
assert all(event.kwargs["standard_logging_object"]["status"] == "failure" for event in failures)
|
||||
assert "log_success_event" not in recorder.names
|
||||
|
||||
|
||||
def test_native_ocr_success_hooks_all_run_before_any_success_dispatch(ocr_server: RecordingServer) -> None:
|
||||
order: Final = []
|
||||
finished: Final = threading.Event()
|
||||
|
||||
class Hooked(CustomLogger):
|
||||
def __init__(self, name: str) -> None:
|
||||
super().__init__()
|
||||
self.name = name
|
||||
|
||||
def logging_hook(self, kwargs, result, call_type):
|
||||
order.append(("hook", self.name))
|
||||
kwargs[f"seen-by-{self.name}"] = True
|
||||
return kwargs, result
|
||||
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
order.append(("log", self.name, kwargs.get("seen-by-a"), kwargs.get("seen-by-b")))
|
||||
if self.name == "b":
|
||||
finished.set()
|
||||
|
||||
call_native_ocr_with_callbacks(ocr_server, [Hooked("a"), Hooked("b")])
|
||||
|
||||
assert finished.wait(10)
|
||||
assert order == [("hook", "a"), ("hook", "b"), ("log", "a", True, True), ("log", "b", True, True)]
|
||||
|
||||
|
||||
def test_native_ocr_hook_failure_is_contained_and_target_still_dispatches(ocr_server: RecordingServer) -> None:
|
||||
order: Final = []
|
||||
finished: Final = threading.Event()
|
||||
|
||||
class Broken(CustomLogger):
|
||||
def logging_hook(self, kwargs, result, call_type):
|
||||
raise RuntimeError("hook failed")
|
||||
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
order.append("broken-log")
|
||||
|
||||
class Healthy(CustomLogger):
|
||||
def logging_hook(self, kwargs, result, call_type):
|
||||
order.append("healthy-hook")
|
||||
return kwargs, result
|
||||
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
order.append("healthy-log")
|
||||
finished.set()
|
||||
|
||||
call_native_ocr_with_callbacks(ocr_server, [Broken(), Healthy()])
|
||||
|
||||
assert finished.wait(10)
|
||||
assert order == ["healthy-hook", "broken-log", "healthy-log"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_aocr_hook_replacement_of_result_reaches_success_dispatch(ocr_server: RecordingServer) -> None:
|
||||
replacement: Final = object()
|
||||
observed: Final = []
|
||||
|
||||
class Replace(CustomLogger):
|
||||
async def async_logging_hook(self, kwargs, result, call_type):
|
||||
return kwargs, replacement
|
||||
|
||||
class Observe(CustomLogger):
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
observed.append(response_obj)
|
||||
|
||||
recorder: Final = RecordingLogger()
|
||||
await call_native_aocr_with_callbacks(ocr_server, [Replace(), Observe(), recorder])
|
||||
await recorder.wait_for_async("async_log_success_event")
|
||||
|
||||
assert observed == [replacement]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_aocr_shared_logging_object_dispatches_success_once(ocr_server: RecordingServer) -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
ocr_server.expected_requests = 2
|
||||
recorder: Final = RecordingLogger()
|
||||
litellm.callbacks.append(recorder)
|
||||
logger: Final = Logging(
|
||||
model="mistral-ocr-latest",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="aocr",
|
||||
start_time=__import__("datetime").datetime.now(),
|
||||
litellm_call_id="shared",
|
||||
function_id="shared",
|
||||
)
|
||||
logger.dynamic_async_success_callbacks = [recorder]
|
||||
|
||||
await call_native_aocr(ocr_server, litellm_logging_obj=logger)
|
||||
await call_native_aocr(ocr_server, litellm_logging_obj=logger)
|
||||
await recorder.wait_for_async("async_log_success_event")
|
||||
from tests.test_litellm_rust.support.callback_recorder import drain_logging
|
||||
|
||||
await drain_logging()
|
||||
|
||||
assert logger.model_call_details["has_logged_async_success"] is True
|
||||
assert recorder.names.count("async_log_success_event") == 1
|
||||
|
||||
|
||||
def test_native_ocr_logging_preparation_failure_does_not_fail_request(
|
||||
ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
def broken_payload(*args, **kwargs):
|
||||
raise RuntimeError("payload unavailable")
|
||||
|
||||
monkeypatch.setattr(litellm_logging, "get_standard_logging_object_payload", broken_payload)
|
||||
unraisable: Final = []
|
||||
monkeypatch.setattr(__import__("sys"), "unraisablehook", lambda event: unraisable.append(event.exc_value))
|
||||
recorder: Final = RecordingLogger()
|
||||
|
||||
response: Final = call_native_ocr_with_callbacks(ocr_server, [recorder])
|
||||
events: Final = recorder.wait_for("log_success_event")
|
||||
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
assert len(events) == 1
|
||||
assert events[0].thread is not threading.current_thread()
|
||||
assert any(str(error) == "payload unavailable" for error in unraisable)
|
||||
|
||||
|
||||
def test_native_ocr_writes_success_marker_and_honours_existing_marker(
|
||||
ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.rust_bridge import setup as native_setup
|
||||
|
||||
ocr_server.expected_requests = 2
|
||||
loggers: Final = []
|
||||
original_build: Final = native_setup.build_logging
|
||||
|
||||
def build_logging(**kwargs):
|
||||
logger = original_build(**kwargs)
|
||||
if loggers:
|
||||
logger.model_call_details["has_logged_sync_success"] = True
|
||||
loggers.append(logger)
|
||||
return logger
|
||||
|
||||
monkeypatch.setattr(native_setup, "build_logging", build_logging)
|
||||
recorder: Final = RecordingLogger()
|
||||
|
||||
call_native_ocr_with_callbacks(ocr_server, [recorder])
|
||||
recorder.wait_for("log_success_event")
|
||||
assert loggers[0].model_call_details["has_logged_sync_success"] is True
|
||||
|
||||
call_native_ocr_with_callbacks(ocr_server, [recorder])
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
|
||||
executor.submit(lambda: None).result(10)
|
||||
assert recorder.names.count("log_success_event") == 1
|
||||
assert recorder.names.count("logging_hook") == 1
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ async def test_native_bindings_run_callbacks_and_send_their_mutations(
|
|||
class Observe(Logging):
|
||||
def pre_call(self, input, api_key, additional_args):
|
||||
body: Final = additional_args["complete_input_dict"]
|
||||
assert body["pages"] is pages
|
||||
assert body["pages"] == pages and body["pages"] is not pages
|
||||
observed.append(self.model_call_details["litellm_params"]["metadata"]["marker"])
|
||||
body["pages"].append(2)
|
||||
additional_args["headers"]["x-callback"] = "native"
|
||||
|
|
@ -98,6 +98,7 @@ async def test_native_bindings_run_callbacks_and_send_their_mutations(
|
|||
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
assert observed == [marker] and observed[0] is marker
|
||||
assert pages == [0]
|
||||
assert len(ocr_server.requests) == 1
|
||||
assert ocr_server.requests[0].body["pages"] == [0, 2]
|
||||
assert ocr_server.requests[0].headers["x-callback"] == "native"
|
||||
|
|
@ -263,7 +264,6 @@ async def test_deployment_hook_replaces_complete_routing_request(ocr_server: Rec
|
|||
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
assert observed == [(replacement, "replacement-key")]
|
||||
assert observed[0][0] is replacement
|
||||
assert replacement == original
|
||||
assert replacement is not original
|
||||
assert original == {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}
|
||||
|
|
@ -621,7 +621,7 @@ def test_sync_pre_call_can_make_nested_native_request(ocr_server: RecordingServe
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retained_argument_aliases_and_body_roots_survive_envelope_replacement(
|
||||
async def test_callback_body_roots_survive_envelope_replacement(
|
||||
ocr_server: RecordingServer,
|
||||
) -> None:
|
||||
pages: Final = [0]
|
||||
|
|
@ -633,8 +633,8 @@ async def test_retained_argument_aliases_and_body_roots_survive_envelope_replace
|
|||
def pre_call(self, input, api_key, additional_args):
|
||||
body: Final = additional_args["complete_input_dict"]
|
||||
headers: Final = additional_args["headers"]
|
||||
observed.append((body["document"] is document, body["pages"] is pages))
|
||||
pages.append(2)
|
||||
observed.append((body["document"] == document, body["pages"] == pages))
|
||||
body["pages"].append(2)
|
||||
headers["x-retained"] = "yes"
|
||||
additional_args["complete_input_dict"] = {"discarded": True}
|
||||
additional_args["headers"] = {}
|
||||
|
|
@ -672,6 +672,7 @@ async def test_retained_argument_aliases_and_body_roots_survive_envelope_replace
|
|||
assert observed[0] == (False, False, True)
|
||||
assert observed[1] == (True, True)
|
||||
assert observed[3] == (True, True)
|
||||
assert pages == [0]
|
||||
assert ocr_server.requests[0].body["pages"] == [0, 2]
|
||||
assert ocr_server.requests[0].headers["x-retained"] == "yes"
|
||||
|
||||
|
|
@ -933,72 +934,6 @@ async def test_response_limit_is_enforced_at_the_public_boundary(
|
|||
assert "max_response_bytes" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("failure", [False, True])
|
||||
async def test_empty_callbacks_keep_bookkeeping_without_optional_dispatch(
|
||||
ocr_server: RecordingServer,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
asynchronous: bool,
|
||||
failure: bool,
|
||||
created_loggers: list[Logging],
|
||||
) -> None:
|
||||
from litellm import utils
|
||||
from litellm.litellm_core_utils import litellm_logging, logging_worker
|
||||
|
||||
class DispatchProbe:
|
||||
deployments = 0
|
||||
submissions = 0
|
||||
enqueues = 0
|
||||
|
||||
def deployment(self, *args: object, **kwargs: object) -> None:
|
||||
self.deployments += 1
|
||||
|
||||
def submit(self, *args: object, **kwargs: object) -> None:
|
||||
self.submissions += 1
|
||||
|
||||
def ensure_initialized_and_enqueue(self, coroutine: Coroutine[object, object, object]) -> None:
|
||||
self.enqueues += 1
|
||||
coroutine.close()
|
||||
|
||||
probe: Final = DispatchProbe()
|
||||
for name in (
|
||||
"async_pre_call_deployment_hook",
|
||||
"async_post_call_success_deployment_hook",
|
||||
"async_post_call_failure_deployment_hook",
|
||||
):
|
||||
monkeypatch.setattr(utils, name, probe.deployment)
|
||||
monkeypatch.setattr(litellm_logging, "executor", probe)
|
||||
monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", probe)
|
||||
if failure:
|
||||
ocr_server.enqueue(ResponseSpec(body={"message": "provider failed"}, status=500))
|
||||
trace_id_var.set("callback-free-parent")
|
||||
arguments: Final = {"litellm_trace_id": "callback-free-call", "litellm_call_id": "callback-free-id"}
|
||||
if failure:
|
||||
with pytest.raises(litellm.InternalServerError):
|
||||
await call_aocr(ocr_server, **arguments) if asynchronous else call_ocr(ocr_server, **arguments)
|
||||
else:
|
||||
response: Final = (
|
||||
await call_aocr(ocr_server, **arguments) if asynchronous else call_ocr(ocr_server, **arguments)
|
||||
)
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
assert response._hidden_params["litellm_call_id"] == "callback-free-id"
|
||||
assert response._hidden_params["response_cost"] is not None
|
||||
assert response._hidden_params["_response_ms"] > 0
|
||||
assert trace_id_var.get() == "callback-free-parent"
|
||||
assert probe.deployments == probe.submissions == probe.enqueues == 0
|
||||
assert len(created_loggers) == 1
|
||||
logger: Final = created_loggers[0]
|
||||
assert not hasattr(logger, "_native_pending_logging")
|
||||
assert logger.model_call_details["first_api_call_start_time"] <= logger.model_call_details["end_time"]
|
||||
assert "standard_logging_object" not in logger.model_call_details
|
||||
assert (
|
||||
"original_response" not in logger.model_call_details or logger.model_call_details["original_response"] is None
|
||||
)
|
||||
assert "complete_input_dict" not in logger.model_call_details.get("additional_args", {})
|
||||
assert logger.model_call_details["response_cost"] == (0 if failure else response._hidden_params["response_cost"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"registration", ["success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"]
|
||||
|
|
@ -1032,26 +967,18 @@ async def test_terminal_registration_added_during_http_is_observed(
|
|||
|
||||
@pytest.fixture
|
||||
def created_loggers(monkeypatch: pytest.MonkeyPatch) -> list[Logging]:
|
||||
from litellm import utils
|
||||
from litellm.rust_bridge import setup as native_setup
|
||||
|
||||
original_setup: Final = utils.function_setup
|
||||
original_build: Final = native_setup.build_logging
|
||||
loggers: Final[list[Logging]] = []
|
||||
|
||||
def setup(
|
||||
call_type: str,
|
||||
rules: utils.Rules,
|
||||
start: datetime.datetime,
|
||||
*args: object,
|
||||
is_async_call: bool = True,
|
||||
**kwargs: object,
|
||||
) -> tuple[Logging, dict[str, object]]:
|
||||
logger, prepared = original_setup(call_type, rules, start, *args, is_async_call=is_async_call, **kwargs)
|
||||
assert isinstance(logger, Logging)
|
||||
def build_logging(**kwargs: object) -> Logging:
|
||||
logger: Final = original_build(**kwargs) # pyright: ignore[reportArgumentType] # passthrough of the factory signature
|
||||
setattr(logger, "_defer_async_logging", True)
|
||||
loggers.append(logger)
|
||||
return logger, prepared
|
||||
return logger
|
||||
|
||||
monkeypatch.setattr(utils, "function_setup", setup)
|
||||
monkeypatch.setattr(native_setup, "build_logging", build_logging)
|
||||
return loggers
|
||||
|
||||
|
||||
|
|
@ -1076,30 +1003,3 @@ async def test_explicit_logging_consumers_keep_request_and_response_payloads(
|
|||
assert details["raw_request_typed_dict"]["raw_request_body"]["model"] == "mistral-ocr-latest"
|
||||
if consumer == "logger_fn":
|
||||
assert [item["log_event_type"] for item in snapshots] == ["pre_api_call", "post_api_call"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registration_removed_before_deferred_release_skips_queue(
|
||||
ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch, created_loggers: list[Logging]
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils import logging_worker
|
||||
|
||||
class QueueProbe:
|
||||
enqueues = 0
|
||||
|
||||
def ensure_initialized_and_enqueue(self, coroutine: Coroutine[object, object, object]) -> None:
|
||||
self.enqueues += 1
|
||||
coroutine.close()
|
||||
|
||||
observer: Final = RecordingLogger()
|
||||
litellm._async_success_callback.append(observer)
|
||||
await call_aocr(ocr_server)
|
||||
logger: Final = created_loggers[0]
|
||||
assert hasattr(logger, "_native_pending_logging")
|
||||
litellm._async_success_callback.clear()
|
||||
probe: Final = QueueProbe()
|
||||
monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", probe)
|
||||
ProxyBaseLLMRequestProcessing._flush_deferred_async_logging(logger, False)
|
||||
assert probe.enqueues == 0
|
||||
assert not observer.names
|
||||
assert logger.model_call_details["response_cost"] is not None
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from tests.test_litellm_rust.support.requests import (
|
|||
OCR_RESPONSE,
|
||||
call_native_aocr,
|
||||
call_native_ocr,
|
||||
request_body,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
|
@ -535,7 +536,7 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen
|
|||
class Observer(RecordingLogger):
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
super().log_pre_api_call(model, messages, kwargs)
|
||||
pages.append(2)
|
||||
request_body(kwargs)["pages"].append(2)
|
||||
|
||||
arguments: Final = {
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
|
|
@ -548,3 +549,4 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen
|
|||
assert response.pages[0].markdown == "native OCR response"
|
||||
assert ocr_server.requests[0].headers["authorization"] == f"Bearer {expected_key}"
|
||||
assert ocr_server.requests[0].body["pages"] == [0, 2]
|
||||
assert pages == [0]
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
import json
|
||||
import asyncio
|
||||
import base64
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import weakref
|
||||
import gc
|
||||
from pathlib import Path
|
||||
from collections.abc import Generator
|
||||
from collections.abc import Coroutine, Generator
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Final, Protocol
|
||||
|
||||
import httpx
|
||||
|
|
@ -340,7 +341,12 @@ def test_native_lifecycle_core_encodes_python_file_input(
|
|||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("source", ["path", "pathlike", "reader", "text", "bytes"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, asynchronous, source):
|
||||
async def test_native_file_sources_preserve_sdk_behavior(
|
||||
ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]],
|
||||
tmp_path: Path,
|
||||
asynchronous: bool,
|
||||
source: str,
|
||||
) -> None:
|
||||
server, requests = ocr_server
|
||||
path: Final = tmp_path / "scan.png"
|
||||
content: Final = b"document bytes"
|
||||
|
|
@ -388,9 +394,12 @@ async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, a
|
|||
"api_base": f"http://127.0.0.1:{server.server_port}",
|
||||
}
|
||||
response: Final = await litellm.aocr(**kwargs) if asynchronous else litellm.ocr(**kwargs)
|
||||
assert isinstance(response, OCRResponse)
|
||||
assert response.pages[0].markdown == "native OCR response"
|
||||
assert len(requests) == 1
|
||||
assert requests[0]["body"]["document"] == {
|
||||
body: Final = requests[0]["body"]
|
||||
assert isinstance(body, dict)
|
||||
assert body["document"] == {
|
||||
"type": "image_url",
|
||||
"image_url": "data:image/png;base64," + base64.b64encode(content).decode(),
|
||||
}
|
||||
|
|
@ -400,7 +409,11 @@ async def test_native_file_sources_preserve_sdk_behavior(ocr_server, tmp_path, a
|
|||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_file_failures_preserve_identity_and_do_not_send(ocr_server, tmp_path, asynchronous):
|
||||
async def test_native_file_failures_preserve_identity_and_do_not_send(
|
||||
ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]],
|
||||
tmp_path: Path,
|
||||
asynchronous: bool,
|
||||
) -> None:
|
||||
from litellm.rust_bridge import _native
|
||||
|
||||
server, requests = ocr_server
|
||||
|
|
@ -416,37 +429,47 @@ async def test_native_file_failures_preserve_identity_and_do_not_send(ocr_server
|
|||
|
||||
reader: Final = Reader()
|
||||
path: Final = tmp_path / "missing.pdf"
|
||||
kwargs: Final = {
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"api_key": "test-key",
|
||||
"api_base": f"http://127.0.0.1:{server.server_port}",
|
||||
}
|
||||
|
||||
async def invoke(source: Reader | Path) -> None:
|
||||
if asynchronous:
|
||||
await _native.aocr(
|
||||
model="mistral/mistral-ocr-latest",
|
||||
document={"type": "file", "file": source},
|
||||
api_key="test-key",
|
||||
api_base=f"http://127.0.0.1:{server.server_port}",
|
||||
)
|
||||
else:
|
||||
_native.ocr(
|
||||
model="mistral/mistral-ocr-latest",
|
||||
document={"type": "file", "file": source},
|
||||
api_key="test-key",
|
||||
api_base=f"http://127.0.0.1:{server.server_port}",
|
||||
)
|
||||
|
||||
for source, error in ((reader, KeyError), (path, FileNotFoundError)):
|
||||
with pytest.raises(error) as caught:
|
||||
if asynchronous:
|
||||
await _native.aocr(document={"type": "file", "file": source}, **kwargs)
|
||||
else:
|
||||
_native.ocr(document={"type": "file", "file": source}, **kwargs)
|
||||
await invoke(source)
|
||||
if source is reader:
|
||||
assert caught.value is failure
|
||||
else:
|
||||
assert caught.value.filename == str(path)
|
||||
assert isinstance(caught.value, FileNotFoundError)
|
||||
assert getattr(caught.value, "filename") == str(path)
|
||||
assert reader.calls == 1
|
||||
assert requests == []
|
||||
|
||||
|
||||
def test_unstarted_native_file_call_does_not_read_and_releases_reader():
|
||||
def test_unstarted_native_file_call_does_not_read_and_releases_reader() -> None:
|
||||
from litellm.rust_bridge import _native
|
||||
|
||||
class Reader:
|
||||
pending: Coroutine[object, object, OCRResponse] | None = None
|
||||
|
||||
def read(self) -> bytes:
|
||||
raise AssertionError("unstarted call consumed its document")
|
||||
|
||||
def create_call():
|
||||
def create_call() -> tuple[Coroutine[object, object, OCRResponse], weakref.ReferenceType[Reader]]:
|
||||
reader: Final = Reader()
|
||||
pending: Final = _native.aocr(
|
||||
model="mistral/mistral-ocr-latest", document={"type": "file", "file": reader}
|
||||
)
|
||||
pending: Final = _native.aocr(model="mistral/mistral-ocr-latest", document={"type": "file", "file": reader})
|
||||
reader.pending = pending
|
||||
return pending, weakref.ref(reader)
|
||||
|
||||
|
|
@ -457,6 +480,49 @@ def test_unstarted_native_file_call_does_not_read_and_releases_reader():
|
|||
assert reference() is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="requires Unix named pipes")
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_path_cancellation_waits_for_read_completion(
|
||||
ocr_server: tuple[ThreadingHTTPServer, list[dict[str, object]]], tmp_path: Path
|
||||
) -> None:
|
||||
from litellm.rust_bridge import _native
|
||||
|
||||
server, requests = ocr_server
|
||||
path: Final = tmp_path / "document.pdf"
|
||||
os.mkfifo(path)
|
||||
entered: Final = threading.Event()
|
||||
release: Final = threading.Event()
|
||||
|
||||
def supply_document() -> None:
|
||||
with path.open("wb") as stream:
|
||||
stream.write(b"document")
|
||||
stream.flush()
|
||||
entered.set()
|
||||
release.wait(5)
|
||||
|
||||
writer: Final = threading.Thread(target=supply_document, daemon=True)
|
||||
writer.start()
|
||||
task: Final = asyncio.create_task(
|
||||
_native.aocr(
|
||||
model="mistral/mistral-ocr-latest",
|
||||
document={"type": "file", "file": path},
|
||||
api_key="test-key",
|
||||
api_base=f"http://127.0.0.1:{server.server_port}",
|
||||
)
|
||||
)
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 3)
|
||||
task.cancel()
|
||||
await asyncio.sleep(0.05)
|
||||
assert not task.done()
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.to_thread(writer.join, 3)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert requests == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("model", ["mistral/mistral-ocr-latest", "azure_ai/doc-intelligence/prebuilt-read"])
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue