diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ed658e074ee..d5fc960b39f 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4158,6 +4158,7 @@ dependencies = [ "futures-util", "litellm-auth", "litellm-auth-aws", + "litellm-cache", "litellm-cache-memory", "litellm-cache-response", "litellm-core-utils", diff --git a/litellm-rust/crates/cache-response/src/service.rs b/litellm-rust/crates/cache-response/src/service.rs index 0bdf948ec48..ed4dccdc355 100644 --- a/litellm-rust/crates/cache-response/src/service.rs +++ b/litellm-rust/crates/cache-response/src/service.rs @@ -27,6 +27,10 @@ impl Default for ResponseCacheConfig { pub trait ResponseCacheService: Send + Sync { fn config(&self) -> &ResponseCacheConfig; + fn keyed_by_caller(&self) -> bool { + false + } + fn lookup<'a>( &'a self, request: &'a ResponseCacheRequest, diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index 8a212223523..4c2adcec49f 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -11,7 +11,8 @@ "logger", "kwargs", "start_time", - "end_time" + "end_time", + "cache_hit" ], "update_logging": [ "logger", @@ -89,6 +90,12 @@ "start", "end" ], + "cache_hit_success": [ + "logger", + "response", + "start", + "end" + ], "enqueue_logging": [ "coroutine" ], @@ -120,7 +127,8 @@ "logger", "kwargs", "start_time", - "end_time" + "end_time", + "cache_hit" ], "stream_success": [ "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index f68541c5c7b..6dda15490d2 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -264,6 +264,7 @@ impl LegacyLogging { self.call.kwargs(), &self.start, &self.end, + self.messages_cache_hit(), )?; self.response .as_ref() @@ -271,6 +272,25 @@ impl LegacyLogging { .ok_or_else(|| HostError::MissingState.into()) } + fn messages_cache_hit(&self) -> bool { + self.operation == LoggingOperation::Messages && self.cache_key.is_some() + } + + fn dispatch_cache_hit(&self, py: Python<'_>) -> PyResult<()> { + let logger = self.logger()?; + let logged = Logging::CacheHitSuccess.call( + py, + (logger.object(py), &self.response, &self.start, &self.end), + ); + match logged { + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(logger.object(py))); + Ok(()) + } + result => result.map(|_| ()), + } + } + fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> { match self.try_dispatch_success(py) { Err(error) if error.is_instance_of::(py) => { @@ -567,7 +587,7 @@ impl LegacyLogging { ) -> PyResult>> { self.end = Some(datetime(py, timing.end_time)?); self.response = Some(response); - if self.runs_deployment_hooks() { + if self.runs_deployment_hooks() && !self.messages_cache_hit() { return Ok(HookStep::Await( DeploymentHooks::after_success( py, @@ -588,7 +608,17 @@ impl LegacyLogging { ) -> PyResult> { use litellm_host::interceptors::{Cost, ResultSource}; + self.cache_key = match &facts.source { + ResultSource::Provider => None, + ResultSource::Cache { key } => Some(key.clone()), + }; let logger = self.logger()?.object(py); + if self.messages_cache_hit() { + logger + .getattr("model_call_details")? + .set_item("cache_hit", true)?; + return Ok(HookStep::Ready(())); + } let params = logger .getattr("litellm_params")? .cast_into::()? @@ -616,10 +646,6 @@ impl LegacyLogging { }; logger.setattr("_provider_reported_cost", reported_cost)?; details.call_method1("pop", ("response_cost", py.None()))?; - self.cache_key = match &facts.source { - ResultSource::Provider => None, - ResultSource::Cache { key } => Some(key.clone()), - }; if self.cache_key.is_some() { details.set_item("cache_hit", true)?; } else { @@ -674,6 +700,7 @@ impl LegacyLogging { self.response = Some(response.clone_ref(py)); match &self.stream { Some(stream) => self.stream_success(py, stream, false)?, + None if self.messages_cache_hit() => self.dispatch_cache_hit(py)?, None => self.dispatch_success(py)?, } Ok(HookStep::Ready(())) @@ -708,7 +735,11 @@ impl LegacyLogging { if self.stream_billing().is_none() { return Err(HostError::MissingState.into()); } - if let Some(key) = &self.cache_key { + if let Some(key) = self + .cache_key + .as_ref() + .filter(|_| !self.messages_cache_hit()) + { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } @@ -723,6 +754,7 @@ impl LegacyLogging { self.call.kwargs(), &self.start, datetime(py, epoch_seconds())?, + self.messages_cache_hit(), ), )?; } @@ -747,7 +779,7 @@ impl LegacyLogging { // A caller that goes away mid-stream cancels the call before it ends: Python's Messages // stream iterator still bills the chunks it read, straight away rather than deferred. fn bill_abandoned_stream(&mut self, py: Python<'_>) -> PyResult<()> { - if !self.asynchronous || self.end.is_some() { + if !self.asynchronous || self.end.is_some() || self.messages_cache_hit() { return Ok(()); } let Some(stream) = self.stream.take() else { diff --git a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs index 38f3bf29828..3019d87a20b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs @@ -77,8 +77,12 @@ pub fn finalize( kwargs: &Py, start: &Py, end: &Option>, + cache_hit: bool, ) -> PyResult<()> { - Wrapper::Finalize.call(py, (response, logger.object(py), kwargs, start, end))?; + Wrapper::Finalize.call( + py, + (response, logger.object(py), kwargs, start, end, cache_hit), + )?; Ok(()) } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index 5b6faa3dbc2..fc1918eea31 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -62,6 +62,8 @@ pub(crate) enum Logging { SubmitSuccess, #[strum(serialize = "async_success_handler")] AsyncSuccessHandler, + #[strum(serialize = "cache_hit_success")] + CacheHitSuccess, #[strum(serialize = "mark_async_success_enqueued")] MarkAsyncSuccessEnqueued, #[strum(serialize = "enqueue_logging")] diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index ee16e0c0059..bcf98f1f41c 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -53,7 +53,7 @@ def setup(call_type, args, kwargs, start, asynchronous): FAKES = { 'setup': setup, - 'finalize': lambda response, logger, kwargs, start, end: logger.record('finalize', response), + 'finalize': lambda response, logger, kwargs, start, end, cache_hit: logger.record('finalize', response), 'update_logging': lambda logger, kwargs, model, optional_params, litellm_params, provider: logger.update_from_kwargs( kwargs=kwargs, model=model, @@ -86,6 +86,7 @@ FAKES = { )(error, ''.join(traceback.format_exception(error)), start, end), 'submit_success': lambda logger, response, start, end: logger.record('submit', (response, start, end)), 'async_success_handler': lambda logger, response, start, end: logger.async_success_handler(response, start, end), + 'cache_hit_success': lambda logger, response, start, end: logger.record('cache_hit_success', response), 'mark_async_success_enqueued': lambda logger: logger.record('mark_async_success_enqueued', None), 'enqueue_logging': lambda coroutine: coroutine.enqueue(), 'restore_context': lambda logger: logger.record('restore', None), @@ -97,7 +98,7 @@ FAKES = { ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), 'stream_opened': lambda logger, head: logger.record('stream_opened', head), - 'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head), + 'stamp_stream_head': lambda head, logger, kwargs, start, end, cache_hit: logger.record('stamp_stream_head', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml index 0c5be52550d..c0b6cd8a997 100644 --- a/litellm-rust/crates/inference-messages/Cargo.toml +++ b/litellm-rust/crates/inference-messages/Cargo.toml @@ -30,6 +30,7 @@ tracing.workspace = true [dev-dependencies] litellm-error.workspace = true litellm-framer = { workspace = true, features = ["sse"] } +litellm-cache.workspace = true litellm-cache-memory.workspace = true litellm-host-native.workspace = true litellm-http = { workspace = true, features = ["test-support"] } diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index c7a7d93bf98..e003be64583 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -35,7 +35,11 @@ use super::{ route::Messages, types::invalid_request, }; -use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request}; +use litellm_inference::{ + caching::{CacheRequest, CallCache}, + context::CallContext, + outbound::outbound_request, +}; struct ProviderCall { identity: ProviderIdentity, @@ -52,6 +56,42 @@ impl ProviderCall { } } +pub(super) async fn cached( + route: &MessagesRoute, + provider: MessagesProvider, + model: &str, + context: &CallContext<'_, impl Interceptors>, +) -> Result, Error> { + let Some(cache) = caller_keyed(route) else { + return Ok(None); + }; + let identity = ProviderIdentity { + model: model.to_owned(), + provider: provider.as_str().to_string(), + }; + let Some((output, source)) = CallCache::::new( + Some(cache), + context.cache, + CacheRequest::from_wire(identity.clone(), None), + ) + .lookup() + .await + else { + return Ok(None); + }; + context + .result_ready(execution_facts(provider, identity, &output, &source)) + .await?; + Ok(Some(output)) +} + +fn caller_keyed(route: &MessagesRoute) -> Option<&litellm_cache_response::ScopedCache> { + route + .cache + .as_ref() + .filter(|cache| cache.service.keyed_by_caller()) +} + pub(super) async fn execute( route: &MessagesRoute, provider: MessagesProvider, @@ -60,6 +100,7 @@ pub(super) async fn execute( secrets: Secrets, context: &CallContext<'_, impl Interceptors>, ) -> Result { + let caller_keyed = caller_keyed(route).is_some(); let request = provider_call( &route.auth, provider, @@ -71,34 +112,50 @@ pub(super) async fn execute( .boxed() .await?; let cache = CallCache::::from_wire( - route.cache.as_ref().filter(|_| request.cacheable()), + route + .cache + .as_ref() + .filter(|_| caller_keyed || request.cacheable()), context.cache, &request.identity, &request.wire, ); let identity = request.identity.clone(); - let (output, source) = match cache.lookup().await { + let hit = match caller_keyed { + true => None, + false => cache.lookup().await, + }; + let (output, source) = match hit { Some(hit) => hit, None => ( call_provider(&route.http, request, context).await?, ResultSource::Provider, ), }; - let reported = match &output { + context + .result_ready(execution_facts(provider, identity, &output, &source)) + .await?; + Ok(cache.finish(output, &source).await) +} + +fn execution_facts( + provider: MessagesProvider, + identity: ProviderIdentity, + output: &MessagesCallResponse, + source: &ResultSource, +) -> ExecutionFacts { + let reported = match output { MessagesCallResponse::Complete(response) => match response { MessagesBody::Message(message) => provider.config().reported_cost(message), MessagesBody::Unchecked(_) => None, }, MessagesCallResponse::Stream { .. } => None, }; - context - .result_ready(ExecutionFacts { - provider: identity, - cost: Cost::settle(&source, reported), - source: source.clone(), - }) - .await?; - Ok(cache.finish(output, &source).await) + ExecutionFacts { + provider: identity, + cost: Cost::settle(source, reported), + source: source.clone(), + } } async fn provider_call( diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index 25ebeaacb49..2dc4be959c4 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -89,6 +89,9 @@ impl MessagesRoute { <&'static str>::from(provider.provider), ); let model = provider.model.to_owned(); + if let Some(output) = handler::cached(self, config, &model, &context).await? { + return Ok(output); + } let secrets = self.secrets.resolve(config.config().secret_names()).await?; handler::execute(self, config, model, call, secrets, &context) .boxed() diff --git a/litellm-rust/crates/inference-messages/tests/caching.rs b/litellm-rust/crates/inference-messages/tests/caching.rs index 787501a8fed..786d86c3ff3 100644 --- a/litellm-rust/crates/inference-messages/tests/caching.rs +++ b/litellm-rust/crates/inference-messages/tests/caching.rs @@ -301,3 +301,89 @@ async fn provider_reported_cost_is_delivered_once_and_never_charged_for_a_cache_ assert!(matches!(cached.source, ResultSource::Cache { .. })); assert_eq!(cached.cost, Cost::CacheHit); } + +#[derive(Default)] +struct CallerKeyed { + config: ResponseCacheConfig, + entry: std::sync::Mutex>, +} + +impl ResponseCacheService for CallerKeyed { + fn config(&self) -> &ResponseCacheConfig { + &self.config + } + + fn keyed_by_caller(&self) -> bool { + true + } + + fn lookup<'a>( + &'a self, + _: &'a litellm_cache_response::ResponseCacheRequest, + _: Duration, + ) -> std::pin::Pin< + Box< + dyn std::future::Future, litellm_cache::Error>> + + Send + + 'a, + >, + > { + Box::pin(async move { Ok(self.entry.lock().unwrap().clone()) }) + } + + fn store<'a>( + &'a self, + _: &'a litellm_cache_response::ResponseCacheRequest, + value: Value, + _: Duration, + ) -> std::pin::Pin< + Box> + Send + 'a>, + > { + Box::pin(async move { + *self.entry.lock().unwrap() = Some(value); + Ok(()) + }) + } +} + +#[tokio::test] +async fn a_caller_keyed_hit_is_served_before_the_request_is_built_or_intercepted() { + use litellm_inference_messages::{MessagesCall, MessagesCallResponse}; + + let response = json!({"id": "msg_caller", "type": "message", "role": "assistant", "model": "native-test", + "content": [{"type": "text", "text": "answer"}], "stop_reason": "end_turn", "stop_sequence": null, + "usage": {"input_tokens": 3, "output_tokens": 2}}); + let upstream = support::upstream([support::json_response(response)]).await; + let hooks = ChangingHooks::default(); + let route = support::messages_route(litellm_inference_testing::no_secrets()).with_cache( + ScopedCache::new(Arc::new(CallerKeyed::default()), CacheScope::Shared), + ); + let request = |api_key: Option<&str>| { + MessagesCall { + body: serde_json::from_value(json!({"model": "native-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})).unwrap(), + api_key: api_key.map(Into::into), api_base: Some(upstream.uri()), custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), extra_headers: None, provider_specific_header: None, + timeout: None, shaping: Default::default(), + } + }; + let first = route + .execute(request(Some("key")), &hooks, None) + .await + .unwrap(); + let second = route.execute(request(None), &hooks, None).await.unwrap(); + let (MessagesCallResponse::Complete(first), MessagesCallResponse::Complete(second)) = + (first, second) + else { + panic!("expected Messages responses"); + }; + assert_eq!(first, second); + assert_eq!(support::received(&upstream).await.len(), 1); + assert_eq!(hooks.calls.load(Ordering::SeqCst), 1); + let facts = hooks.facts.lock().unwrap(); + let [provider, cached] = facts.as_slice() else { + panic!("expected provider and cache results"); + }; + assert_eq!(provider.source, ResultSource::Provider); + assert!(matches!(cached.source, ResultSource::Cache { .. })); + assert_eq!(cached.cost, Cost::CacheHit); +} diff --git a/litellm-rust/crates/inference/AGENTS.md b/litellm-rust/crates/inference/AGENTS.md index bcbc98a209e..bc9be4825ea 100644 --- a/litellm-rust/crates/inference/AGENTS.md +++ b/litellm-rust/crates/inference/AGENTS.md @@ -81,3 +81,4 @@ - Cache lookup follows provider preparation, credential resolution, and the request interceptor - keys describe the effective provider URL, authenticated headers, and rewritten body - signed requests bypass caching until the signing identity has a stable cache representation +- A service that keys entries by the caller's arguments (`keyed_by_caller`, the Python host's legacy cache) is looked up before provider preparation, as Python's `@client` wrapper does, and a hit skips request building and the request interceptor diff --git a/litellm-rust/crates/inference/src/caching.rs b/litellm-rust/crates/inference/src/caching.rs index d9b5b891015..b80fdd46714 100644 --- a/litellm-rust/crates/inference/src/caching.rs +++ b/litellm-rust/crates/inference/src/caching.rs @@ -238,12 +238,19 @@ impl CallCache

{ identity: &ProviderIdentity, wire: &WireRequest, ) -> Self { + Self::new( + cache, + policy, + CacheRequest::from_wire(identity.clone(), Some(wire)), + ) + } + + pub fn new(cache: Option<&ScopedCache>, policy: CachePolicy, request: CacheRequest) -> Self { let session = cache.and_then(|cache| { if !policy.enabled() { return None; } let options = cache.options(Some(policy)); - let request = CacheRequest::from_wire(identity.clone(), Some(wire)); Some(CacheSession { request: options.request( &cache.service.config().namespace, diff --git a/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md b/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md index 3b516251f43..9aba11a3376 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md +++ b/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md @@ -1,6 +1,6 @@ # Python cache delegation -This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` calls the Python cache's sync or async API and delivers the result back to Rust. +This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` delivers each operation's result back to Rust. Asynchronous lookups and stream stores go through `litellm.rust_bridge.host.cache`, which reads with the `LLMCachingHandler` the `@client` wrapper uses, for the legacy function the route names, and writes with its dual cache. `configure` disables the cache for synchronous calls. Keep the shared inference protocol wrapper and adapter selection in the parent module. Receive the configured cache and prepared arguments from the parent module. Do not discover global configuration, choose native backends, or move inference to Python. Core remains independent of Python objects and cache implementation details diff --git a/litellm-rust/crates/python-bridge/src/cache/python/host.rs b/litellm-rust/crates/python-bridge/src/cache/python/host.rs index a8cbd0ce607..10e16a54547 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/host.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/host.rs @@ -18,6 +18,7 @@ enum Pending { pub(crate) struct PythonCache { cache: Option>, + function: Option>, arguments: Option>, pending: Option, asynchronous: bool, @@ -28,6 +29,7 @@ impl PythonCache { pub fn new(asynchronous: bool) -> Self { Self { cache: None, + function: None, arguments: None, pending: None, asynchronous, @@ -35,12 +37,18 @@ impl PythonCache { } } + pub(in crate::cache) fn asynchronous(&self) -> bool { + self.asynchronous + } + pub(in crate::cache) fn bind( &mut self, cache: Bound<'_, PyAny>, + function: &Bound<'_, PyAny>, arguments: &Bound<'_, PyDict>, ) { self.cache = Some(cache.unbind()); + self.function = Some(function.clone().unbind()); self.arguments = Some(arguments.clone().unbind()); } @@ -59,7 +67,7 @@ impl PythonCache { self.pending = Some(Pending::Lookup(reply)); ( if self.asynchronous { - "async_get_cache" + "lookup" } else { "get_cache" }, @@ -77,7 +85,7 @@ impl PythonCache { self.pending = Some(Pending::Store(reply)); ( if self.asynchronous { - "async_add_cache" + "store_stream" } else { "add_cache" }, @@ -86,9 +94,20 @@ impl PythonCache { } }; let result = match result { + Some(value) if self.asynchronous => py + .import(CACHE_HOST_MODULE)? + .getattr(method)? + .call((cache, value), Some(&arguments)), Some(value) => cache .bind(py) .call_method(method, (value,), Some(&arguments)), + None if self.asynchronous => py.import(CACHE_HOST_MODULE)?.getattr(method)?.call( + ( + cache, + self.function.as_ref().ok_or(HandleError::Unconfigured)?, + ), + Some(&arguments), + ), None => cache.bind(py).call_method(method, (), Some(&arguments)), } .map(Bound::unbind); @@ -148,11 +167,13 @@ impl PythonCache { self.pending = None; self.deferred = false; self.cache = None; + self.function = None; self.arguments = None; } pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.cache)?; + visit.call(&self.function)?; visit.call(&self.arguments) } } diff --git a/litellm-rust/crates/python-bridge/src/cache/python/service.rs b/litellm-rust/crates/python-bridge/src/cache/python/service.rs index 018c8647043..618d990bc8e 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/service.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/service.rs @@ -79,6 +79,10 @@ where &self.config } + fn keyed_by_caller(&self) -> bool { + true + } + fn lookup<'a>( &'a self, _: &'a ResponseCacheRequest, diff --git a/litellm-rust/crates/python-bridge/src/cache/selection.rs b/litellm-rust/crates/python-bridge/src/cache/selection.rs index 78c1606ea40..66321a8ae87 100644 --- a/litellm-rust/crates/python-bridge/src/cache/selection.rs +++ b/litellm-rust/crates/python-bridge/src/cache/selection.rs @@ -90,9 +90,14 @@ pub(crate) fn configure( python: &mut python::PythonCache, py: Python<'_>, arguments: &Bound<'_, PyDict>, - call_type: &str, + function: &Bound<'_, PyAny>, ) -> PyResult { - let selected = selected_cache(py, arguments, call_type)?; + let call_type: String = function.getattr("__name__")?.extract()?; + let selected = if python.asynchronous() { + selected_cache(py, arguments, &call_type)? + } else { + None + }; let Some(cache) = selected else { return Ok(Selection { backend: Backend::Disabled, @@ -129,7 +134,7 @@ pub(crate) fn configure( .map(|value| value.extract()) .transpose()? .unwrap_or_default(); - python.bind(cache, arguments); + python.bind(cache, function, arguments); Ok(Selection { backend: if enabled { Backend::Python { namespace } diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 2a5f79ca393..d8cfdf60a0f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -227,9 +227,12 @@ impl PythonBinding for MessagesPythonHost { self.view .resolve(py, arguments) .map_err(InvokeError::Python)?; - let selection = - crate::cache::configure(&mut self.cache, py, arguments, "anthropic_messages") - .map_err(InvokeError::Python)?; + let function = py + .import(ROUTE_HOST_MODULE) + .and_then(|module| module.call_method0("cached_function")) + .map_err(InvokeError::Python)?; + let selection = crate::cache::configure(&mut self.cache, py, arguments, &function) + .map_err(InvokeError::Python)?; self.projection(py, arguments) .map_err(|error| InvokeError::Python(self.map_failure(py, error)))? .map_err(InvokeError::Native) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index ab5d2332c21..f87031513cd 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -19,6 +19,7 @@ import datetime import inspect import time from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError @@ -78,6 +79,19 @@ from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper EmbeddingCacheInputElement = str | list[int] | ChatCompletionFileObject +@dataclass(frozen=True, slots=True) +class CacheRead: + kwargs: dict[str, object] # mutable-ok: the @client call kwargs the embedding path keeps reading and rewriting + checked: bool = False + cached_result: object | None = None + model: str = "" + end_time: datetime.datetime | None = None + + @property + def hit(self) -> bool: + return self.end_time is not None + + class CachingHandlerResponse(LiteLLMBaseModel): """ This is the response object for the caching handler. We need to separate embedding cached responses and (completion / text_completion / transcription) cached responses @@ -395,121 +409,146 @@ class LLMCachingHandler: Raises: None """ + if not self.reads_cache(model=model, kwargs=kwargs): + return None + args = args or () + final_embedding_cached_response: EmbeddingResponse | None = None + embedding_all_elements_cache_hit: bool = False + read: Final = await self.async_read_cache( + model=model, + original_function=original_function, + logging_obj=logging_obj, + call_type=call_type, + kwargs=kwargs, + args=args, + ) + kwargs = read.kwargs + cached_result: object | None = read.cached_result + if litellm.cache is not None and read.checked: + if read.end_time is not None: + cached_result = self._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=original_function.__name__, + kwargs=kwargs, + logging_obj=logging_obj, + model=read.model, + custom_llm_provider=kwargs.get("custom_llm_provider", None), + args=args, + ) + if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result): + # LOG SUCCESS + self._async_log_cache_hit_on_callbacks( + logging_obj=logging_obj, + cached_result=cached_result, + start_time=start_time, + end_time=read.end_time, + cache_hit=True, + ) + _set_cached_hidden_param(cached_result, "cache_key", self.hit_cache_key()) + return CachingHandlerResponse(cached_result=cached_result) + elif ( + call_type == CallTypes.aembedding.value + and cached_result is not None + and isinstance(cached_result, list) + and litellm.cache is not None + and not isinstance(litellm.cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude. + ): + ( + final_embedding_cached_response, + embedding_all_elements_cache_hit, + ) = self._process_async_embedding_cached_response( + final_embedding_cached_response=final_embedding_cached_response, + cached_result=cached_result, + kwargs=kwargs, + logging_obj=logging_obj, + start_time=start_time, + model=model, + ) + return CachingHandlerResponse( + final_embedding_cached_response=final_embedding_cached_response, + embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, + embedding_uncached_input=self.handle_kwargs_input_list_or_str(kwargs), + ) + + verbose_logger.debug("CACHE RESULT: %s", cached_result) + return CachingHandlerResponse( + cached_result=cached_result, + final_embedding_cached_response=final_embedding_cached_response, + ) + + _async_get_cache = async_get_cache + + def reads_cache( + self, + model: str, + kwargs: dict[str, Any], # pyright: ignore[reportExplicitAny] # mutable-ok: the @client call kwargs, read as the wrapper reads them + ) -> bool: if _is_response_cache_excluded(model=model, kwargs=kwargs): self.request_kwargs = _drop_logging_obj_from_kwargs(kwargs) - return None - - # Check if caching should be performed BEFORE doing expensive operations - if ( + return False + return ( (kwargs.get("caching", None) is None and litellm.cache is not None) or kwargs.get("caching", False) is True ) and ( kwargs.get("cache", {}).get("no-cache", False) is not True - ): # allow users to control returning cached responses from the completion function - args = args or () - final_embedding_cached_response: EmbeddingResponse | None = None - embedding_all_elements_cache_hit: bool = False - cached_result: object | None = None - kwargs = kwargs.copy() - ######################################################### - # Init cache timing metrics - ######################################################### - cache_check_start_time: Final = time.perf_counter() - cache_check_end_time: float | None = None - ######################################################### - parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - kwargs["parent_otel_span"] = parent_otel_span + ) # allow users to control returning cached responses from the completion function - if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function): - verbose_logger.debug("Checking Async Cache") - logging_obj.reset_caching_details() - cached_result = await self._retrieve_from_cache( - call_type=call_type, - kwargs=kwargs, - args=args, - ) - cache_check_end_time = time.perf_counter() + async def async_read_cache( + self, + model: str, + original_function: Callable[..., object], + logging_obj: LiteLLMLoggingObj, + call_type: str, + kwargs: Mapping[str, Any], # pyright: ignore[reportExplicitAny] # the @client call kwargs, read as the wrapper reads them + args: tuple[object, ...], + ) -> CacheRead: + cache_check_start_time: Final = time.perf_counter() + read_kwargs: Final = dict(kwargs) + read_kwargs["parent_otel_span"] = get_parent_otel_span_from_kwargs(read_kwargs) + if litellm.cache is None or not self._is_call_type_supported_by_cache(original_function=original_function): + return CacheRead(kwargs=read_kwargs) + verbose_logger.debug("Checking Async Cache") + logging_obj.reset_caching_details() + cached_result: Final = await self._retrieve_from_cache( + call_type=call_type, + kwargs=read_kwargs, + args=args, + ) + cache_check_end_time: Final = time.perf_counter() + if cached_result is None or isinstance(cached_result, list): + return CacheRead(kwargs=read_kwargs, checked=True, cached_result=cached_result) + verbose_logger.debug("Cache Hit!") + end_time: Final = datetime.datetime.now() + provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, + custom_llm_provider=read_kwargs.get("custom_llm_provider", None), + api_base=read_kwargs.get("api_base", None), + api_key=read_kwargs.get("api_key", None), + ) + self._update_litellm_logging_obj_environment( + logging_obj=logging_obj, + model=provider_model, + kwargs=read_kwargs, + cached_result=cached_result, + is_async=True, + custom_llm_provider=custom_llm_provider, + cache_duration_ms=(cache_check_end_time - cache_check_start_time) * 1000, + ) + return CacheRead( + kwargs=read_kwargs, + checked=True, + cached_result=cached_result, + model=provider_model, + end_time=end_time, + ) - if cached_result is not None and not isinstance(cached_result, list): - verbose_logger.debug("Cache Hit!") - cache_hit: Final = True - end_time: Final = datetime.datetime.now() - provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, - custom_llm_provider=kwargs.get("custom_llm_provider", None), - api_base=kwargs.get("api_base", None), - api_key=kwargs.get("api_key", None), - ) - cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000 - self._update_litellm_logging_obj_environment( - logging_obj=logging_obj, - model=provider_model, - kwargs=kwargs, - cached_result=cached_result, - is_async=True, - custom_llm_provider=custom_llm_provider, - cache_duration_ms=cache_duration_ms, - ) - - call_type = original_function.__name__ - - cached_result = self._convert_cached_result_to_model_response( - cached_result=cached_result, - call_type=call_type, - kwargs=kwargs, - logging_obj=logging_obj, - model=provider_model, - custom_llm_provider=kwargs.get("custom_llm_provider", None), - args=args, - ) - if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result): - # LOG SUCCESS - self._async_log_cache_hit_on_callbacks( - logging_obj=logging_obj, - cached_result=cached_result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - ) - cache_key: Final = ( - self.preset_cache_key - or self.request_kwargs.get("cache_key") - or litellm.cache.get_cache_key(**self.request_kwargs) - ) - _set_cached_hidden_param(cached_result, "cache_key", cache_key) - return CachingHandlerResponse(cached_result=cached_result) - elif ( - call_type == CallTypes.aembedding.value - and cached_result is not None - and isinstance(cached_result, list) - and litellm.cache is not None - and not isinstance(litellm.cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude. - ): - ( - final_embedding_cached_response, - embedding_all_elements_cache_hit, - ) = self._process_async_embedding_cached_response( - final_embedding_cached_response=final_embedding_cached_response, - cached_result=cached_result, - kwargs=kwargs, - logging_obj=logging_obj, - start_time=start_time, - model=model, - ) - return CachingHandlerResponse( - final_embedding_cached_response=final_embedding_cached_response, - embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, - embedding_uncached_input=self.handle_kwargs_input_list_or_str(kwargs), - ) - - verbose_logger.debug("CACHE RESULT: %s", cached_result) - return CachingHandlerResponse( - cached_result=cached_result, - final_embedding_cached_response=final_embedding_cached_response, - ) - # Caching disabled - return None to indicate no caching attempted - return None - - _async_get_cache = async_get_cache + def hit_cache_key(self) -> object: + if litellm.cache is None: + return None + return ( + self.preset_cache_key + or self.request_kwargs.get("cache_key") + or litellm.cache.get_cache_key(**self.request_kwargs) + ) def sync_get_cache( self, @@ -882,14 +921,14 @@ class LLMCachingHandler: _combine_cached_embedding_response_with_api_result = combine_cached_embedding_response_with_api_result - def _async_log_cache_hit_on_callbacks( + def async_log_cache_hit_on_callbacks( self, logging_obj: LiteLLMLoggingObj, cached_result: object, start_time: datetime.datetime, end_time: datetime.datetime, cache_hit: bool, - ): + ) -> None: """ Helper function to log the success of a cached result on callbacks @@ -930,6 +969,8 @@ class LLMCachingHandler: cache_hit=cache_hit, ) + _async_log_cache_hit_on_callbacks = async_log_cache_hit_on_callbacks + async def _retrieve_from_cache( self, call_type: str, kwargs: dict[str, object], args: tuple[object, ...] ) -> object | None: diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index faa604a25ce..36f9752a2e8 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -138,7 +138,6 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType( **{feature: detect for feature, detect in detectors.COMMON.items() if feature is not Feature.TIMEOUT_VARIANTS}, Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity, Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier, - Feature.RESPONSE_CACHE: detectors.response_cache, Feature.MODIFY_PARAMS: detectors.modify_params, Feature.WORKLOAD_IDENTITY: route_detectors.anthropic_workload_identity, Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index c493617003d..0e6b75349a6 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -179,7 +179,6 @@ MESSAGES_PYTHON_FEATURES: Final = (COMMON_PYTHON_FEATURES - {Feature.TIMEOUT_VAR Feature.PRE_REQUEST_HOOK, Feature.WEBSEARCH_INTERCEPTION, Feature.MESSAGES_INTERCEPTOR, - Feature.RESPONSE_CACHE, Feature.MODIFY_PARAMS, Feature.WORKLOAD_IDENTITY, Feature.CACHE_CONTROL_INJECTION, diff --git a/litellm/rust_bridge/host/cache.py b/litellm/rust_bridge/host/cache.py index bf2655f9f62..14332e70b7b 100644 --- a/litellm/rust_bridge/host/cache.py +++ b/litellm/rust_bridge/host/cache.py @@ -1,7 +1,8 @@ from __future__ import annotations -from collections.abc import Mapping -from typing import TYPE_CHECKING, Final +import weakref +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # bounded cast of the served response from pydantic import TypeAdapter @@ -9,9 +10,12 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_par if TYPE_CHECKING: from litellm.caching.caching import Cache + from litellm.litellm_core_utils.litellm_logging import Logging _RUST_RESPONSE_HEADER: Final = "x-litellm-rust" _HEADERS: Final = TypeAdapter(dict[str, object]) +_EVENTS: Final = TypeAdapter(list[str]) +_HITS: Final[weakref.WeakKeyDictionary[Logging, Mapping[str, object]]] = weakref.WeakKeyDictionary() def configured_cache() -> Cache | None: @@ -25,17 +29,14 @@ def _python_entry(response: Mapping[str, object]) -> Mapping[str, object]: headers: Final = hidden.get("additional_headers") if not isinstance(headers, Mapping) or _RUST_RESPONSE_HEADER not in headers: return response - return { - **response, - "_hidden_params": { - **hidden, - "additional_headers": { - name: value - for name, value in _HEADERS.validate_python(headers).items() - if name != _RUST_RESPONSE_HEADER - }, - }, + kept: Final = { + name: value for name, value in _HEADERS.validate_python(headers).items() if name != _RUST_RESPONSE_HEADER } + stored: Final = {name: value for name, value in hidden.items() if name != "additional_headers" or kept} | ( + {"additional_headers": kept} if kept else {} + ) + entry: Final = {name: value for name, value in response.items() if name != "_hidden_params"} + return {**entry, "_hidden_params": stored} if stored else entry def store_response( @@ -52,3 +53,87 @@ def store_response( **kwargs, # pyright: ignore[reportArgumentType] # the cache call's own keyword arguments, as Python passes them ) ) + + +async def store_stream( + cache: Cache, + entry: Mapping[str, object], + /, + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> None: + from litellm.caching.caching_handler import LLMCachingHandler + from litellm.types.caching import CACHED_STREAM_EVENTS_KEY + + events: Final = entry.get(CACHED_STREAM_EVENTS_KEY) + handler: Final = getattr(kwargs.get("litellm_logging_obj"), "llm_caching_handler", None) + await cache.async_add_cache( # pyright: ignore[reportUnknownMemberType] # Cache.async_add_cache leaves its result untyped + {**entry, CACHED_STREAM_EVENTS_KEY: tuple(_EVENTS.validate_python(events))} + if isinstance(events, list) + else entry, + dynamic_cache_object=handler.dual_cache if isinstance(handler, LLMCachingHandler) else None, + **kwargs, # pyright: ignore[reportArgumentType] # the cache call's own keyword arguments, as Python passes them + ) + + +async def lookup( + _cache: Cache, + original_function: Callable[..., object], + /, + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> object | None: + from litellm.caching.caching_handler import LLMCachingHandler + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return None + handler: Final = LLMCachingHandler( + original_function=original_function, request_kwargs=dict(kwargs), start_time=logging_obj.start_time + ) + logging_obj.llm_caching_handler = handler + model: Final = kwargs.get("model") + model_name: Final = model if isinstance(model, str) else "" + if not handler.reads_cache(model=model_name, kwargs=kwargs): + return None + read: Final = await handler.async_read_cache( + model=model_name, + original_function=original_function, + logging_obj=logging_obj, + call_type=original_function.__name__, + kwargs=kwargs, + args=(), + ) + if not read.hit: + return None + _remember(logging_obj, read.cached_result) + return read.cached_result + + +def _remember(logging_obj: Logging, cached: object) -> None: + if isinstance(cached, Mapping): + _HITS[logging_obj] = _HEADERS.validate_python(cached) + + +def _replace(response: object, cached: Mapping[str, object] | None) -> None: + if isinstance(response, dict) and cached is not None: + served: Final = cast( # cast-ok: a Messages response is the str-keyed dict the route serves + "dict[str, object]", response + ) + served.clear() + served.update(cached) + + +def restore_hit(response: object, logger: Logging) -> None: + from litellm.caching.caching_handler import ( + _set_cached_hidden_param, # pyright: ignore[reportPrivateUsage] # the @client cache hit's own hidden params + ) + + _replace(response, _HITS.pop(logger, None)) + _set_cached_hidden_param(response, "cache_hit", True) + + +def hit_cache_key(logger: Logging) -> object: + from litellm.caching.caching_handler import LLMCachingHandler + + handler: Final = logger.llm_caching_handler + return handler.hit_cache_key() if isinstance(handler, LLMCachingHandler) else None diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index ba557f769eb..c66f34feaf2 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -41,6 +41,7 @@ class MetadataUpdater(Protocol): kwargs: dict[str, object], start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool | None = None, ) -> None: ... @@ -97,6 +98,7 @@ def finalize( kwargs: dict[str, object], start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool, ) -> None: from litellm.litellm_core_utils.llm_response_utils import response_metadata @@ -104,6 +106,16 @@ def finalize( update: Final = cast( # cast-ok: legacy metadata function accepts concrete kwargs MetadataUpdater, response_metadata.update_response_metadata ) + if cache_hit: + from litellm.caching.caching_handler import ( + _set_cached_hidden_param, # pyright: ignore[reportPrivateUsage] # the @client cache hit's own hidden params + ) + from litellm.rust_bridge.host.cache import hit_cache_key, restore_hit + + restore_hit(response, logger) + update(response, logger, logger.model, kwargs, start_time, end_time, cache_hit=True) + _set_cached_hidden_param(response, "cache_key", hit_cache_key(logger)) + return update(response, logger, model if isinstance(model, str) else None, kwargs, start_time, end_time) cache_key: Final = logger.model_call_details.get("cache_key") if logger.model_call_details.get("cache_hit") is True and isinstance(cache_key, str): @@ -369,6 +381,16 @@ def async_success_handler( return logger.async_success_handler(response, start, end, cache_hit=_cache_hit(logger)) +def cache_hit_success(logger: Logging, response: object, start: datetime.datetime, end: datetime.datetime) -> None: + from litellm.caching.caching_handler import LLMCachingHandler + + handler: Final = logger.llm_caching_handler + if isinstance(handler, LLMCachingHandler): + handler.async_log_cache_hit_on_callbacks( + logging_obj=logger, cached_result=response, start_time=start, end_time=end, cache_hit=True + ) + + def enqueue_logging(coroutine: Coroutine[object, object, None]) -> None: from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -442,9 +464,10 @@ def stamp_stream_head( kwargs: dict[str, object], # mutable-ok: `finalize` takes the call kwargs as a dict start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool, ) -> None: carrier: Final = _HiddenParamsCarrier(head) - finalize(carrier, logger, kwargs, start_time, end_time) + finalize(carrier, logger, kwargs, start_time, end_time, cache_hit) stamped: Final = carrier._hidden_params # pyright: ignore[reportPrivateUsage] # the carrier's own field if stamped is not head and isinstance(stamped, dict): head.update(cast("dict[str, object]", stamped)) # cast-ok: update_response_metadata writes a str-keyed dict diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index c6568264edb..dcb2b3518f4 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -116,6 +116,13 @@ def _with_deployment_effort(params: Mapping[str, object], request: Mapping[str, ) +def cached_function() -> Callable[..., object]: + return cast( # cast-ok: the @client cache read names and checks the legacy handler it wraps + "Callable[..., object]", + handler.anthropic_messages, # noqa: TID251 # the @client cache read names and checks the legacy handler it wraps + ) + + def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: from litellm.llms.anthropic.messages.native.streaming_iterator import ( anthropic_messages_stream_hidden_params, diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py index 41008f0d57f..241ff1f3414 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py @@ -34,6 +34,10 @@ import pytest from pydantic import JsonValue, TypeAdapter import litellm +from litellm.caching.caching import Cache, CacheMode +from litellm.caching.caching_handler import ( + _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # a primed entry must land before the measured call reads it +) from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.messages import dispatch from litellm.rust_bridge import catalog @@ -319,6 +323,7 @@ class Case: env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) settings: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) private_keys: bool = False + cache: Callable[[], Cache] | None = None gaps: Mapping[str, str | None] = field(default_factory=lambda: MappingProxyType({})) def params(self) -> Iterator[object]: @@ -769,7 +774,45 @@ FALLBACK_CASES: Final = ( Case("stream_timeout_kwarg", _kw(stream=True, stream_timeout=30), upstream=(_stream(TEXT_STREAM),)), ) -CASES: Final = (*REQUEST_CASES, *RESPONSE_CASES, *FAILURE_CASES, *STREAM_CASES, *SYNC_CASES, *FALLBACK_CASES) + +def _default_off_cache() -> Cache: + return Cache(mode=CacheMode.default_off) + + +def _completion_only_cache() -> Cache: + return Cache(supported_call_types=["acompletion"]) + + +CACHE_CASES: Final = ( + Case("cache_hit", upstream=(_ok(), _ok()), cache=Cache), + Case("cache_hit_stream", _kw(stream=True), upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), cache=Cache), + Case("cache_no_cache", _kw(cache={"no-cache": True}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_no_store", _kw(cache={"no-store": True}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_caching_false", _kw(caching=False), upstream=(_ok(), _ok()), cache=Cache), + Case( + "cache_ttl_and_namespace", _kw(cache={"ttl": 60, "namespace": "parity"}), upstream=(_ok(), _ok()), cache=Cache + ), + Case("cache_default_off", upstream=(_ok(), _ok()), cache=_default_off_cache), + Case( + "cache_default_off_use_cache", + _kw(cache={"use-cache": True}), + upstream=(_ok(), _ok()), + cache=_default_off_cache, + ), + Case("cache_unsupported_call_type", upstream=(_ok(), _ok()), cache=_completion_only_cache), + Case("cache_hit_with_metadata", _kw(metadata={"user_id": "user-123"}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_sync", sync=True, upstream=(_ok(), _ok()), cache=Cache), +) + +CASES: Final = ( + *REQUEST_CASES, + *RESPONSE_CASES, + *FAILURE_CASES, + *STREAM_CASES, + *SYNC_CASES, + *FALLBACK_CASES, + *CACHE_CASES, +) @dataclass(frozen=True, slots=True) @@ -843,6 +886,15 @@ def _settings(settings: Mapping[str, object]) -> Generator[None]: setattr(litellm, name, value) +@contextmanager +def _cache(cache: Callable[[], Cache] | None) -> Generator[None]: + if cache is None: + yield + return + with rebound(litellm, "cache", cache()): + yield + + def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]: # Deep: Python writes into the caller's own `litellm_metadata`, which must not leak into the # other leg. @@ -960,21 +1012,24 @@ async def _invoke( }, hidden -async def _terminal_events(recorder: RecordingLogger) -> tuple[str, ...]: +async def _terminal_events(recorder: RecordingLogger, since: int = 0) -> tuple[str, ...]: for _ in range(100): await drain_logging() - if any(name in _TERMINAL_EVENTS for name in recorder.names): + if any(name in _TERMINAL_EVENTS for name in recorder.names[since:]): break await asyncio.sleep(0.05) await asyncio.sleep(0.2) await drain_logging() - return tuple(name for name in recorder.names if name in _TERMINAL_EVENTS) + return tuple(name for name in recorder.names[since:] if name in _TERMINAL_EVENTS) -def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping[str, object], Mapping[str, object]]: - terminal: Final = [event for event in recorder.events if event.name in _TERMINAL_EVENTS] +def _logged( + recorder: RecordingLogger, events: tuple[str, ...], since: int = 0 +) -> tuple[Mapping[str, object], Mapping[str, object]]: + terminal: Final = [event for event in recorder.events[since:] if event.name in _TERMINAL_EVENTS] + hooks: Final = tuple(sorted(set(recorder.names[since:]))) if not terminal: - return {"events": events}, {} + return {"events": events, "hooks": hooks}, {} details: Final = _mapping(terminal[-1].kwargs) standard: Final = _mapping(details.get("standard_logging_object")) error_class: Final = _mapping(standard.get("error_information")).get("error_class") @@ -983,6 +1038,7 @@ def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping metadata_keys: Final = _mapping(metadata).keys() logged: Final[Mapping[str, object]] = { "events": events, + "hooks": hooks, **{name: standard.get(name) for name in _COMPARED_LOG_FIELDS}, "error_class": error_class, } @@ -1000,18 +1056,29 @@ def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping return logged, callback_kwargs +async def _prime(case: Case, server: RecordingServer, recorder: RecordingLogger, native: bool) -> int: + if case.cache is None: + return 0 + outcome: Final = await _invoke(case, _kwargs(case, server), native=native) + assert case.sync or outcome[0]["kind"] != "error", outcome + await _terminal_events(recorder) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + return len(recorder.events) + + async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) -> Leg: - with isolated_callback_registries(), _environment(case.env), _settings(case.settings): - recorder: Final = RecordingLogger() - litellm.callbacks.append(recorder) + with isolated_callback_registries(), _environment(case.env), _settings(case.settings), _cache(case.cache): for spec in case.upstream: server.enqueue(replace(spec)) request_count: Final = len(server.requests) + recorder: Final = RecordingLogger() + litellm.callbacks.append(recorder) fallbacks_before: Final = 0 if probe is None else probe.calls + primed: Final = await _prime(case, server, recorder, native=probe is not None) outcome, hidden = await _invoke(case, _kwargs(case, server), native=probe is not None) - events: Final = await _terminal_events(recorder) + events: Final = await _terminal_events(recorder, primed) server.responses.clear() - logged, callback_kwargs = _logged(recorder, events) + logged, callback_kwargs = _logged(recorder, events, primed) return Leg( fell_back=probe is not None and probe.calls > fallbacks_before, requests=tuple( diff --git a/tests/test_litellm_rust/cache/test_python_cache.py b/tests/test_litellm_rust/cache/test_python_cache.py index d88feec7d5e..632a7d788c2 100644 --- a/tests/test_litellm_rust/cache/test_python_cache.py +++ b/tests/test_litellm_rust/cache/test_python_cache.py @@ -6,6 +6,7 @@ from pydantic import TypeAdapter import litellm from litellm.caching.caching import Cache, CacheMode +from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.caching_handler import ( _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request ) @@ -33,6 +34,14 @@ from tests.test_litellm_rust.support.routing import native_route pytestmark = pytest.mark.requires_rust_extension +_STORED: Final = TypeAdapter(dict[str, object]) + + +def _entries(storage: InMemoryCache) -> dict[str, object]: + return _STORED.validate_python( + storage.cache_dict # pyright: ignore[reportUnknownMemberType] # InMemoryCache leaves its dict untyped + ) + @pytest.mark.asyncio @pytest.mark.parametrize( @@ -110,7 +119,9 @@ async def test_legacy_cache_controls_and_backend_credential_key_semantics(record recording_server.expected_requests = 3 litellm.cache = Cache() await invoke("messages", recording_server, {"cache": {"no-store": True}}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) await invoke("messages", recording_server, {}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) await invoke("messages", recording_server, {}) assert len(recording_server.requests) == 2 await invoke("messages", recording_server, {"cache": {"no-cache": True}}) @@ -174,13 +185,18 @@ async def test_rust_messages_uses_a_legacy_cache_without_python_inference( "api_base": recording_server.base_url, } first: Final = await invoke("messages", recording_server, parameters) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) second: Final = await invoke("messages", recording_server, parameters) - assert cache_key(second) - assert cache_key(first) is None assert payload(first) == payload(second) assert len(recording_server.requests) == 1 + assert cache_key(first) is None + assert cache_key(second) is None - await logger.wait_for_async("async_log_success_event", count=2) + successes: Final = await logger.wait_for_async("async_log_success_event", count=2) + assert [TypeAdapter(dict[str, object]).validate_python(event.kwargs).get("cache_hit") for event in successes] == [ + None, + True, + ] assert logger.names.count("async_log_success_event") == 2 assert "log_failure_event" not in logger.names assert "async_log_failure_event" not in logger.names @@ -210,7 +226,9 @@ async def test_legacy_cache_default_off_requires_opt_in_even_for_existing_entrie @pytest.mark.asyncio -async def test_legacy_cache_lookup_uses_backend_request_callback_semantics(recording_server: RecordingServer) -> None: +async def test_a_legacy_cache_hit_skips_the_pre_call_callback_that_would_rewrite_the_request( + recording_server: RecordingServer, +) -> None: class Rewrite(RecordingLogger): temperature = 0.1 @@ -222,15 +240,16 @@ async def test_legacy_cache_lookup_uses_backend_request_callback_semantics(recor litellm.cache = Cache() recording_server.expected_requests = 1 await invoke("messages", recording_server, {"callbacks": [logger]}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) first_hit: Final = await invoke("messages", recording_server, {"callbacks": [logger]}) logger.temperature = 0.8 await invoke("messages", recording_server, {"callbacks": [logger]}) second_hit: Final = await invoke("messages", recording_server, {"callbacks": [logger]}) - assert logger.names.count("log_pre_api_call") == 4 + assert logger.names.count("log_pre_api_call") == 1 + assert cache_key(first_hit) is None + assert cache_key(second_hit) is None assert len(recording_server.requests) == recording_server.expected_requests assert recording_server.requests[0].body["temperature"] == 0.1 - assert isinstance(cache_key(first_hit), str) - assert isinstance(cache_key(second_hit), str) @pytest.mark.asyncio @@ -314,7 +333,11 @@ async def test_python_cache_operations_stay_in_the_rust_callers_task_except_the_ second: Final = await invoke("messages", recording_server, {"stream": stream}) second_payload: Final = await collect(second) if stream else payload(second) assert first_payload == second_payload - assert cache_key(second) + assert cache_key(first) is None + if stream: + assert cache_key(second) in _entries(storage) + else: + assert cache_key(second) is None assert len(recording_server.requests) == 1 @@ -389,6 +412,8 @@ async def test_messages_cache_stores_the_response_a_yielding_deployment_hook_ret async def test_rust_messages_legacy_cache_honors_request_namespaces( recording_server: RecordingServer, namespace_source: str ) -> None: + from litellm.caching.in_memory_cache import InMemoryCache + litellm.cache = Cache() recording_server.expected_requests = 2 first_options: Final = ( @@ -399,13 +424,20 @@ async def test_rust_messages_legacy_cache_honors_request_namespaces( if namespace_source == "cache" else {"metadata": {"redis_namespace": "second"}} ) + storage: Final = litellm.cache.cache + assert isinstance(storage, InMemoryCache) await invoke("messages", recording_server, first_options) - second: Final = await invoke("messages", recording_server, second_options) - first_hit: Final = await invoke("messages", recording_server, first_options) - second_hit: Final = await invoke("messages", recording_server, second_options) - assert cache_key(second) is None - assert cache_key(first_hit) is not None - assert cache_key(second_hit) is not None + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + first_keys: Final = set(_entries(storage)) + await invoke("messages", recording_server, second_options) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + second_keys: Final = set(_entries(storage)) - first_keys + assert len(recording_server.requests) == 2 + assert [key.split(":", 1)[0] for key in first_keys] == ["first"] + assert [key.split(":", 1)[0] for key in second_keys] == ["second"] + assert {key.split(":", 1)[1] for key in first_keys} == {key.split(":", 1)[1] for key in second_keys} + await invoke("messages", recording_server, first_options) + await invoke("messages", recording_server, second_options) assert len(recording_server.requests) == 2 @@ -420,9 +452,8 @@ async def test_rust_messages_legacy_semantic_cache_preserves_python_scope( first_options: Final = {"messages": [{"role": "user", "content": "hello"}]} second_options: Final = {"messages": [{"role": "user", "content": "hi"}]} first: Final = await invoke("messages", recording_server, first_options) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) second: Final = await invoke("messages", recording_server, second_options) - assert cache_key(first) is None - assert cache_key(second) is not None assert payload(first) == payload(second) assert len(recording_server.requests) == 1 @@ -443,3 +474,62 @@ async def test_legacy_cache_keeps_public_messages_responses_compatible( second_payload: Final = await collect(second) if stream else payload(second) assert second_payload == first_payload assert len(recording_server.requests) == 1 + + +_PER_CALL_HIDDEN_PARAMS: Final = frozenset( + {"_response_ms", "litellm_overhead_time_ms", "callback_duration_ms", "litellm_call_id"} +) + + +async def _write_then_read( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + writer: bool, + reader: bool, + stream: bool, +) -> tuple[object, object, object, int]: + from litellm.caching.in_memory_cache import InMemoryCache + + storage: Final = InMemoryCache() + monkeypatch.setattr(litellm, "cache", Cache(_backend=storage)) + requests_before: Final = len(recording_server.requests) + monkeypatch.setenv("LITELLM_RUST", "1" if writer else "0") + first: Final = await invoke("messages", recording_server, {"stream": stream}, native=writer) + if stream: + await collect(first) + else: + add_retry_headers_to_response( + _add_headers_to_response(first, {"x-litellm-model-group": "group"}), attempted_retries=0 + ) + get_hidden_params_dict(first)["model_id"] = "deployment" + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + (entry,) = _entries(storage).values() + stored: Final = TypeAdapter(dict[str, object]).validate_python(entry)["response"] + monkeypatch.setenv("LITELLM_RUST", "1" if reader else "0") + hit: Final = await invoke("messages", recording_server, {"stream": stream}, native=reader) + hidden: Final = { + name: value for name, value in get_hidden_params_dict(hit).items() if name not in _PER_CALL_HIDDEN_PARAMS + } + headers: Final = TypeAdapter(dict[str, object]).validate_python(hidden.pop("additional_headers", {})) + kept: Final = { + name: value for name, value in headers.items() if name not in {"x-litellm-rust", "x-litellm-cache-key"} + } + served: Final = await collect(hit) if stream else payload(hit) + return stored, served, {**hidden, "additional_headers": kept}, len(recording_server.requests) - requests_before + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", (False, True), ids=("response", "stream")) +@pytest.mark.parametrize(("writer", "reader"), ((True, False), (False, True)), ids=("rust_to_python", "python_to_rust")) +async def test_a_cache_entry_written_by_one_leg_is_stored_and_served_like_the_readers_own( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + writer: bool, + reader: bool, + stream: bool, +) -> None: + recording_server.expected_requests = 2 + crossed: Final = await _write_then_read(recording_server, monkeypatch, writer, reader, stream) + own: Final = await _write_then_read(recording_server, monkeypatch, reader, reader, stream) + assert crossed == own + assert crossed[3] == 1 diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index b580eca2e42..872a712c2fc 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -18,7 +18,7 @@ from litellm.messages.dispatch import ( ) from litellm.proxy import proxy_server from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Feature, Route, RouteRule, Rules, Target +from litellm.rust_bridge.catalog import MESSAGES_PYTHON_FEATURES, Feature, Route, RouteRule, Rules, Target from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES, NativeAmessages, NativeMessages from litellm.rust_bridge.public_call import NativeCall @@ -476,7 +476,6 @@ STRICT_TOOL: Final = {"name": "a", "input_schema": {"type": "object"}, "strict": ), (Feature.CALLER_CLIENT, "anthropic/claude-sonnet-4-5", {"client": object()}, {}), (Feature.AGENTIC_LOOP_HOOK, "anthropic/claude-sonnet-4-5", {}, {"callbacks": [_Agentic()]}), - (Feature.RESPONSE_CACHE, "anthropic/claude-sonnet-4-5", {}, {"cache": Cache()}), (Feature.TLS_POLICY, "anthropic/claude-sonnet-4-5", {}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}), (Feature.MODIFY_PARAMS, "anthropic/claude-sonnet-4-5", {}, {"modify_params": True}), ( @@ -617,6 +616,45 @@ async def test_strict_tools_stay_native_unless_python_could_drop_strict( assert served is native_result +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kwargs", + ( + pytest.param({}, id="reads-and-writes"), + pytest.param({"cache": {"no-store": True}}, id="no-store"), + pytest.param({"cache": {"ttl": 60, "namespace": "team"}}, id="ttl-and-namespace"), + pytest.param({"stream": True}, id="stream"), + ), +) +async def test_a_configured_response_cache_stays_native( + monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object] +) -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=MESSAGES_PYTHON_FEATURES), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + native_result: Final = response() + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "cache", Cache()) + + async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + pytest.fail("Rust reads and writes the configured response cache") + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + served: Final = await _ADISPATCH.arun( + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + kwargs, + python=python, + binding=amessages_binding(native), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=rules, + ) + + assert served is native_result + + ASYNC_ONLY_FEATURES: Final = ( pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"), pytest.param( diff --git a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py index cb594767908..5c857ffe67f 100644 --- a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py @@ -73,7 +73,7 @@ def test_cache_hit_finalization_preserves_execution_provider_attribution(explici ) prepared.logger.model_call_details.update({"cache_hit": True, "cache_key": "cached-response"}) response: Final = ModelResponse(model="cache-test-model") - legacy.finalize(response, prepared.logger, prepared.kwargs, now, now) + legacy.finalize(response, prepared.logger, prepared.kwargs, now, now, False) assert prepared.logger.model_call_details["custom_llm_provider"] == "azure" assert prepared.logger.model_call_details["model"] == "resolved-cache-model" assert prepared.logger.litellm_params["metadata"]["user_api_key"] == "key-hash" @@ -334,6 +334,7 @@ def test_stream_head_is_stamped_in_place_with_the_call_hidden_params() -> None: {"model": "anthropic/claude-sonnet-4-5", "litellm_call_id": "messages"}, started, started + datetime.timedelta(milliseconds=5), + False, ) assert head["additional_headers"] == {"request-id": "req_native", "llm_provider-request-id": "req_native"} diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index 6ceea94d843..44b5876a078 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -1223,6 +1223,13 @@ ALIAS_CASES: Final = ( False, ), ("litellm.caching.caching_handler", "LLMCachingHandler", "_async_get_cache", "async_get_cache", False), + ( + "litellm.caching.caching_handler", + "LLMCachingHandler", + "_async_log_cache_hit_on_callbacks", + "async_log_cache_hit_on_callbacks", + False, + ), ( "litellm.caching.caching_handler", "LLMCachingHandler", @@ -1957,7 +1964,13 @@ ALIAS_CASES: Final = ( "get_tags_from_request_kwargs", False, ), - ("litellm.litellm_core_utils.add_retry_fallback_headers", "HiddenParamsAsyncIteratorWrapper", "_inner", "inner", True), + ( + "litellm.litellm_core_utils.add_retry_fallback_headers", + "HiddenParamsAsyncIteratorWrapper", + "_inner", + "inner", + True, + ), ("litellm.litellm_core_utils.add_retry_fallback_headers", "", "_HiddenParamsHost", "HiddenParamsHost", False), ( "litellm.litellm_core_utils.batch_utils", @@ -2301,1837 +2314,1837 @@ LLMS_ALIAS_CASES: Final = ( ) PROXY_ALIAS_CASES: Final = ( ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - '', - '_is_mcp_admitted_user_subject', - 'is_mcp_admitted_user_subject', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "", + "_is_mcp_admitted_user_subject", + "is_mcp_admitted_user_subject", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_auth_header_from_headers', - 'get_mcp_auth_header_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_auth_header_from_headers", + "get_mcp_auth_header_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_server_auth_headers_from_headers', - 'get_mcp_server_auth_headers_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_server_auth_headers_from_headers", + "get_mcp_server_auth_headers_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_servers_from_access_groups', - 'get_mcp_servers_from_access_groups', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_servers_from_access_groups", + "get_mcp_servers_from_access_groups", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_oauth2_headers_from_headers', - 'get_oauth2_headers_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_oauth2_headers_from_headers", + "get_oauth2_headers_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_safe_get_headers_from_scope', - 'safe_get_headers_from_scope', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_safe_get_headers_from_scope", + "safe_get_headers_from_scope", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_bridge_mint_error_response', - 'bridge_mint_error_response', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_bridge_mint_error_response", + "bridge_mint_error_response", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_extract_user_id_from_request', - 'extract_user_id_from_request', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_extract_user_id_from_request", + "extract_user_id_from_request", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_finish_bridge_mint', - 'finish_bridge_mint', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_finish_bridge_mint", + "finish_bridge_mint", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_prepare_bridge_mint', - 'prepare_bridge_mint', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_prepare_bridge_mint", + "prepare_bridge_mint", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_prepare_bridge_refresh', - 'prepare_bridge_refresh', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_prepare_bridge_refresh", + "prepare_bridge_refresh", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_reload_active_user_by_id', - 'reload_active_user_by_id', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_reload_active_user_by_id", + "reload_active_user_by_id", False, ), ( - 'litellm.proxy.mcp_server.byok_oauth_endpoints', - '', - '_user_id_from_session_cookie', - 'user_id_from_session_cookie', + "litellm.proxy.mcp_server.byok_oauth_endpoints", + "", + "_user_id_from_session_cookie", + "user_id_from_session_cookie", False, ), ( - 'litellm.proxy.mcp_server.db', - '', - '_decode_oauth_payload', - 'decode_oauth_payload', + "litellm.proxy.mcp_server.db", + "", + "_decode_oauth_payload", + "decode_oauth_payload", False, ), ( - 'litellm.proxy.mcp_server.discoverable_endpoints', - '', - '_raise_if_not_oauth2', - 'raise_if_not_oauth2', + "litellm.proxy.mcp_server.discoverable_endpoints", + "", + "_raise_if_not_oauth2", + "raise_if_not_oauth2", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_active_toolset_id', - 'mcp_active_toolset_id', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_active_toolset_id", + "mcp_active_toolset_id", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_gateway_initialize_instructions', - 'mcp_gateway_initialize_instructions', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_gateway_initialize_instructions", + "mcp_gateway_initialize_instructions", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_gateway_server_name', - 'mcp_gateway_server_name', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_gateway_server_name", + "mcp_gateway_server_name", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES', - 'UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES", + "UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_caller_authorization_fans_out', - 'caller_authorization_fans_out', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_caller_authorization_fans_out", + "caller_authorization_fans_out", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_client_forwarded_authorization_headers', - 'client_forwarded_authorization_headers', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_client_forwarded_authorization_headers", + "client_forwarded_authorization_headers", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_resolve_openapi_tool_auth', - 'resolve_openapi_tool_auth', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_resolve_openapi_tool_auth", + "resolve_openapi_tool_auth", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_should_strip_caller_authorization', - 'should_strip_caller_authorization', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_should_strip_caller_authorization", + "should_strip_caller_authorization", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_build_mcp_server_table', - 'build_mcp_server_table', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_build_mcp_server_table", + "build_mcp_server_table", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_build_stdio_env', - 'build_stdio_env', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_build_stdio_env", + "build_stdio_env", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_create_mcp_client', - 'create_mcp_client', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_create_mcp_client", + "create_mcp_client", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_ensure_upstream_initialize_instructions_cached', - 'ensure_upstream_initialize_instructions_cached', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_ensure_upstream_initialize_instructions_cached", + "ensure_upstream_initialize_instructions_cached", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_extract_subject_token', - 'extract_subject_token', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_extract_subject_token", + "extract_subject_token", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_get_mcp_server_from_tool_name', - 'get_mcp_server_from_tool_name', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_get_mcp_server_from_tool_name", + "get_mcp_server_from_tool_name", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_get_tools_from_server', - 'get_tools_from_server', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_get_tools_from_server", + "get_tools_from_server", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_is_server_accessible_from_ip', - 'is_server_accessible_from_ip', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_is_server_accessible_from_ip", + "is_server_accessible_from_ip", False, ), ( - 'litellm.proxy.mcp_server.oauth2_token_cache', - '', - '_compute_per_user_token_ttl', - 'compute_per_user_token_ttl', + "litellm.proxy.mcp_server.oauth2_token_cache", + "", + "_compute_per_user_token_ttl", + "compute_per_user_token_ttl", False, ), ( - 'litellm.proxy.mcp_server.oauth_utils', - '', - '_redact_mcp_resource_url', - 'redact_mcp_resource_url', + "litellm.proxy.mcp_server.oauth_utils", + "", + "_redact_mcp_resource_url", + "redact_mcp_resource_url", False, ), ( - 'litellm.proxy.mcp_server.server', - '', - '_redact_mcp_resource_url', - 'redact_mcp_resource_url', + "litellm.proxy.mcp_server.server", + "", + "_redact_mcp_resource_url", + "redact_mcp_resource_url", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_OPENAPI_TOOL_NAME_MAX_LEN', - 'OPENAPI_TOOL_NAME_MAX_LEN', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_OPENAPI_TOOL_NAME_MAX_LEN", + "OPENAPI_TOOL_NAME_MAX_LEN", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_auth_header', - 'request_auth_header', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_auth_header", + "request_auth_header", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_extra_headers', - 'request_extra_headers', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_extra_headers", + "request_extra_headers", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_resolved_auth_headers', - 'request_resolved_auth_headers', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_resolved_auth_headers", + "request_resolved_auth_headers", False, ), ( - 'litellm.proxy.mcp_server.semantic_tool_filter', - 'SemanticMCPToolFilter', - '_extract_tool_info', - 'extract_tool_info', + "litellm.proxy.mcp_server.semantic_tool_filter", + "SemanticMCPToolFilter", + "_extract_tool_info", + "extract_tool_info", False, ), ( - 'litellm.proxy.mcp_server.server', - '', - '_apply_toolset_scope', - 'apply_toolset_scope', + "litellm.proxy.mcp_server.server", + "", + "_apply_toolset_scope", + "apply_toolset_scope", False, ), ( - 'litellm.proxy.mcp_server.server_resolution', - 'MCPServerRegistry', - '_build_mcp_server_table', - 'build_mcp_server_table', + "litellm.proxy.mcp_server.server_resolution", + "MCPServerRegistry", + "_build_mcp_server_table", + "build_mcp_server_table", False, ), ( - 'litellm.proxy.mcp_server.server_resolution', - 'MCPServerRegistry', - '_is_server_accessible_from_ip', - 'is_server_accessible_from_ip', + "litellm.proxy.mcp_server.server_resolution", + "MCPServerRegistry", + "_is_server_accessible_from_ip", + "is_server_accessible_from_ip", False, ), ( - 'litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace', - '', - '_get_prisma_client', - 'get_prisma_client', + "litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace", + "", + "_get_prisma_client", + "get_prisma_client", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_access_object', - 'cache_access_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_access_object", + "cache_access_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_key_object', - 'cache_key_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_key_object", + "cache_key_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_team_object', - 'cache_team_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_team_object", + "cache_team_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_can_object_call_model', - 'can_object_call_model', + "litellm.proxy.auth.auth_checks", + "", + "_can_object_call_model", + "can_object_call_model", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_end_user_budget', - 'check_end_user_budget', + "litellm.proxy.auth.auth_checks", + "", + "_check_end_user_budget", + "check_end_user_budget", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_model_access_helper', - 'check_model_access_helper', + "litellm.proxy.auth.auth_checks", + "", + "_check_model_access_helper", + "check_model_access_helper", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_team_member_model_access', - 'check_team_member_model_access', + "litellm.proxy.auth.auth_checks", + "", + "_check_team_member_model_access", + "check_team_member_model_access", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_copy_user_api_key_auth_for_cache', - 'copy_user_api_key_auth_for_cache', + "litellm.proxy.auth.auth_checks", + "", + "_copy_user_api_key_auth_for_cache", + "copy_user_api_key_auth_for_cache", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_delete_cache_access_object', - 'delete_cache_access_object', + "litellm.proxy.auth.auth_checks", + "", + "_delete_cache_access_object", + "delete_cache_access_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_delete_cache_key_object', - 'delete_cache_key_object', + "litellm.proxy.auth.auth_checks", + "", + "_delete_cache_key_object", + "delete_cache_key_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_fetch_key_object_from_db_with_reconnect', - 'fetch_key_object_from_db_with_reconnect', + "litellm.proxy.auth.auth_checks", + "", + "_fetch_key_object_from_db_with_reconnect", + "fetch_key_object_from_db_with_reconnect", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_agent_ids_from_access_groups', - 'get_agent_ids_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_agent_ids_from_access_groups", + "get_agent_ids_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_mcp_server_ids_from_access_groups', - 'get_mcp_server_ids_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_mcp_server_ids_from_access_groups", + "get_mcp_server_ids_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_models_from_access_groups', - 'get_models_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_models_from_access_groups", + "get_models_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_team_object_from_cache', - 'get_team_object_from_cache', + "litellm.proxy.auth.auth_checks", + "", + "_get_team_object_from_cache", + "get_team_object_from_cache", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_user_role', - 'get_user_role', + "litellm.proxy.auth.auth_checks", + "", + "_get_user_role", + "get_user_role", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_is_model_cost_zero', - 'is_model_cost_zero', + "litellm.proxy.auth.auth_checks", + "", + "_is_model_cost_zero", + "is_model_cost_zero", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_is_user_proxy_admin', - 'is_user_proxy_admin', + "litellm.proxy.auth.auth_checks", + "", + "_is_user_proxy_admin", + "is_user_proxy_admin", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_key_access_group_grants_model', - 'key_access_group_grants_model', + "litellm.proxy.auth.auth_checks", + "", + "_key_access_group_grants_model", + "key_access_group_grants_model", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_organization_max_budget_check', - 'organization_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_organization_max_budget_check", + "organization_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_team_max_budget_check', - 'team_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_team_max_budget_check", + "team_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_team_member_max_budget_alert_check', - 'team_member_max_budget_alert_check', + "litellm.proxy.auth.auth_checks", + "", + "_team_member_max_budget_alert_check", + "team_member_max_budget_alert_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_max_budget_alert_check', - 'virtual_key_max_budget_alert_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_max_budget_alert_check", + "virtual_key_max_budget_alert_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_max_budget_check', - 'virtual_key_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_max_budget_check", + "virtual_key_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_soft_budget_check', - 'virtual_key_soft_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_soft_budget_check", + "virtual_key_soft_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks_organization', - '', - '_user_is_org_admin', - 'user_is_org_admin', + "litellm.proxy.auth.auth_checks_organization", + "", + "_user_is_org_admin", + "user_is_org_admin", False, ), ( - 'litellm.proxy.auth.auth_exception_handler', - 'UserAPIKeyAuthExceptionHandler', - '_handle_authentication_error', - 'handle_authentication_error', + "litellm.proxy.auth.auth_exception_handler", + "UserAPIKeyAuthExceptionHandler", + "_handle_authentication_error", + "handle_authentication_error", False, ), ( - 'litellm.proxy.auth.auth_utils', - '', - '_get_request_ip_address', - 'get_request_ip_address', + "litellm.proxy.auth.auth_utils", + "", + "_get_request_ip_address", + "get_request_ip_address", False, ), ( - 'litellm.proxy.auth.resolvers.store', - 'IdentityStore', - '_principal_from_key', - 'principal_from_key', + "litellm.proxy.auth.resolvers.store", + "IdentityStore", + "_principal_from_key", + "principal_from_key", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_get_request_method', - 'get_request_method', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_get_request_method", + "get_request_method", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_is_assistants_api_request', - 'is_assistants_api_request', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_is_assistants_api_request", + "is_assistants_api_request", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_is_wildcard_pattern', - 'is_wildcard_pattern', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_is_wildcard_pattern", + "is_wildcard_pattern", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_enforce_key_and_fallback_model_access', - 'enforce_key_and_fallback_model_access', + "litellm.proxy.auth.user_api_key_auth", + "", + "_enforce_key_and_fallback_model_access", + "enforce_key_and_fallback_model_access", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_fetch_global_spend_with_event_coordination', - 'fetch_global_spend_with_event_coordination', + "litellm.proxy.auth.user_api_key_auth", + "", + "_fetch_global_spend_with_event_coordination", + "fetch_global_spend_with_event_coordination", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_get_bearer_token', - 'get_bearer_token', + "litellm.proxy.auth.user_api_key_auth", + "", + "_get_bearer_token", + "get_bearer_token", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_run_centralized_common_checks', - 'run_centralized_common_checks', + "litellm.proxy.auth.user_api_key_auth", + "", + "_run_centralized_common_checks", + "run_centralized_common_checks", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_user_api_key_auth_builder', - 'user_api_key_auth_builder', + "litellm.proxy.auth.user_api_key_auth", + "", + "_user_api_key_auth_builder", + "user_api_key_auth_builder", False, ), ( - 'litellm.proxy.common_request_processing', - '', - '_is_azure_model_router_request', - 'is_azure_model_router_request', + "litellm.proxy.common_request_processing", + "", + "_is_azure_model_router_request", + "is_azure_model_router_request", False, ), ( - 'litellm.proxy.common_request_processing', - '', - '_should_return_raw_model_name', - 'should_return_raw_model_name', + "litellm.proxy.common_request_processing", + "", + "_should_return_raw_model_name", + "should_return_raw_model_name", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_finalize_streaming_generator_cleanup', - 'finalize_streaming_generator_cleanup', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_finalize_streaming_generator_cleanup", + "finalize_streaming_generator_cleanup", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_handle_llm_api_exception', - 'handle_llm_api_exception', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_handle_llm_api_exception", + "handle_llm_api_exception", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_process_chunk_with_cost_injection', - 'process_chunk_with_cost_injection', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_process_chunk_with_cost_injection", + "process_chunk_with_cost_injection", False, ), ( - 'litellm.proxy.common_utils.callback_utils', - '', - '_CALLBACK_VAR_ENCRYPTED_PREFIX', - 'CALLBACK_VAR_ENCRYPTED_PREFIX', + "litellm.proxy.common_utils.callback_utils", + "", + "_CALLBACK_VAR_ENCRYPTED_PREFIX", + "CALLBACK_VAR_ENCRYPTED_PREFIX", False, ), ( - 'litellm.proxy.common_utils.config_sync_pubsub', - '', - '_ConfigSyncPubSub', - 'ConfigSyncPubSub', + "litellm.proxy.common_utils.config_sync_pubsub", + "", + "_ConfigSyncPubSub", + "ConfigSyncPubSub", False, ), ( - 'litellm.proxy.common_utils.config_sync_pubsub', - '', - '_pubsub_capable_client', - 'pubsub_capable_client', + "litellm.proxy.common_utils.config_sync_pubsub", + "", + "_pubsub_capable_client", + "pubsub_capable_client", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_ALGO_AES_GCM', - 'ALGO_AES_GCM', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_ALGO_AES_GCM", + "ALGO_AES_GCM", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_ENCRYPTION_ALGORITHM_SETTING', - 'ENCRYPTION_ALGORITHM_SETTING', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_ENCRYPTION_ALGORITHM_SETTING", + "ENCRYPTION_ALGORITHM_SETTING", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_V2_GCM_PREFIX', - 'V2_GCM_PREFIX', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_V2_GCM_PREFIX", + "V2_GCM_PREFIX", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_get_salt_key', - 'get_salt_key', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_get_salt_key", + "get_salt_key", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_read_request_body', - 'read_request_body', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_read_request_body", + "read_request_body", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_get_request_headers', - 'safe_get_request_headers', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_get_request_headers", + "safe_get_request_headers", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_get_request_query_params', - 'safe_get_request_query_params', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_get_request_query_params", + "safe_get_request_query_params", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_set_request_parsed_body', - 'safe_set_request_parsed_body', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_set_request_parsed_body", + "safe_set_request_parsed_body", False, ), ( - 'litellm.proxy.common_utils.realtime_utils', - '', - '_realtime_request_body', - 'realtime_request_body', + "litellm.proxy.common_utils.realtime_utils", + "", + "_realtime_request_body", + "realtime_request_body", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_commit_daily_tag_spend_to_db', - 'commit_daily_tag_spend_to_db', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_commit_daily_tag_spend_to_db", + "commit_daily_tag_spend_to_db", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_commit_daily_tag_spend_to_db_with_redis', - 'commit_daily_tag_spend_to_db_with_redis', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_commit_daily_tag_spend_to_db_with_redis", + "commit_daily_tag_spend_to_db_with_redis", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_handle_spend_update_failure', - 'handle_spend_update_failure', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_handle_spend_update_failure", + "handle_spend_update_failure", False, ), ( - 'litellm.proxy.db.db_transaction_queue.redis_update_buffer', - 'RedisUpdateBuffer', - '_should_commit_spend_updates_to_redis', - 'should_commit_spend_updates_to_redis', + "litellm.proxy.db.db_transaction_queue.redis_update_buffer", + "RedisUpdateBuffer", + "_should_commit_spend_updates_to_redis", + "should_commit_spend_updates_to_redis", False, ), ( - 'litellm.proxy.db.log_db_metrics', - '', - '_is_exception_related_to_db', - 'is_exception_related_to_db', + "litellm.proxy.db.log_db_metrics", + "", + "_is_exception_related_to_db", + "is_exception_related_to_db", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.azure.base', - '', - '_RESPONSES_API_CALL_TYPES', - 'RESPONSES_API_CALL_TYPES', + "litellm.proxy.guardrails.guardrail_hooks.azure.base", + "", + "_RESPONSES_API_CALL_TYPES", + "RESPONSES_API_CALL_TYPES", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.cisco_ai_defense.cisco_ai_defense_mcp', - '', - '_CiscoAIDefenseMcpMixin', - 'CiscoAIDefenseMcpMixin', + "litellm.proxy.guardrails.guardrail_hooks.cisco_ai_defense.cisco_ai_defense_mcp", + "", + "_CiscoAIDefenseMcpMixin", + "CiscoAIDefenseMcpMixin", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_compile_marker', - 'compile_marker', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_compile_marker", + "compile_marker", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_count_signals', - 'count_signals', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_count_signals", + "count_signals", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_word_boundary_match', - 'word_boundary_match', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_word_boundary_match", + "word_boundary_match", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.presidio', - '', - '_OPTIONAL_PresidioPIIMasking', - 'OPTIONAL_PresidioPIIMasking', + "litellm.proxy.guardrails.guardrail_hooks.presidio", + "", + "_OPTIONAL_PresidioPIIMasking", + "OPTIONAL_PresidioPIIMasking", False, ), ( - 'litellm.proxy.health_check', - '', - '_clean_endpoint_data', - 'clean_endpoint_data', + "litellm.proxy.health_check", + "", + "_clean_endpoint_data", + "clean_endpoint_data", False, ), ( - 'litellm.proxy.health_check', - '', - '_update_litellm_params_for_health_check', - 'update_litellm_params_for_health_check', + "litellm.proxy.health_check", + "", + "_update_litellm_params_for_health_check", + "update_litellm_params_for_health_check", False, ), ( - 'litellm.proxy.health_endpoints._health_endpoints', - '', - '_convert_health_check_to_dict', - 'convert_health_check_to_dict', + "litellm.proxy.health_endpoints._health_endpoints", + "", + "_convert_health_check_to_dict", + "convert_health_check_to_dict", False, ), ( - 'litellm.proxy.health_endpoints._health_endpoints', - '', - '_save_background_health_checks_to_db', - 'save_background_health_checks_to_db', + "litellm.proxy.health_endpoints._health_endpoints", + "", + "_save_background_health_checks_to_db", + "save_background_health_checks_to_db", False, ), ( - 'litellm.proxy.hooks.azure_content_safety', - '', - '_PROXY_AzureContentSafety', - 'PROXY_AzureContentSafety', + "litellm.proxy.hooks.azure_content_safety", + "", + "_PROXY_AzureContentSafety", + "PROXY_AzureContentSafety", False, ), ( - 'litellm.proxy.hooks.batch_rate_limiter', - '', - '_PROXY_BatchRateLimiter', - 'PROXY_BatchRateLimiter', + "litellm.proxy.hooks.batch_rate_limiter", + "", + "_PROXY_BatchRateLimiter", + "PROXY_BatchRateLimiter", False, ), ( - 'litellm.proxy.hooks.batch_redis_get', - '', - '_PROXY_BatchRedisRequests', - 'PROXY_BatchRedisRequests', + "litellm.proxy.hooks.batch_redis_get", + "", + "_PROXY_BatchRedisRequests", + "PROXY_BatchRedisRequests", False, ), ( - 'litellm.proxy.hooks.cache_control_check', - '', - '_PROXY_CacheControlCheck', - 'PROXY_CacheControlCheck', + "litellm.proxy.hooks.cache_control_check", + "", + "_PROXY_CacheControlCheck", + "PROXY_CacheControlCheck", False, ), ( - 'litellm.proxy.hooks.dynamic_rate_limiter', - '', - '_PROXY_DynamicRateLimitHandler', - 'PROXY_DynamicRateLimitHandler', + "litellm.proxy.hooks.dynamic_rate_limiter", + "", + "_PROXY_DynamicRateLimitHandler", + "PROXY_DynamicRateLimitHandler", False, ), ( - 'litellm.proxy.hooks.dynamic_rate_limiter_v3', - '', - '_PROXY_DynamicRateLimitHandlerV3', - 'PROXY_DynamicRateLimitHandlerV3', + "litellm.proxy.hooks.dynamic_rate_limiter_v3", + "", + "_PROXY_DynamicRateLimitHandlerV3", + "PROXY_DynamicRateLimitHandlerV3", False, ), ( - 'litellm.proxy.hooks.max_budget_per_session_limiter', - '', - '_PROXY_MaxBudgetPerSessionHandler', - 'PROXY_MaxBudgetPerSessionHandler', + "litellm.proxy.hooks.max_budget_per_session_limiter", + "", + "_PROXY_MaxBudgetPerSessionHandler", + "PROXY_MaxBudgetPerSessionHandler", False, ), ( - 'litellm.proxy.hooks.max_iterations_limiter', - '', - '_PROXY_MaxIterationsHandler', - 'PROXY_MaxIterationsHandler', + "litellm.proxy.hooks.max_iterations_limiter", + "", + "_PROXY_MaxIterationsHandler", + "PROXY_MaxIterationsHandler", False, ), ( - 'litellm.proxy.hooks.model_max_budget_limiter', - '', - '_PROXY_VirtualKeyModelMaxBudgetLimiter', - 'PROXY_VirtualKeyModelMaxBudgetLimiter', + "litellm.proxy.hooks.model_max_budget_limiter", + "", + "_PROXY_VirtualKeyModelMaxBudgetLimiter", + "PROXY_VirtualKeyModelMaxBudgetLimiter", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter', - '', - '_PROXY_MaxParallelRequestsHandler', - 'PROXY_MaxParallelRequestsHandler', + "litellm.proxy.hooks.parallel_request_limiter", + "", + "_PROXY_MaxParallelRequestsHandler", + "PROXY_MaxParallelRequestsHandler", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter_v3', - '', - '_PROXY_MaxParallelRequestsHandler_v3', - 'PROXY_MaxParallelRequestsHandler_v3', + "litellm.proxy.hooks.parallel_request_limiter_v3", + "", + "_PROXY_MaxParallelRequestsHandler_v3", + "PROXY_MaxParallelRequestsHandler_v3", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter_v3', - '_PROXY_MaxParallelRequestsHandler_v3', - '_create_rate_limit_descriptors', - 'create_rate_limit_descriptors', + "litellm.proxy.hooks.parallel_request_limiter_v3", + "_PROXY_MaxParallelRequestsHandler_v3", + "_create_rate_limit_descriptors", + "create_rate_limit_descriptors", False, ), ( - 'litellm.proxy.hooks.prompt_injection_detection', - '', - '_OPTIONAL_PromptInjectionDetection', - 'OPTIONAL_PromptInjectionDetection', + "litellm.proxy.hooks.prompt_injection_detection", + "", + "_OPTIONAL_PromptInjectionDetection", + "OPTIONAL_PromptInjectionDetection", False, ), ( - 'litellm.proxy.hooks.proxy_track_cost_callback', - '', - '_ProxyDBLogger', - 'ProxyDBLogger', + "litellm.proxy.hooks.proxy_track_cost_callback", + "", + "_ProxyDBLogger", + "ProxyDBLogger", False, ), ( - 'litellm.proxy.hooks.sensitive_data_routing', - '', - '_PROXY_SensitiveDataRoutingHandler', - 'PROXY_SensitiveDataRoutingHandler', + "litellm.proxy.hooks.sensitive_data_routing", + "", + "_PROXY_SensitiveDataRoutingHandler", + "PROXY_SensitiveDataRoutingHandler", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_add_guardrails_from_key_or_team_metadata', - 'add_guardrails_from_key_or_team_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_add_guardrails_from_key_or_team_metadata", + "add_guardrails_from_key_or_team_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_dynamic_logging_metadata', - 'get_dynamic_logging_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_dynamic_logging_metadata", + "get_dynamic_logging_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_metadata_variable_name', - 'get_metadata_variable_name', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_metadata_variable_name", + "get_metadata_variable_name", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_validated_callback_metadata', - 'get_validated_callback_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_validated_callback_metadata", + "get_validated_callback_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - 'LiteLLMProxyRequestSetup', - '_merge_tags', - 'merge_tags', + "litellm.proxy.litellm_pre_call_utils", + "LiteLLMProxyRequestSetup", + "_merge_tags", + "merge_tags", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_check_disable_global_guardrails_caller_permission', - 'check_disable_global_guardrails_caller_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_check_disable_global_guardrails_caller_permission", + "check_disable_global_guardrails_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_check_passthrough_routes_caller_permission', - 'check_passthrough_routes_caller_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_check_passthrough_routes_caller_permission", + "check_passthrough_routes_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_set_object_metadata_field', - 'set_object_metadata_field', + "litellm.proxy.management_endpoints.common_utils", + "", + "_set_object_metadata_field", + "set_object_metadata_field", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_team_member_has_permission', - 'team_member_has_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_team_member_has_permission", + "team_member_has_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_update_metadata_fields', - 'update_metadata_fields', + "litellm.proxy.management_endpoints.common_utils", + "", + "_update_metadata_fields", + "update_metadata_fields", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_upsert_budget_and_membership', - 'upsert_budget_and_membership', + "litellm.proxy.management_endpoints.common_utils", + "", + "_upsert_budget_and_membership", + "upsert_budget_and_membership", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_user_has_admin_privileges', - 'user_has_admin_privileges', + "litellm.proxy.management_endpoints.common_utils", + "", + "_user_has_admin_privileges", + "user_has_admin_privileges", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_user_has_admin_view', - 'user_api_key_has_admin_view', + "litellm.proxy.management_endpoints.common_utils", + "", + "_user_has_admin_view", + "user_api_key_has_admin_view", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_clear_hashicorp_vault_state', - 'clear_hashicorp_vault_state', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_clear_hashicorp_vault_state", + "clear_hashicorp_vault_state", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_get_current_env_values', - 'get_current_env_values', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_get_current_env_values", + "get_current_env_values", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_parse_config_value', - 'parse_config_value', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_parse_config_value", + "parse_config_value", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_set_env_vars', - 'set_env_vars', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_set_env_vars", + "set_env_vars", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_calculate_key_rotation_time', - 'calculate_key_rotation_time', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_calculate_key_rotation_time", + "calculate_key_rotation_time", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_check_permissions_caller_permission', - 'check_permissions_caller_permission', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_check_permissions_caller_permission", + "check_permissions_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_get_caller_team_role', - 'get_caller_team_role', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_get_caller_team_role", + "get_caller_team_role", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_list_key_helper', - 'list_key_helper', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_list_key_helper", + "list_key_helper", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_persist_deleted_verification_tokens', - 'persist_deleted_verification_tokens', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_persist_deleted_verification_tokens", + "persist_deleted_verification_tokens", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_rotate_master_key', - 'rotate_master_key', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_rotate_master_key", + "rotate_master_key", False, ), ( - 'litellm.proxy.management_endpoints.mcp_management_endpoints', - '', - '_inherit_credentials_from_existing_server', - 'inherit_credentials_from_existing_server', + "litellm.proxy.management_endpoints.mcp_management_endpoints", + "", + "_inherit_credentials_from_existing_server", + "inherit_credentials_from_existing_server", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_add_model_to_db', - 'add_model_to_db', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_add_model_to_db", + "add_model_to_db", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_add_team_model_to_db', - 'add_team_model_to_db', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_add_team_model_to_db", + "add_team_model_to_db", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_deduplicate_litellm_router_models', - 'deduplicate_litellm_router_models', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_deduplicate_litellm_router_models", + "deduplicate_litellm_router_models", False, ), ( - 'litellm.proxy.management_endpoints.organization_endpoints', - '', - '_verify_org_access', - 'verify_org_access', + "litellm.proxy.management_endpoints.organization_endpoints", + "", + "_verify_org_access", + "verify_org_access", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_all_names_per_competitor', - 'build_all_names_per_competitor', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_all_names_per_competitor", + "build_all_names_per_competitor", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_comparison_blocked_words', - 'build_comparison_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_comparison_blocked_words", + "build_comparison_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_competitor_guardrail_definitions', - 'build_competitor_guardrail_definitions', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_competitor_guardrail_definitions", + "build_competitor_guardrail_definitions", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_name_blocked_words', - 'build_name_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_name_blocked_words", + "build_name_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_recommendation_blocked_words', - 'build_recommendation_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_recommendation_blocked_words", + "build_recommendation_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_refinement_prompt', - 'build_refinement_prompt', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_refinement_prompt", + "build_refinement_prompt", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_clean_competitor_line', - 'clean_competitor_line', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_clean_competitor_line", + "clean_competitor_line", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_parse_variations_response', - 'parse_variations_response', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_parse_variations_response", + "parse_variations_response", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - '', - '_cleanup_members_with_roles', - 'cleanup_members_with_roles', + "litellm.proxy.management_endpoints.team_endpoints", + "", + "_cleanup_members_with_roles", + "cleanup_members_with_roles", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - '', - '_refresh_cached_team', - 'refresh_cached_team', + "litellm.proxy.management_endpoints.team_endpoints", + "", + "_refresh_cached_team", + "refresh_cached_team", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - 'TeamMemberBudgetHandler', - '_clean_team_member_fields', - 'clean_team_member_fields', + "litellm.proxy.management_endpoints.team_endpoints", + "TeamMemberBudgetHandler", + "_clean_team_member_fields", + "clean_team_member_fields", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - '', - '_sso_return_to_redirect', - 'sso_return_to_redirect', + "litellm.proxy.management_endpoints.ui_sso", + "", + "_sso_return_to_redirect", + "sso_return_to_redirect", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_delete_pkce_verifier', - 'delete_pkce_verifier', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_delete_pkce_verifier", + "delete_pkce_verifier", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_get_cli_state', - 'get_cli_state', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_get_cli_state", + "get_cli_state", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_get_user_email_and_id_from_result', - 'get_user_email_and_id_from_result', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_get_user_email_and_id_from_result", + "get_user_email_and_id_from_result", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_pkce_token_exchange', - 'pkce_token_exchange', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_pkce_token_exchange", + "pkce_token_exchange", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_validate_return_to', - 'validate_return_to', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_validate_return_to", + "validate_return_to", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_get_allow_all_keys_server_ids', - 'get_allow_all_keys_server_ids', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_get_allow_all_keys_server_ids", + "get_allow_all_keys_server_ids", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_get_team_allowed_mcp_servers', - 'get_team_allowed_mcp_servers', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_get_team_allowed_mcp_servers", + "get_team_allowed_mcp_servers", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_set_object_permission', - 'set_object_permission', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_set_object_permission", + "set_object_permission", False, ), ( - 'litellm.proxy.openai_files_endpoints.common_utils', - '', - '_is_base64_encoded_unified_file_id', - 'is_base64_encoded_unified_file_id', + "litellm.proxy.openai_files_endpoints.common_utils", + "", + "_is_base64_encoded_unified_file_id", + "is_base64_encoded_unified_file_id", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints', - '', - '_extract_model_from_bedrock_endpoint', - 'extract_model_from_bedrock_endpoint', + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints", + "", + "_extract_model_from_bedrock_endpoint", + "extract_model_from_bedrock_endpoint", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints', - 'BaseOpenAIPassThroughHandler', - '_base_openai_pass_through_handler', - 'base_openai_pass_through_handler', + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints", + "BaseOpenAIPassThroughHandler", + "_base_openai_pass_through_handler", + "base_openai_pass_through_handler", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_build_complete_streaming_response', - 'build_complete_streaming_response', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_build_complete_streaming_response", + "build_complete_streaming_response", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_build_usage_only_response_from_chunks', - 'build_usage_only_response_from_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_build_usage_only_response_from_chunks", + "build_usage_only_response_from_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_handle_logging_anthropic_collected_chunks', - 'handle_logging_anthropic_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_handle_logging_anthropic_collected_chunks", + "handle_logging_anthropic_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_get_assembly_base_url_from_region', - 'get_assembly_base_url_from_region', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_get_assembly_base_url_from_region", + "get_assembly_base_url_from_region", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_get_assembly_region_from_url', - 'get_assembly_region_from_url', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_get_assembly_region_from_url", + "get_assembly_region_from_url", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_should_log_request', - 'should_log_request', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_should_log_request", + "should_log_request", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler', - '', - '_is_openai_compatible_url', - 'is_openai_compatible_url', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler", + "", + "_is_openai_compatible_url", + "is_openai_compatible_url", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler', - 'OpenAIPassthroughLoggingHandler', - '_handle_logging_openai_collected_chunks', - 'handle_logging_openai_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler", + "OpenAIPassthroughLoggingHandler", + "_handle_logging_openai_collected_chunks", + "handle_logging_openai_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler', - 'VertexPassthroughLoggingHandler', - '_handle_logging_vertex_collected_chunks', - 'handle_logging_vertex_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler", + "VertexPassthroughLoggingHandler", + "_handle_logging_vertex_collected_chunks", + "handle_logging_vertex_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.pass_through_endpoints', - 'HttpPassThroughEndpointHelpers', - '_init_kwargs_for_pass_through_endpoint', - 'init_kwargs_for_pass_through_endpoint', + "litellm.proxy.pass_through_endpoints.pass_through_endpoints", + "HttpPassThroughEndpointHelpers", + "_init_kwargs_for_pass_through_endpoint", + "init_kwargs_for_pass_through_endpoint", False, ), ( - 'litellm.proxy.pass_through_endpoints.pass_through_endpoints', - 'HttpPassThroughEndpointHelpers', - '_update_stream_param_based_on_request_body', - 'update_stream_param_based_on_request_body', + "litellm.proxy.pass_through_endpoints.pass_through_endpoints", + "HttpPassThroughEndpointHelpers", + "_update_stream_param_based_on_request_body", + "update_stream_param_based_on_request_body", False, ), ( - 'litellm.proxy.pass_through_endpoints.streaming_handler', - 'PassThroughStreamingHandler', - '_route_streaming_logging_to_handler', - 'route_streaming_logging_to_handler', + "litellm.proxy.pass_through_endpoints.streaming_handler", + "PassThroughStreamingHandler", + "_route_streaming_logging_to_handler", + "route_streaming_logging_to_handler", False, ), ( - 'litellm.proxy.pass_through_endpoints.success_handler', - 'PassThroughEndpointLogging', - '_handle_logging', - 'handle_logging', + "litellm.proxy.pass_through_endpoints.success_handler", + "PassThroughEndpointLogging", + "_handle_logging", + "handle_logging", False, ), ( - 'litellm.proxy.policy_engine.policy_registry', - 'PolicyRegistry', - '_parse_policy', - 'parse_policy', + "litellm.proxy.policy_engine.policy_registry", + "PolicyRegistry", + "_parse_policy", + "parse_policy", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_apply_uvicorn_max_requests_jitter', - 'apply_uvicorn_max_requests_jitter', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_apply_uvicorn_max_requests_jitter", + "apply_uvicorn_max_requests_jitter", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_configure_dev_reload', - 'configure_dev_reload', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_configure_dev_reload", + "configure_dev_reload", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_echo_litellm_version', - 'echo_litellm_version', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_echo_litellm_version", + "echo_litellm_version", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_get_default_unvicorn_init_args', - 'get_default_unvicorn_init_args', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_get_default_unvicorn_init_args", + "get_default_unvicorn_init_args", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_get_loop_type', - 'get_loop_type', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_get_loop_type", + "get_loop_type", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_init_granian_server', - 'init_granian_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_init_granian_server", + "init_granian_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_init_hypercorn_server', - 'init_hypercorn_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_init_hypercorn_server", + "init_hypercorn_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_is_port_in_use', - 'is_port_in_use', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_is_port_in_use", + "is_port_in_use", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_maybe_setup_prometheus_multiproc_dir', - 'maybe_setup_prometheus_multiproc_dir', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_maybe_setup_prometheus_multiproc_dir", + "maybe_setup_prometheus_multiproc_dir", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_config_validation', - 'run_config_validation', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_config_validation", + "run_config_validation", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_gunicorn_server', - 'run_gunicorn_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_gunicorn_server", + "run_gunicorn_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_health_check', - 'run_health_check', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_health_check", + "run_health_check", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_ollama_serve', - 'run_ollama_serve', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_ollama_serve", + "run_ollama_serve", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_test_chat_completion', - 'run_test_chat_completion', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_test_chat_completion", + "run_test_chat_completion", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_build_redis_usage_cache', - 'build_redis_usage_cache', + "litellm.proxy.proxy_server", + "", + "_build_redis_usage_cache", + "build_redis_usage_cache", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_ensure_spend_counter_initialized', - 'ensure_spend_counter_initialized', + "litellm.proxy.proxy_server", + "", + "_ensure_spend_counter_initialized", + "ensure_spend_counter_initialized", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_ensure_window_spend_counter_initialized', - 'ensure_window_spend_counter_initialized', + "litellm.proxy.proxy_server", + "", + "_ensure_window_spend_counter_initialized", + "ensure_window_spend_counter_initialized", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_environment_has_redis_connection_target', - 'environment_has_redis_connection_target', + "litellm.proxy.proxy_server", + "", + "_environment_has_redis_connection_target", + "environment_has_redis_connection_target", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_get_model_group_info', - 'get_model_group_info', + "litellm.proxy.proxy_server", + "", + "_get_model_group_info", + "get_model_group_info", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_increment_spend_counter_cache', - 'increment_spend_counter_cache', + "litellm.proxy.proxy_server", + "", + "_increment_spend_counter_cache", + "increment_spend_counter_cache", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_initialize_shared_aiohttp_session', - 'initialize_shared_aiohttp_session', + "litellm.proxy.proxy_server", + "", + "_initialize_shared_aiohttp_session", + "initialize_shared_aiohttp_session", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_invalidate_spend_counter', - 'invalidate_spend_counter', + "litellm.proxy.proxy_server", + "", + "_invalidate_spend_counter", + "invalidate_spend_counter", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_title', - 'title', + "litellm.proxy.proxy_server", + "", + "_title", + "title", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_add_deployment_locked', - 'add_deployment_locked', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_add_deployment_locked", + "add_deployment_locked", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_decrypt_and_set_db_env_variables', - 'decrypt_and_set_db_env_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_decrypt_and_set_db_env_variables", + "decrypt_and_set_db_env_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_decrypt_db_variables', - 'decrypt_db_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_decrypt_db_variables", + "decrypt_db_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_encrypt_env_variables', - 'encrypt_env_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_encrypt_env_variables", + "encrypt_env_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_encrypt_env_variables_for_db', - 'encrypt_env_variables_for_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_encrypt_env_variables_for_db", + "encrypt_env_variables_for_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_get_models_from_db', - 'get_models_from_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_get_models_from_db", + "get_models_from_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_init_cache', - 'init_cache', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_init_cache", + "init_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_init_semantic_filter_settings_in_db', - 'init_semantic_filter_settings_in_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_init_semantic_filter_settings_in_db", + "init_semantic_filter_settings_in_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_serve_pass_through_endpoints', - 'serve_pass_through_endpoints', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_serve_pass_through_endpoints", + "serve_pass_through_endpoints", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_add_proxy_budget_to_db', - 'add_proxy_budget_to_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_add_proxy_budget_to_db", + "add_proxy_budget_to_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_attach_router_to_prompt_injection_detectors', - 'attach_router_to_prompt_injection_detectors', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_attach_router_to_prompt_injection_detectors", + "attach_router_to_prompt_injection_detectors", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_get_transaction_buffer_redis_cache', - 'get_transaction_buffer_redis_cache', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_get_transaction_buffer_redis_cache", + "get_transaction_buffer_redis_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_coordination_redis_from_db', - 'init_coordination_redis_from_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_coordination_redis_from_db", + "init_coordination_redis_from_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_dd_tracer', - 'init_dd_tracer', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_dd_tracer", + "init_dd_tracer", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_pyroscope', - 'init_pyroscope', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_pyroscope", + "init_pyroscope", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_jwt_auth', - 'initialize_jwt_auth', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_jwt_auth", + "initialize_jwt_auth", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_semantic_tool_filter', - 'initialize_semantic_tool_filter', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_semantic_tool_filter", + "initialize_semantic_tool_filter", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_startup_logging', - 'initialize_startup_logging', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_startup_logging", + "initialize_startup_logging", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_setup_prisma_client', - 'setup_prisma_client', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_setup_prisma_client", + "setup_prisma_client", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_sync_ui_settings_to_general_settings', - 'sync_ui_settings_to_general_settings', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_sync_ui_settings_to_general_settings", + "sync_ui_settings_to_general_settings", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_update_default_team_member_budget', - 'update_default_team_member_budget', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_update_default_team_member_budget", + "update_default_team_member_budget", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_validate_redis_transaction_buffer_config', - 'validate_redis_transaction_buffer_config', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_validate_redis_transaction_buffer_config", + "validate_redis_transaction_buffer_config", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warm_global_spend_cache', - 'warm_global_spend_cache', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warm_global_spend_cache", + "warm_global_spend_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_budget_without_db', - 'warn_budget_without_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_budget_without_db", + "warn_budget_without_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_fail_closed_rate_limits_without_redis', - 'warn_fail_closed_rate_limits_without_redis', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_fail_closed_rate_limits_without_redis", + "warn_fail_closed_rate_limits_without_redis", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_if_mock_testing_params_enabled', - 'warn_if_mock_testing_params_enabled', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_if_mock_testing_params_enabled", + "warn_if_mock_testing_params_enabled", False, ), ( - 'litellm.proxy.spend_tracking.spend_management_endpoints', - '', - '_get_spend_report_for_time_range', - 'get_spend_report_for_time_range', + "litellm.proxy.spend_tracking.spend_management_endpoints", + "", + "_get_spend_report_for_time_range", + "get_spend_report_for_time_range", False, ), ( - 'litellm.proxy.spend_tracking.spend_management_endpoints', - '', - '_is_admin_view_safe', - 'is_admin_view_safe', + "litellm.proxy.spend_tracking.spend_management_endpoints", + "", + "_is_admin_view_safe", + "is_admin_view_safe", False, ), ( - 'litellm.proxy.spend_tracking.spend_tracking_utils', - '', - '_is_master_key', - 'is_master_key', + "litellm.proxy.spend_tracking.spend_tracking_utils", + "", + "_is_master_key", + "is_master_key", False, ), ( - 'litellm.proxy.spend_tracking.spend_tracking_utils', - '', - '_sanitize_error_information_for_spend_logs', - 'sanitize_error_information_for_spend_logs', + "litellm.proxy.spend_tracking.spend_tracking_utils", + "", + "_sanitize_error_information_for_spend_logs", + "sanitize_error_information_for_spend_logs", False, ), ( - 'litellm.proxy.utils', - '', - '_cache_user_row', - 'cache_user_row', + "litellm.proxy.utils", + "", + "_cache_user_row", + "cache_user_row", False, ), ( - 'litellm.proxy.utils', - '', - '_check_and_merge_model_level_guardrails', - 'check_and_merge_model_level_guardrails', + "litellm.proxy.utils", + "", + "_check_and_merge_model_level_guardrails", + "check_and_merge_model_level_guardrails", False, ), ( - 'litellm.proxy.utils', - '', - '_get_docs_url', - 'get_docs_url', + "litellm.proxy.utils", + "", + "_get_docs_url", + "get_docs_url", False, ), ( - 'litellm.proxy.utils', - '', - '_get_openapi_url', - 'get_openapi_url', + "litellm.proxy.utils", + "", + "_get_openapi_url", + "get_openapi_url", False, ), ( - 'litellm.proxy.utils', - '', - '_get_projected_spend_over_limit', - 'get_projected_spend_over_limit', + "litellm.proxy.utils", + "", + "_get_projected_spend_over_limit", + "get_projected_spend_over_limit", False, ), ( - 'litellm.proxy.utils', - '', - '_get_redoc_url', - 'get_redoc_url', + "litellm.proxy.utils", + "", + "_get_redoc_url", + "get_redoc_url", False, ), ( - 'litellm.proxy.utils', - '', - '_hash_token_if_needed', - 'hash_token_if_needed', + "litellm.proxy.utils", + "", + "_hash_token_if_needed", + "hash_token_if_needed", False, ), ( - 'litellm.proxy.utils', - '', - '_is_projected_spend_over_limit', - 'is_projected_spend_over_limit', + "litellm.proxy.utils", + "", + "_is_projected_spend_over_limit", + "is_projected_spend_over_limit", False, ), ( - 'litellm.proxy.utils', - '', - '_is_valid_team_configs', - 'is_valid_team_configs', + "litellm.proxy.utils", + "", + "_is_valid_team_configs", + "is_valid_team_configs", False, ), ( - 'litellm.proxy.utils', - '', - '_monitor_spend_logs_queue', - 'monitor_spend_logs_queue', + "litellm.proxy.utils", + "", + "_monitor_spend_logs_queue", + "monitor_spend_logs_queue", False, ), ( - 'litellm.proxy.utils', - '', - '_premium_user_check', - 'premium_user_check', + "litellm.proxy.utils", + "", + "_premium_user_check", + "premium_user_check", False, ), ( - 'litellm.proxy.utils', - '', - '_raise_failed_update_spend_exception', - 'raise_failed_update_spend_exception', + "litellm.proxy.utils", + "", + "_raise_failed_update_spend_exception", + "raise_failed_update_spend_exception", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_arelease_max_parallel_requests_on_disconnect', - 'arelease_max_parallel_requests_on_disconnect', + "litellm.proxy.utils", + "ProxyLogging", + "_arelease_max_parallel_requests_on_disconnect", + "arelease_max_parallel_requests_on_disconnect", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_callback_capabilities', - 'callback_capabilities', + "litellm.proxy.utils", + "ProxyLogging", + "_callback_capabilities", + "callback_capabilities", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_convert_mcp_hook_response_to_kwargs', - 'convert_mcp_hook_response_to_kwargs', + "litellm.proxy.utils", + "ProxyLogging", + "_convert_mcp_hook_response_to_kwargs", + "convert_mcp_hook_response_to_kwargs", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_create_mcp_request_object_from_kwargs', - 'create_mcp_request_object_from_kwargs', + "litellm.proxy.utils", + "ProxyLogging", + "_create_mcp_request_object_from_kwargs", + "create_mcp_request_object_from_kwargs", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_fire_deferred_stream_logging', - 'fire_deferred_stream_logging', + "litellm.proxy.utils", + "ProxyLogging", + "_fire_deferred_stream_logging", + "fire_deferred_stream_logging", False, ), ) @@ -5188,17 +5201,16 @@ LLMS_FORWARDER_CASES: Final = ( ) PROXY_FORWARDER_CASES: Final = ( ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_convert_mcp_to_llm_format', - 'convert_mcp_to_llm_format', - 'instance', + "litellm.proxy.utils", + "ProxyLogging", + "_convert_mcp_to_llm_format", + "convert_mcp_to_llm_format", + "instance", False, ), ) - def _get_owner(module_path: str, owner_name: str) -> object: module: Final = import_module(module_path or "litellm") if module_path == "litellm.llms.openai_like.dynamic_config" and owner_name == "JSONProviderConfig": @@ -5292,14 +5304,10 @@ def _make_private_override( def recorded_args(args: tuple[object, ...]) -> tuple[object, ...]: return args[1:] if descriptor in ("instance", "classmethod") else args - def sync_override( - *args: object, **kwargs: object - ) -> tuple[object, tuple[object, ...], dict[str, object]]: + def sync_override(*args: object, **kwargs: object) -> tuple[object, tuple[object, ...], dict[str, object]]: return result, recorded_args(args), kwargs - async def async_override( - *args: object, **kwargs: object - ) -> tuple[object, tuple[object, ...], dict[str, object]]: + async def async_override(*args: object, **kwargs: object) -> tuple[object, tuple[object, ...], dict[str, object]]: return result, recorded_args(args), kwargs implementation: Final = async_override if is_async else sync_override