From 3262029c990c95408db68f10ec44ca9d3b9344cc Mon Sep 17 00:00:00 2001 From: narmstrong-berry Date: Tue, 10 Nov 2026 20:28:06 +0000 Subject: [PATCH] feat(messages): serve response cache reads on rust --- litellm-rust/Cargo.lock | 1 + .../crates/cache-response/src/service.rs | 4 + .../python_contract.json | 12 +- .../callbacks-legacy-python/src/adapter.rs | 46 +- .../callbacks-legacy-python/src/logger.rs | 6 +- .../callbacks-legacy-python/src/python.rs | 2 + .../src/test_support.rs | 5 +- .../crates/inference-messages/Cargo.toml | 1 + .../crates/inference-messages/src/handler.rs | 81 +- .../crates/inference-messages/src/lib.rs | 3 + .../inference-messages/tests/caching.rs | 86 + litellm-rust/crates/inference/AGENTS.md | 1 + litellm-rust/crates/inference/src/caching.rs | 9 +- .../python-bridge/src/cache/python/AGENTS.md | 2 +- .../python-bridge/src/cache/python/host.rs | 25 +- .../python-bridge/src/cache/python/service.rs | 4 + .../python-bridge/src/cache/selection.rs | 32 +- .../python-bridge/src/routes/messages/host.rs | 9 +- litellm/caching/caching_handler.py | 282 ++- litellm/messages/dispatch.py | 2 - litellm/rust_bridge/catalog.py | 1 - litellm/rust_bridge/host/cache.py | 109 +- .../host/callbacks_legacy_python.py | 25 +- litellm/rust_bridge/host/routes/messages.py | 7 + litellm/rust_bridge/messages/detectors.py | 27 - .../sdk/messages/test_native_parity.py | 81 +- .../cache/test_python_cache.py | 122 +- .../messages/test_dispatch.py | 51 +- .../host/test_callbacks_legacy_python.py | 3 +- tests/unit/rust_bridge/test_detectors.py | 49 +- tests/unit/test_private_usage_aliases.py | 2130 +++++++++-------- 31 files changed, 1865 insertions(+), 1353 deletions(-) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ed658e074ee..d5fc960b39f 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4158,6 +4158,7 @@ dependencies = [ "futures-util", "litellm-auth", "litellm-auth-aws", + "litellm-cache", "litellm-cache-memory", "litellm-cache-response", "litellm-core-utils", diff --git a/litellm-rust/crates/cache-response/src/service.rs b/litellm-rust/crates/cache-response/src/service.rs index 0bdf948ec48..ed4dccdc355 100644 --- a/litellm-rust/crates/cache-response/src/service.rs +++ b/litellm-rust/crates/cache-response/src/service.rs @@ -27,6 +27,10 @@ impl Default for ResponseCacheConfig { pub trait ResponseCacheService: Send + Sync { fn config(&self) -> &ResponseCacheConfig; + fn keyed_by_caller(&self) -> bool { + false + } + fn lookup<'a>( &'a self, request: &'a ResponseCacheRequest, diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index 0378e3621c0..f19aa9bc426 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -11,7 +11,8 @@ "logger", "kwargs", "start_time", - "end_time" + "end_time", + "cache_hit" ], "update_logging": [ "logger", @@ -95,6 +96,12 @@ "start", "end" ], + "cache_hit_success": [ + "logger", + "response", + "start", + "end" + ], "enqueue_logging": [ "coroutine" ], @@ -126,7 +133,8 @@ "logger", "kwargs", "start_time", - "end_time" + "end_time", + "cache_hit" ], "stream_success": [ "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 2c30becab97..9210edb95fd 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -273,6 +273,7 @@ impl LegacyLogging { self.call.kwargs(), &self.start, &self.end, + self.messages_cache_hit(), )?; self.response .as_ref() @@ -280,6 +281,25 @@ impl LegacyLogging { .ok_or_else(|| HostError::MissingState.into()) } + fn messages_cache_hit(&self) -> bool { + self.operation == LoggingOperation::Messages && self.cache_key.is_some() + } + + fn dispatch_cache_hit(&self, py: Python<'_>) -> PyResult<()> { + let logger = self.logger()?; + let logged = Logging::CacheHitSuccess.call( + py, + (logger.object(py), &self.response, &self.start, &self.end), + ); + match logged { + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(logger.object(py))); + Ok(()) + } + result => result.map(|_| ()), + } + } + fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> { match self.try_dispatch_success(py) { Err(error) if error.is_instance_of::(py) => { @@ -576,7 +596,7 @@ impl LegacyLogging { ) -> PyResult>> { self.end = Some(datetime(py, timing.end_time)?); self.response = Some(response); - if self.runs_deployment_hooks() { + if self.runs_deployment_hooks() && !self.messages_cache_hit() { return Ok(HookStep::Await( DeploymentHooks::after_success( py, @@ -597,7 +617,17 @@ impl LegacyLogging { ) -> PyResult> { use litellm_host::interceptors::{Cost, ResultSource}; + self.cache_key = match &facts.source { + ResultSource::Provider => None, + ResultSource::Cache { key } => Some(key.clone()), + }; let logger = self.logger()?.object(py); + if self.messages_cache_hit() { + logger + .getattr("model_call_details")? + .set_item("cache_hit", true)?; + return Ok(HookStep::Ready(())); + } let params = logger .getattr("litellm_params")? .cast_into::()? @@ -625,10 +655,6 @@ impl LegacyLogging { }; logger.setattr("_provider_reported_cost", reported_cost)?; details.call_method1("pop", ("response_cost", py.None()))?; - self.cache_key = match &facts.source { - ResultSource::Provider => None, - ResultSource::Cache { key } => Some(key.clone()), - }; if self.cache_key.is_some() { details.set_item("cache_hit", true)?; } else { @@ -717,6 +743,7 @@ impl LegacyLogging { self.response = Some(response.clone_ref(py)); match &self.stream { Some(stream) => self.stream_success(py, stream, false)?, + None if self.messages_cache_hit() => self.dispatch_cache_hit(py)?, None => self.dispatch_success(py)?, } Ok(HookStep::Ready(())) @@ -751,7 +778,11 @@ impl LegacyLogging { if self.stream_billing().is_none() { return Err(HostError::MissingState.into()); } - if let Some(key) = &self.cache_key { + if let Some(key) = self + .cache_key + .as_ref() + .filter(|_| !self.messages_cache_hit()) + { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } @@ -766,6 +797,7 @@ impl LegacyLogging { self.call.kwargs(), &self.start, datetime(py, epoch_seconds())?, + self.messages_cache_hit(), ), )?; } @@ -790,7 +822,7 @@ impl LegacyLogging { // A caller that goes away mid-stream cancels the call before it ends: Python's Messages // stream iterator still bills the chunks it read, straight away rather than deferred. fn bill_abandoned_stream(&mut self, py: Python<'_>) -> PyResult<()> { - if !self.asynchronous || self.end.is_some() { + if !self.asynchronous || self.end.is_some() || self.messages_cache_hit() { return Ok(()); } let Some(stream) = self.stream.take() else { diff --git a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs index 38f3bf29828..3019d87a20b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs @@ -77,8 +77,12 @@ pub fn finalize( kwargs: &Py, start: &Py, end: &Option>, + cache_hit: bool, ) -> PyResult<()> { - Wrapper::Finalize.call(py, (response, logger.object(py), kwargs, start, end))?; + Wrapper::Finalize.call( + py, + (response, logger.object(py), kwargs, start, end, cache_hit), + )?; Ok(()) } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index 7b96301832e..5f9291e42cc 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -64,6 +64,8 @@ pub(crate) enum Logging { SubmitSuccess, #[strum(serialize = "async_success_handler")] AsyncSuccessHandler, + #[strum(serialize = "cache_hit_success")] + CacheHitSuccess, #[strum(serialize = "mark_async_success_enqueued")] MarkAsyncSuccessEnqueued, #[strum(serialize = "enqueue_logging")] diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index f54fd88aefe..5c8fb038719 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -53,7 +53,7 @@ def setup(call_type, args, kwargs, start, asynchronous): FAKES = { 'setup': setup, - 'finalize': lambda response, logger, kwargs, start, end: logger.record('finalize', response), + 'finalize': lambda response, logger, kwargs, start, end, cache_hit: logger.record('finalize', response), 'update_logging': lambda logger, kwargs, model, optional_params, litellm_params, provider: logger.update_from_kwargs( kwargs=kwargs, model=model, @@ -90,6 +90,7 @@ FAKES = { )(error, ''.join(traceback.format_exception(error)), start, end), 'submit_success': lambda logger, response, start, end: logger.record('submit', (response, start, end)), 'async_success_handler': lambda logger, response, start, end: logger.async_success_handler(response, start, end), + 'cache_hit_success': lambda logger, response, start, end: logger.record('cache_hit_success', response), 'mark_async_success_enqueued': lambda logger: logger.record('mark_async_success_enqueued', None), 'enqueue_logging': lambda coroutine: coroutine.enqueue(), 'restore_context': lambda logger: logger.record('restore', None), @@ -101,7 +102,7 @@ FAKES = { ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), 'stream_opened': lambda logger, head: logger.record('stream_opened', head), - 'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head), + 'stamp_stream_head': lambda head, logger, kwargs, start, end, cache_hit: logger.record('stamp_stream_head', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml index 0c5be52550d..c0b6cd8a997 100644 --- a/litellm-rust/crates/inference-messages/Cargo.toml +++ b/litellm-rust/crates/inference-messages/Cargo.toml @@ -30,6 +30,7 @@ tracing.workspace = true [dev-dependencies] litellm-error.workspace = true litellm-framer = { workspace = true, features = ["sse"] } +litellm-cache.workspace = true litellm-cache-memory.workspace = true litellm-host-native.workspace = true litellm-http = { workspace = true, features = ["test-support"] } diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 1694aa6a9a3..456dac28c80 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -35,7 +35,11 @@ use super::{ route::Messages, types::invalid_request, }; -use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request}; +use litellm_inference::{ + caching::{CacheRequest, CallCache}, + context::CallContext, + outbound::outbound_request, +}; struct ProviderCall { identity: ProviderIdentity, @@ -54,6 +58,42 @@ impl ProviderCall { } } +pub(super) async fn cached( + route: &MessagesRoute, + provider: MessagesProvider, + model: &str, + context: &CallContext<'_, impl Interceptors>, +) -> Result, Error> { + let Some(cache) = caller_keyed(route) else { + return Ok(None); + }; + let identity = ProviderIdentity { + model: model.to_owned(), + provider: provider.as_str().to_string(), + }; + let Some((output, source)) = CallCache::::new( + Some(cache), + context.cache, + CacheRequest::from_wire(identity.clone(), None), + ) + .lookup() + .await + else { + return Ok(None); + }; + context + .result_ready(execution_facts(provider, identity, &output, &source)) + .await?; + Ok(Some(output)) +} + +fn caller_keyed(route: &MessagesRoute) -> Option<&litellm_cache_response::ScopedCache> { + route + .cache + .as_ref() + .filter(|cache| cache.service.keyed_by_caller()) +} + pub(super) async fn execute( route: &MessagesRoute, provider: MessagesProvider, @@ -62,6 +102,7 @@ pub(super) async fn execute( secrets: Secrets, context: &CallContext<'_, impl Interceptors>, ) -> Result { + let caller_keyed = caller_keyed(route).is_some(); let request = provider_call( &route.auth, provider, @@ -73,34 +114,50 @@ pub(super) async fn execute( .boxed() .await?; let cache = CallCache::::from_wire( - route.cache.as_ref().filter(|_| request.cacheable()), + route + .cache + .as_ref() + .filter(|_| caller_keyed || request.cacheable()), context.cache, &request.identity, &request.wire, ); let identity = request.identity.clone(); - let (output, source) = match cache.lookup().await { + let hit = match caller_keyed { + true => None, + false => cache.lookup().await, + }; + let (output, source) = match hit { Some(hit) => hit, None => ( call_provider(&route.http, request, context).await?, ResultSource::Provider, ), }; - let reported = match &output { + context + .result_ready(execution_facts(provider, identity, &output, &source)) + .await?; + Ok(cache.finish(output, &source).await) +} + +fn execution_facts( + provider: MessagesProvider, + identity: ProviderIdentity, + output: &MessagesCallResponse, + source: &ResultSource, +) -> ExecutionFacts { + let reported = match output { MessagesCallResponse::Complete(response) => match response { MessagesBody::Message(message) => provider.config().reported_cost(message), MessagesBody::Unchecked(_) => None, }, MessagesCallResponse::Stream { .. } => None, }; - context - .result_ready(ExecutionFacts { - provider: identity, - cost: Cost::settle(&source, reported), - source: source.clone(), - }) - .await?; - Ok(cache.finish(output, &source).await) + ExecutionFacts { + provider: identity, + cost: Cost::settle(source, reported), + source: source.clone(), + } } async fn provider_call( diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index 23dd8f4a920..ca9e0345ffe 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -94,6 +94,9 @@ impl MessagesRoute { <&'static str>::from(provider.provider), ); let model = provider.model.to_owned(); + if let Some(output) = handler::cached(self, config, &model, &context).await? { + return Ok(output); + } let secrets = self.secrets.resolve(config.config().secret_names()).await?; handler::execute(self, config, model, call, secrets, &context) .boxed() diff --git a/litellm-rust/crates/inference-messages/tests/caching.rs b/litellm-rust/crates/inference-messages/tests/caching.rs index 787501a8fed..786d86c3ff3 100644 --- a/litellm-rust/crates/inference-messages/tests/caching.rs +++ b/litellm-rust/crates/inference-messages/tests/caching.rs @@ -301,3 +301,89 @@ async fn provider_reported_cost_is_delivered_once_and_never_charged_for_a_cache_ assert!(matches!(cached.source, ResultSource::Cache { .. })); assert_eq!(cached.cost, Cost::CacheHit); } + +#[derive(Default)] +struct CallerKeyed { + config: ResponseCacheConfig, + entry: std::sync::Mutex>, +} + +impl ResponseCacheService for CallerKeyed { + fn config(&self) -> &ResponseCacheConfig { + &self.config + } + + fn keyed_by_caller(&self) -> bool { + true + } + + fn lookup<'a>( + &'a self, + _: &'a litellm_cache_response::ResponseCacheRequest, + _: Duration, + ) -> std::pin::Pin< + Box< + dyn std::future::Future, litellm_cache::Error>> + + Send + + 'a, + >, + > { + Box::pin(async move { Ok(self.entry.lock().unwrap().clone()) }) + } + + fn store<'a>( + &'a self, + _: &'a litellm_cache_response::ResponseCacheRequest, + value: Value, + _: Duration, + ) -> std::pin::Pin< + Box> + Send + 'a>, + > { + Box::pin(async move { + *self.entry.lock().unwrap() = Some(value); + Ok(()) + }) + } +} + +#[tokio::test] +async fn a_caller_keyed_hit_is_served_before_the_request_is_built_or_intercepted() { + use litellm_inference_messages::{MessagesCall, MessagesCallResponse}; + + let response = json!({"id": "msg_caller", "type": "message", "role": "assistant", "model": "native-test", + "content": [{"type": "text", "text": "answer"}], "stop_reason": "end_turn", "stop_sequence": null, + "usage": {"input_tokens": 3, "output_tokens": 2}}); + let upstream = support::upstream([support::json_response(response)]).await; + let hooks = ChangingHooks::default(); + let route = support::messages_route(litellm_inference_testing::no_secrets()).with_cache( + ScopedCache::new(Arc::new(CallerKeyed::default()), CacheScope::Shared), + ); + let request = |api_key: Option<&str>| { + MessagesCall { + body: serde_json::from_value(json!({"model": "native-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})).unwrap(), + api_key: api_key.map(Into::into), api_base: Some(upstream.uri()), custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), extra_headers: None, provider_specific_header: None, + timeout: None, shaping: Default::default(), + } + }; + let first = route + .execute(request(Some("key")), &hooks, None) + .await + .unwrap(); + let second = route.execute(request(None), &hooks, None).await.unwrap(); + let (MessagesCallResponse::Complete(first), MessagesCallResponse::Complete(second)) = + (first, second) + else { + panic!("expected Messages responses"); + }; + assert_eq!(first, second); + assert_eq!(support::received(&upstream).await.len(), 1); + assert_eq!(hooks.calls.load(Ordering::SeqCst), 1); + let facts = hooks.facts.lock().unwrap(); + let [provider, cached] = facts.as_slice() else { + panic!("expected provider and cache results"); + }; + assert_eq!(provider.source, ResultSource::Provider); + assert!(matches!(cached.source, ResultSource::Cache { .. })); + assert_eq!(cached.cost, Cost::CacheHit); +} diff --git a/litellm-rust/crates/inference/AGENTS.md b/litellm-rust/crates/inference/AGENTS.md index bcbc98a209e..bc9be4825ea 100644 --- a/litellm-rust/crates/inference/AGENTS.md +++ b/litellm-rust/crates/inference/AGENTS.md @@ -81,3 +81,4 @@ - Cache lookup follows provider preparation, credential resolution, and the request interceptor - keys describe the effective provider URL, authenticated headers, and rewritten body - signed requests bypass caching until the signing identity has a stable cache representation +- A service that keys entries by the caller's arguments (`keyed_by_caller`, the Python host's legacy cache) is looked up before provider preparation, as Python's `@client` wrapper does, and a hit skips request building and the request interceptor diff --git a/litellm-rust/crates/inference/src/caching.rs b/litellm-rust/crates/inference/src/caching.rs index d9b5b891015..b80fdd46714 100644 --- a/litellm-rust/crates/inference/src/caching.rs +++ b/litellm-rust/crates/inference/src/caching.rs @@ -238,12 +238,19 @@ impl CallCache

{ identity: &ProviderIdentity, wire: &WireRequest, ) -> Self { + Self::new( + cache, + policy, + CacheRequest::from_wire(identity.clone(), Some(wire)), + ) + } + + pub fn new(cache: Option<&ScopedCache>, policy: CachePolicy, request: CacheRequest) -> Self { let session = cache.and_then(|cache| { if !policy.enabled() { return None; } let options = cache.options(Some(policy)); - let request = CacheRequest::from_wire(identity.clone(), Some(wire)); Some(CacheSession { request: options.request( &cache.service.config().namespace, diff --git a/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md b/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md index 3b516251f43..9aba11a3376 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md +++ b/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md @@ -1,6 +1,6 @@ # Python cache delegation -This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` calls the Python cache's sync or async API and delivers the result back to Rust. +This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` delivers each operation's result back to Rust. Asynchronous lookups and stream stores go through `litellm.rust_bridge.host.cache`, which reads with the `LLMCachingHandler` the `@client` wrapper uses, for the legacy function the route names, and writes with its dual cache. `configure` disables the cache for synchronous calls. Keep the shared inference protocol wrapper and adapter selection in the parent module. Receive the configured cache and prepared arguments from the parent module. Do not discover global configuration, choose native backends, or move inference to Python. Core remains independent of Python objects and cache implementation details diff --git a/litellm-rust/crates/python-bridge/src/cache/python/host.rs b/litellm-rust/crates/python-bridge/src/cache/python/host.rs index a8cbd0ce607..10e16a54547 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/host.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/host.rs @@ -18,6 +18,7 @@ enum Pending { pub(crate) struct PythonCache { cache: Option>, + function: Option>, arguments: Option>, pending: Option, asynchronous: bool, @@ -28,6 +29,7 @@ impl PythonCache { pub fn new(asynchronous: bool) -> Self { Self { cache: None, + function: None, arguments: None, pending: None, asynchronous, @@ -35,12 +37,18 @@ impl PythonCache { } } + pub(in crate::cache) fn asynchronous(&self) -> bool { + self.asynchronous + } + pub(in crate::cache) fn bind( &mut self, cache: Bound<'_, PyAny>, + function: &Bound<'_, PyAny>, arguments: &Bound<'_, PyDict>, ) { self.cache = Some(cache.unbind()); + self.function = Some(function.clone().unbind()); self.arguments = Some(arguments.clone().unbind()); } @@ -59,7 +67,7 @@ impl PythonCache { self.pending = Some(Pending::Lookup(reply)); ( if self.asynchronous { - "async_get_cache" + "lookup" } else { "get_cache" }, @@ -77,7 +85,7 @@ impl PythonCache { self.pending = Some(Pending::Store(reply)); ( if self.asynchronous { - "async_add_cache" + "store_stream" } else { "add_cache" }, @@ -86,9 +94,20 @@ impl PythonCache { } }; let result = match result { + Some(value) if self.asynchronous => py + .import(CACHE_HOST_MODULE)? + .getattr(method)? + .call((cache, value), Some(&arguments)), Some(value) => cache .bind(py) .call_method(method, (value,), Some(&arguments)), + None if self.asynchronous => py.import(CACHE_HOST_MODULE)?.getattr(method)?.call( + ( + cache, + self.function.as_ref().ok_or(HandleError::Unconfigured)?, + ), + Some(&arguments), + ), None => cache.bind(py).call_method(method, (), Some(&arguments)), } .map(Bound::unbind); @@ -148,11 +167,13 @@ impl PythonCache { self.pending = None; self.deferred = false; self.cache = None; + self.function = None; self.arguments = None; } pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.cache)?; + visit.call(&self.function)?; visit.call(&self.arguments) } } diff --git a/litellm-rust/crates/python-bridge/src/cache/python/service.rs b/litellm-rust/crates/python-bridge/src/cache/python/service.rs index 018c8647043..618d990bc8e 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/service.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/service.rs @@ -79,6 +79,10 @@ where &self.config } + fn keyed_by_caller(&self) -> bool { + true + } + fn lookup<'a>( &'a self, _: &'a ResponseCacheRequest, diff --git a/litellm-rust/crates/python-bridge/src/cache/selection.rs b/litellm-rust/crates/python-bridge/src/cache/selection.rs index 78c1606ea40..a95b786f55a 100644 --- a/litellm-rust/crates/python-bridge/src/cache/selection.rs +++ b/litellm-rust/crates/python-bridge/src/cache/selection.rs @@ -51,15 +51,21 @@ fn selected_cache<'py>( kwargs: &Bound<'py, PyDict>, call_type: &str, ) -> PyResult>> { + if kwargs + .get_item("caching")? + .is_some_and(|value| value.is(pyo3::types::PyBool::new(py, false))) + { + return Ok(None); + } + configured_cache(py, call_type) +} + +fn configured_cache<'py>(py: Python<'py>, call_type: &str) -> PyResult>> { let configured = py .import(CACHE_HOST_MODULE)? .getattr("configured_cache")? .call0()?; - if configured.is_none() - || kwargs - .get_item("caching")? - .is_some_and(|value| value.is(pyo3::types::PyBool::new(py, false))) - { + if configured.is_none() { return Ok(None); } let supported = configured.getattr("supported_call_types")?; @@ -90,9 +96,14 @@ pub(crate) fn configure( python: &mut python::PythonCache, py: Python<'_>, arguments: &Bound<'_, PyDict>, - call_type: &str, + function: &Bound<'_, PyAny>, ) -> PyResult { - let selected = selected_cache(py, arguments, call_type)?; + let call_type: String = function.getattr("__name__")?.extract()?; + let selected = if python.asynchronous() { + configured_cache(py, &call_type)? + } else { + None + }; let Some(cache) = selected else { return Ok(Selection { backend: Backend::Disabled, @@ -102,6 +113,9 @@ pub(crate) fn configure( let enabled = cache .call_method("should_use_cache", (), Some(arguments))? .extract::()?; + let skips_read = arguments + .get_item("caching")? + .is_some_and(|value| !value.is_none() && !value.is(pyo3::types::PyBool::new(py, true))); let controls = arguments .get_item("cache")? .filter(|value| !value.is_none()); @@ -117,7 +131,7 @@ pub(crate) fn configure( }; let options = CacheOptions { policy: CachePolicy { - no_cache: boolean("no-cache")?, + no_cache: boolean("no-cache")? || skips_read, no_store: boolean("no-store")?, ..CachePolicy::default() }, @@ -129,7 +143,7 @@ pub(crate) fn configure( .map(|value| value.extract()) .transpose()? .unwrap_or_default(); - python.bind(cache, arguments); + python.bind(cache, function, arguments); Ok(Selection { backend: if enabled { Backend::Python { namespace } diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 57e9cfccee7..0e3a94cd4ef 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -252,9 +252,12 @@ impl PythonBinding for MessagesPythonHost { self.view .resolve(py, arguments) .map_err(InvokeError::Python)?; - let selection = - crate::cache::configure(&mut self.cache, py, arguments, "anthropic_messages") - .map_err(InvokeError::Python)?; + let function = py + .import(ROUTE_HOST_MODULE) + .and_then(|module| module.call_method0("cached_function")) + .map_err(InvokeError::Python)?; + let selection = crate::cache::configure(&mut self.cache, py, arguments, &function) + .map_err(InvokeError::Python)?; let settings = self .host_settings(py, arguments) .map_err(InvokeError::Python)?; diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 9fb3f542164..f87031513cd 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -19,6 +19,7 @@ import datetime import inspect import time from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError @@ -78,6 +79,19 @@ from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper EmbeddingCacheInputElement = str | list[int] | ChatCompletionFileObject +@dataclass(frozen=True, slots=True) +class CacheRead: + kwargs: dict[str, object] # mutable-ok: the @client call kwargs the embedding path keeps reading and rewriting + checked: bool = False + cached_result: object | None = None + model: str = "" + end_time: datetime.datetime | None = None + + @property + def hit(self) -> bool: + return self.end_time is not None + + class CachingHandlerResponse(LiteLLMBaseModel): """ This is the response object for the caching handler. We need to separate embedding cached responses and (completion / text_completion / transcription) cached responses @@ -114,14 +128,7 @@ def _drop_logging_obj_from_kwargs(request_kwargs: dict[str, object]) -> dict[str return {k: v for k, v in request_kwargs.items() if k != "litellm_logging_obj"} -def is_call_type_supported_by_cache(call_type: str) -> bool: - if litellm.cache is None or litellm.cache.supported_call_types is None: - return False - covering_call_types: Final = ("aresponses", "responses") if call_type == "aresponses" else (call_type,) - return any(name in litellm.cache.supported_call_types for name in covering_call_types) - - -def is_response_cache_excluded(model: str | None, kwargs: Mapping[str, object]) -> bool: +def _is_response_cache_excluded(model: str | None, kwargs: Mapping[str, object]) -> bool: from litellm.llms.github_copilot.per_user_auth import is_github_copilot_per_user_request custom_llm_provider: Final = kwargs.get("custom_llm_provider") @@ -402,121 +409,146 @@ class LLMCachingHandler: Raises: None """ - if is_response_cache_excluded(model=model, kwargs=kwargs): - self.request_kwargs = _drop_logging_obj_from_kwargs(kwargs) + if not self.reads_cache(model=model, kwargs=kwargs): return None + args = args or () + final_embedding_cached_response: EmbeddingResponse | None = None + embedding_all_elements_cache_hit: bool = False + read: Final = await self.async_read_cache( + model=model, + original_function=original_function, + logging_obj=logging_obj, + call_type=call_type, + kwargs=kwargs, + args=args, + ) + kwargs = read.kwargs + cached_result: object | None = read.cached_result + if litellm.cache is not None and read.checked: + if read.end_time is not None: + cached_result = self._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=original_function.__name__, + kwargs=kwargs, + logging_obj=logging_obj, + model=read.model, + custom_llm_provider=kwargs.get("custom_llm_provider", None), + args=args, + ) + if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result): + # LOG SUCCESS + self._async_log_cache_hit_on_callbacks( + logging_obj=logging_obj, + cached_result=cached_result, + start_time=start_time, + end_time=read.end_time, + cache_hit=True, + ) + _set_cached_hidden_param(cached_result, "cache_key", self.hit_cache_key()) + return CachingHandlerResponse(cached_result=cached_result) + elif ( + call_type == CallTypes.aembedding.value + and cached_result is not None + and isinstance(cached_result, list) + and litellm.cache is not None + and not isinstance(litellm.cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude. + ): + ( + final_embedding_cached_response, + embedding_all_elements_cache_hit, + ) = self._process_async_embedding_cached_response( + final_embedding_cached_response=final_embedding_cached_response, + cached_result=cached_result, + kwargs=kwargs, + logging_obj=logging_obj, + start_time=start_time, + model=model, + ) + return CachingHandlerResponse( + final_embedding_cached_response=final_embedding_cached_response, + embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, + embedding_uncached_input=self.handle_kwargs_input_list_or_str(kwargs), + ) - # Check if caching should be performed BEFORE doing expensive operations - if ( + verbose_logger.debug("CACHE RESULT: %s", cached_result) + return CachingHandlerResponse( + cached_result=cached_result, + final_embedding_cached_response=final_embedding_cached_response, + ) + + _async_get_cache = async_get_cache + + def reads_cache( + self, + model: str, + kwargs: dict[str, Any], # pyright: ignore[reportExplicitAny] # mutable-ok: the @client call kwargs, read as the wrapper reads them + ) -> bool: + if _is_response_cache_excluded(model=model, kwargs=kwargs): + self.request_kwargs = _drop_logging_obj_from_kwargs(kwargs) + return False + return ( (kwargs.get("caching", None) is None and litellm.cache is not None) or kwargs.get("caching", False) is True ) and ( kwargs.get("cache", {}).get("no-cache", False) is not True - ): # allow users to control returning cached responses from the completion function - args = args or () - final_embedding_cached_response: EmbeddingResponse | None = None - embedding_all_elements_cache_hit: bool = False - cached_result: object | None = None - kwargs = kwargs.copy() - ######################################################### - # Init cache timing metrics - ######################################################### - cache_check_start_time: Final = time.perf_counter() - cache_check_end_time: float | None = None - ######################################################### - parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - kwargs["parent_otel_span"] = parent_otel_span + ) # allow users to control returning cached responses from the completion function - if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function): - verbose_logger.debug("Checking Async Cache") - logging_obj.reset_caching_details() - cached_result = await self._retrieve_from_cache( - call_type=call_type, - kwargs=kwargs, - args=args, - ) - cache_check_end_time = time.perf_counter() + async def async_read_cache( + self, + model: str, + original_function: Callable[..., object], + logging_obj: LiteLLMLoggingObj, + call_type: str, + kwargs: Mapping[str, Any], # pyright: ignore[reportExplicitAny] # the @client call kwargs, read as the wrapper reads them + args: tuple[object, ...], + ) -> CacheRead: + cache_check_start_time: Final = time.perf_counter() + read_kwargs: Final = dict(kwargs) + read_kwargs["parent_otel_span"] = get_parent_otel_span_from_kwargs(read_kwargs) + if litellm.cache is None or not self._is_call_type_supported_by_cache(original_function=original_function): + return CacheRead(kwargs=read_kwargs) + verbose_logger.debug("Checking Async Cache") + logging_obj.reset_caching_details() + cached_result: Final = await self._retrieve_from_cache( + call_type=call_type, + kwargs=read_kwargs, + args=args, + ) + cache_check_end_time: Final = time.perf_counter() + if cached_result is None or isinstance(cached_result, list): + return CacheRead(kwargs=read_kwargs, checked=True, cached_result=cached_result) + verbose_logger.debug("Cache Hit!") + end_time: Final = datetime.datetime.now() + provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, + custom_llm_provider=read_kwargs.get("custom_llm_provider", None), + api_base=read_kwargs.get("api_base", None), + api_key=read_kwargs.get("api_key", None), + ) + self._update_litellm_logging_obj_environment( + logging_obj=logging_obj, + model=provider_model, + kwargs=read_kwargs, + cached_result=cached_result, + is_async=True, + custom_llm_provider=custom_llm_provider, + cache_duration_ms=(cache_check_end_time - cache_check_start_time) * 1000, + ) + return CacheRead( + kwargs=read_kwargs, + checked=True, + cached_result=cached_result, + model=provider_model, + end_time=end_time, + ) - if cached_result is not None and not isinstance(cached_result, list): - verbose_logger.debug("Cache Hit!") - cache_hit: Final = True - end_time: Final = datetime.datetime.now() - provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, - custom_llm_provider=kwargs.get("custom_llm_provider", None), - api_base=kwargs.get("api_base", None), - api_key=kwargs.get("api_key", None), - ) - cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000 - self._update_litellm_logging_obj_environment( - logging_obj=logging_obj, - model=provider_model, - kwargs=kwargs, - cached_result=cached_result, - is_async=True, - custom_llm_provider=custom_llm_provider, - cache_duration_ms=cache_duration_ms, - ) - - call_type = original_function.__name__ - - cached_result = self._convert_cached_result_to_model_response( - cached_result=cached_result, - call_type=call_type, - kwargs=kwargs, - logging_obj=logging_obj, - model=provider_model, - custom_llm_provider=kwargs.get("custom_llm_provider", None), - args=args, - ) - if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result): - # LOG SUCCESS - self._async_log_cache_hit_on_callbacks( - logging_obj=logging_obj, - cached_result=cached_result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - ) - cache_key: Final = ( - self.preset_cache_key - or self.request_kwargs.get("cache_key") - or litellm.cache.get_cache_key(**self.request_kwargs) - ) - _set_cached_hidden_param(cached_result, "cache_key", cache_key) - return CachingHandlerResponse(cached_result=cached_result) - elif ( - call_type == CallTypes.aembedding.value - and cached_result is not None - and isinstance(cached_result, list) - and litellm.cache is not None - and not isinstance(litellm.cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude. - ): - ( - final_embedding_cached_response, - embedding_all_elements_cache_hit, - ) = self._process_async_embedding_cached_response( - final_embedding_cached_response=final_embedding_cached_response, - cached_result=cached_result, - kwargs=kwargs, - logging_obj=logging_obj, - start_time=start_time, - model=model, - ) - return CachingHandlerResponse( - final_embedding_cached_response=final_embedding_cached_response, - embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, - embedding_uncached_input=self.handle_kwargs_input_list_or_str(kwargs), - ) - - verbose_logger.debug("CACHE RESULT: %s", cached_result) - return CachingHandlerResponse( - cached_result=cached_result, - final_embedding_cached_response=final_embedding_cached_response, - ) - # Caching disabled - return None to indicate no caching attempted - return None - - _async_get_cache = async_get_cache + def hit_cache_key(self) -> object: + if litellm.cache is None: + return None + return ( + self.preset_cache_key + or self.request_kwargs.get("cache_key") + or litellm.cache.get_cache_key(**self.request_kwargs) + ) def sync_get_cache( self, @@ -531,7 +563,7 @@ class LLMCachingHandler: cached_result: Any | None = None # Check if caching should be performed BEFORE doing expensive kwargs copy - if is_response_cache_excluded(model=model, kwargs=kwargs): + if _is_response_cache_excluded(model=model, kwargs=kwargs): self.request_kwargs = _drop_logging_obj_from_kwargs(kwargs) return CachingHandlerResponse(cached_result=None) if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function): @@ -889,14 +921,14 @@ class LLMCachingHandler: _combine_cached_embedding_response_with_api_result = combine_cached_embedding_response_with_api_result - def _async_log_cache_hit_on_callbacks( + def async_log_cache_hit_on_callbacks( self, logging_obj: LiteLLMLoggingObj, cached_result: object, start_time: datetime.datetime, end_time: datetime.datetime, cache_hit: bool, - ): + ) -> None: """ Helper function to log the success of a cached result on callbacks @@ -937,6 +969,8 @@ class LLMCachingHandler: cache_hit=cache_hit, ) + _async_log_cache_hit_on_callbacks = async_log_cache_hit_on_callbacks + async def _retrieve_from_cache( self, call_type: str, kwargs: dict[str, object], args: tuple[object, ...] ) -> object | None: @@ -972,7 +1006,7 @@ class LLMCachingHandler: model_value: Final = new_kwargs.get("model") model: Final = model_value if isinstance(model_value, str) else None self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs) - if is_response_cache_excluded(model=model, kwargs=new_kwargs): + if _is_response_cache_excluded(model=model, kwargs=new_kwargs): return None if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs: new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs) @@ -1330,7 +1364,7 @@ class LLMCachingHandler: ) no_store: Final = cache_options.get("no-store", False) return ( - not is_response_cache_excluded(model=model, kwargs=kwargs) + not _is_response_cache_excluded(model=model, kwargs=kwargs) and self._is_call_type_supported_by_cache(original_function=original_function) and no_store is not True ) @@ -1371,7 +1405,11 @@ class LLMCachingHandler: Returns: bool: True if the call type is supported by the cache, False otherwise. """ - return is_call_type_supported_by_cache(str(original_function.__name__)) + if litellm.cache is None or litellm.cache.supported_call_types is None: + return False + call_type: Final = str(original_function.__name__) + covering_call_types: Final = ("aresponses", "responses") if call_type == "aresponses" else (call_type,) + return any(name in litellm.cache.supported_call_types for name in covering_call_types) async def add_streaming_response_to_cache(self, processed_chunk: ModelResponse) -> None: """ diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index fa393850e76..9fac2b70078 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -139,7 +139,6 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType( { **{feature: detect for feature, detect in detectors.COMMON.items() if feature not in _NATIVE_COMMON_FEATURES}, Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier, - Feature.RESPONSE_CACHE: route_detectors.sync_response_cache, Feature.MODIFY_PARAMS: detectors.modify_params, Feature.WORKLOAD_IDENTITY: route_detectors.anthropic_workload_identity, Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, @@ -150,7 +149,6 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( { **_FEATURES, Feature.MOCK_RESPONSE: route_detectors.streamed_mock_response, - Feature.RESPONSE_CACHE: route_detectors.response_cache, Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook, Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 31277713ad5..b7cdfaadc9e 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -169,7 +169,6 @@ MESSAGES_PYTHON_FEATURES: Final = (COMMON_PYTHON_FEATURES - {Feature.TIMEOUT_VAR Feature.PRE_REQUEST_HOOK, Feature.WEBSEARCH_INTERCEPTION, Feature.MESSAGES_INTERCEPTOR, - Feature.RESPONSE_CACHE, Feature.MODIFY_PARAMS, Feature.WORKLOAD_IDENTITY, Feature.CACHE_CONTROL_INJECTION, diff --git a/litellm/rust_bridge/host/cache.py b/litellm/rust_bridge/host/cache.py index bf2655f9f62..14332e70b7b 100644 --- a/litellm/rust_bridge/host/cache.py +++ b/litellm/rust_bridge/host/cache.py @@ -1,7 +1,8 @@ from __future__ import annotations -from collections.abc import Mapping -from typing import TYPE_CHECKING, Final +import weakref +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # bounded cast of the served response from pydantic import TypeAdapter @@ -9,9 +10,12 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_par if TYPE_CHECKING: from litellm.caching.caching import Cache + from litellm.litellm_core_utils.litellm_logging import Logging _RUST_RESPONSE_HEADER: Final = "x-litellm-rust" _HEADERS: Final = TypeAdapter(dict[str, object]) +_EVENTS: Final = TypeAdapter(list[str]) +_HITS: Final[weakref.WeakKeyDictionary[Logging, Mapping[str, object]]] = weakref.WeakKeyDictionary() def configured_cache() -> Cache | None: @@ -25,17 +29,14 @@ def _python_entry(response: Mapping[str, object]) -> Mapping[str, object]: headers: Final = hidden.get("additional_headers") if not isinstance(headers, Mapping) or _RUST_RESPONSE_HEADER not in headers: return response - return { - **response, - "_hidden_params": { - **hidden, - "additional_headers": { - name: value - for name, value in _HEADERS.validate_python(headers).items() - if name != _RUST_RESPONSE_HEADER - }, - }, + kept: Final = { + name: value for name, value in _HEADERS.validate_python(headers).items() if name != _RUST_RESPONSE_HEADER } + stored: Final = {name: value for name, value in hidden.items() if name != "additional_headers" or kept} | ( + {"additional_headers": kept} if kept else {} + ) + entry: Final = {name: value for name, value in response.items() if name != "_hidden_params"} + return {**entry, "_hidden_params": stored} if stored else entry def store_response( @@ -52,3 +53,87 @@ def store_response( **kwargs, # pyright: ignore[reportArgumentType] # the cache call's own keyword arguments, as Python passes them ) ) + + +async def store_stream( + cache: Cache, + entry: Mapping[str, object], + /, + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> None: + from litellm.caching.caching_handler import LLMCachingHandler + from litellm.types.caching import CACHED_STREAM_EVENTS_KEY + + events: Final = entry.get(CACHED_STREAM_EVENTS_KEY) + handler: Final = getattr(kwargs.get("litellm_logging_obj"), "llm_caching_handler", None) + await cache.async_add_cache( # pyright: ignore[reportUnknownMemberType] # Cache.async_add_cache leaves its result untyped + {**entry, CACHED_STREAM_EVENTS_KEY: tuple(_EVENTS.validate_python(events))} + if isinstance(events, list) + else entry, + dynamic_cache_object=handler.dual_cache if isinstance(handler, LLMCachingHandler) else None, + **kwargs, # pyright: ignore[reportArgumentType] # the cache call's own keyword arguments, as Python passes them + ) + + +async def lookup( + _cache: Cache, + original_function: Callable[..., object], + /, + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> object | None: + from litellm.caching.caching_handler import LLMCachingHandler + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return None + handler: Final = LLMCachingHandler( + original_function=original_function, request_kwargs=dict(kwargs), start_time=logging_obj.start_time + ) + logging_obj.llm_caching_handler = handler + model: Final = kwargs.get("model") + model_name: Final = model if isinstance(model, str) else "" + if not handler.reads_cache(model=model_name, kwargs=kwargs): + return None + read: Final = await handler.async_read_cache( + model=model_name, + original_function=original_function, + logging_obj=logging_obj, + call_type=original_function.__name__, + kwargs=kwargs, + args=(), + ) + if not read.hit: + return None + _remember(logging_obj, read.cached_result) + return read.cached_result + + +def _remember(logging_obj: Logging, cached: object) -> None: + if isinstance(cached, Mapping): + _HITS[logging_obj] = _HEADERS.validate_python(cached) + + +def _replace(response: object, cached: Mapping[str, object] | None) -> None: + if isinstance(response, dict) and cached is not None: + served: Final = cast( # cast-ok: a Messages response is the str-keyed dict the route serves + "dict[str, object]", response + ) + served.clear() + served.update(cached) + + +def restore_hit(response: object, logger: Logging) -> None: + from litellm.caching.caching_handler import ( + _set_cached_hidden_param, # pyright: ignore[reportPrivateUsage] # the @client cache hit's own hidden params + ) + + _replace(response, _HITS.pop(logger, None)) + _set_cached_hidden_param(response, "cache_hit", True) + + +def hit_cache_key(logger: Logging) -> object: + from litellm.caching.caching_handler import LLMCachingHandler + + handler: Final = logger.llm_caching_handler + return handler.hit_cache_key() if isinstance(handler, LLMCachingHandler) else None diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index f2c3bb63ec0..a22117817e8 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -42,6 +42,7 @@ class MetadataUpdater(Protocol): kwargs: dict[str, object], start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool | None = None, ) -> None: ... @@ -98,6 +99,7 @@ def finalize( kwargs: dict[str, object], start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool, ) -> None: from litellm.litellm_core_utils.llm_response_utils import response_metadata @@ -105,6 +107,16 @@ def finalize( update: Final = cast( # cast-ok: legacy metadata function accepts concrete kwargs MetadataUpdater, response_metadata.update_response_metadata ) + if cache_hit: + from litellm.caching.caching_handler import ( + _set_cached_hidden_param, # pyright: ignore[reportPrivateUsage] # the @client cache hit's own hidden params + ) + from litellm.rust_bridge.host.cache import hit_cache_key, restore_hit + + restore_hit(response, logger) + update(response, logger, logger.model, kwargs, start_time, end_time, cache_hit=True) + _set_cached_hidden_param(response, "cache_key", hit_cache_key(logger)) + return update(response, logger, model if isinstance(model, str) else None, kwargs, start_time, end_time) cache_key: Final = logger.model_call_details.get("cache_key") if logger.model_call_details.get("cache_hit") is True and isinstance(cache_key, str): @@ -401,6 +413,16 @@ def async_success_handler( return logger.async_success_handler(response, start, end, cache_hit=_cache_hit(logger)) +def cache_hit_success(logger: Logging, response: object, start: datetime.datetime, end: datetime.datetime) -> None: + from litellm.caching.caching_handler import LLMCachingHandler + + handler: Final = logger.llm_caching_handler + if isinstance(handler, LLMCachingHandler): + handler.async_log_cache_hit_on_callbacks( + logging_obj=logger, cached_result=response, start_time=start, end_time=end, cache_hit=True + ) + + def enqueue_logging(coroutine: Coroutine[object, object, None]) -> None: from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -474,9 +496,10 @@ def stamp_stream_head( kwargs: dict[str, object], # mutable-ok: `finalize` takes the call kwargs as a dict start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool, ) -> None: carrier: Final = _HiddenParamsCarrier(head) - finalize(carrier, logger, kwargs, start_time, end_time) + finalize(carrier, logger, kwargs, start_time, end_time, cache_hit) stamped: Final = carrier._hidden_params # pyright: ignore[reportPrivateUsage] # the carrier's own field if stamped is not head and isinstance(stamped, dict): head.update(cast("dict[str, object]", stamped)) # cast-ok: update_response_metadata writes a str-keyed dict diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 0c9ff7f5b00..5385f8f301e 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -139,6 +139,13 @@ def _with_deployment_effort(params: Mapping[str, object], request: Mapping[str, ) +def cached_function() -> Callable[..., object]: + return cast( # cast-ok: the @client cache read names and checks the legacy handler it wraps + "Callable[..., object]", + handler.anthropic_messages, # noqa: TID251 # the @client cache read names and checks the legacy handler it wraps + ) + + def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: from litellm.llms.anthropic.messages.native.streaming_iterator import ( anthropic_messages_stream_hidden_params, diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 346ad8721cd..0a14ea08af0 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -10,33 +10,6 @@ from litellm.rust_bridge.detectors import overridden, present from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str -def _response_cache_consulted(request: NativeCall, call_type: str, *, honours_cache_controls: bool) -> bool: - from litellm.caching.caching_handler import is_call_type_supported_by_cache, is_response_cache_excluded - - cache: Final = litellm.cache - kwargs: Final = request.resolved - if ( - cache is None - or is_response_cache_excluded(optional_str(kwargs.get("model")), kwargs) - or not is_call_type_supported_by_cache(call_type) - ): - return False - if honours_cache_controls and ( - kwargs.get("caching") not in (None, True) - or (optional_mapping(kwargs.get("cache")) or {}).get("no-cache") is True - ): - return False - return cache.should_use_cache(**kwargs) is True # pyright: ignore[reportUnknownMemberType] # Cache.should_use_cache takes untyped kwargs - - -def response_cache(request: NativeCall) -> bool: - return _response_cache_consulted(request, "anthropic_messages", honours_cache_controls=True) - - -def sync_response_cache(request: NativeCall) -> bool: - return _response_cache_consulted(request, "anthropic_messages_handler", honours_cache_controls=False) - - def streamed_mock_response(request: NativeCall) -> bool: return present(request, "mock_response") and request.resolved.get("stream") is True diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py index 11622233551..af53e2069af 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py @@ -35,6 +35,9 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.caching.caching import Cache, CacheMode +from litellm.caching.caching_handler import ( + _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # a primed entry must land before the measured call reads it +) from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.messages import dispatch from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -322,6 +325,7 @@ class Case: env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) settings: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) private_keys: bool = False + cache: Callable[[], Cache] | None = None gaps: Mapping[str, str | None] = field(default_factory=lambda: MappingProxyType({})) def params(self) -> Iterator[object]: @@ -862,6 +866,36 @@ THINKING_RETRY_CASES: Final = ( ), ) + +def _default_off_cache() -> Cache: + return Cache(mode=CacheMode.default_off) + + +def _completion_only_cache() -> Cache: + return Cache(supported_call_types=["acompletion"]) + + +CACHE_CASES: Final = ( + Case("cache_hit", upstream=(_ok(), _ok()), cache=Cache), + Case("cache_hit_stream", _kw(stream=True), upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), cache=Cache), + Case("cache_no_cache", _kw(cache={"no-cache": True}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_no_store", _kw(cache={"no-store": True}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_caching_false", _kw(caching=False), upstream=(_ok(), _ok()), cache=Cache), + Case( + "cache_ttl_and_namespace", _kw(cache={"ttl": 60, "namespace": "parity"}), upstream=(_ok(), _ok()), cache=Cache + ), + Case("cache_default_off", upstream=(_ok(), _ok()), cache=_default_off_cache), + Case( + "cache_default_off_use_cache", + _kw(cache={"use-cache": True}), + upstream=(_ok(), _ok()), + cache=_default_off_cache, + ), + Case("cache_unsupported_call_type", upstream=(_ok(), _ok()), cache=_completion_only_cache), + Case("cache_hit_with_metadata", _kw(metadata={"user_id": "user-123"}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_sync", sync=True, upstream=(_ok(), _ok()), cache=Cache), +) + CASES: Final = ( *REQUEST_CASES, *RESPONSE_CASES, @@ -871,6 +905,7 @@ CASES: Final = ( *SERVED_NATIVELY_CASES, *THINKING_RETRY_CASES, *FALLBACK_CASES, + *CACHE_CASES, ) @@ -945,6 +980,15 @@ def _settings(settings: Mapping[str, object]) -> Generator[None]: setattr(litellm, name, value) +@contextmanager +def _cache(cache: Callable[[], Cache] | None) -> Generator[None]: + if cache is None: + yield + return + with rebound(litellm, "cache", cache()): + yield + + def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]: # Deep: Python writes into the caller's own `litellm_metadata`, which must not leak into the # other leg. @@ -1062,21 +1106,24 @@ async def _invoke( }, hidden -async def _terminal_events(recorder: RecordingLogger) -> tuple[str, ...]: +async def _terminal_events(recorder: RecordingLogger, since: int = 0) -> tuple[str, ...]: for _ in range(100): await drain_logging() - if any(name in _TERMINAL_EVENTS for name in recorder.names): + if any(name in _TERMINAL_EVENTS for name in recorder.names[since:]): break await asyncio.sleep(0.05) await asyncio.sleep(0.2) await drain_logging() - return tuple(name for name in recorder.names if name in _TERMINAL_EVENTS) + return tuple(name for name in recorder.names[since:] if name in _TERMINAL_EVENTS) -def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping[str, object], Mapping[str, object]]: - terminal: Final = [event for event in recorder.events if event.name in _TERMINAL_EVENTS] +def _logged( + recorder: RecordingLogger, events: tuple[str, ...], since: int = 0 +) -> tuple[Mapping[str, object], Mapping[str, object]]: + terminal: Final = [event for event in recorder.events[since:] if event.name in _TERMINAL_EVENTS] + hooks: Final = tuple(sorted(set(recorder.names[since:]))) if not terminal: - return {"events": events}, {} + return {"events": events, "hooks": hooks}, {} details: Final = _mapping(terminal[-1].kwargs) standard: Final = _mapping(details.get("standard_logging_object")) error_class: Final = _mapping(standard.get("error_information")).get("error_class") @@ -1085,6 +1132,7 @@ def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping metadata_keys: Final = _mapping(metadata).keys() logged: Final[Mapping[str, object]] = { "events": events, + "hooks": hooks, **{name: standard.get(name) for name in _COMPARED_LOG_FIELDS}, "error_class": error_class, } @@ -1107,18 +1155,29 @@ def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping return logged, callback_kwargs +async def _prime(case: Case, server: RecordingServer, recorder: RecordingLogger, native: bool) -> int: + if case.cache is None: + return 0 + outcome: Final = await _invoke(case, _kwargs(case, server), native=native) + assert case.sync or outcome[0]["kind"] != "error", outcome + await _terminal_events(recorder) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + return len(recorder.events) + + async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) -> Leg: - with isolated_callback_registries(), _environment(case.env), _settings(case.settings): - recorder: Final = RecordingLogger() - litellm.callbacks.append(recorder) + with isolated_callback_registries(), _environment(case.env), _settings(case.settings), _cache(case.cache): for spec in case.upstream: server.enqueue(replace(spec)) request_count: Final = len(server.requests) + recorder: Final = RecordingLogger() + litellm.callbacks.append(recorder) fallbacks_before: Final = 0 if probe is None else probe.calls + primed: Final = await _prime(case, server, recorder, native=probe is not None) outcome, hidden = await _invoke(case, _kwargs(case, server), native=probe is not None) - events: Final = await _terminal_events(recorder) + events: Final = await _terminal_events(recorder, primed) server.responses.clear() - logged, callback_kwargs = _logged(recorder, events) + logged, callback_kwargs = _logged(recorder, events, primed) return Leg( fell_back=probe is not None and probe.calls > fallbacks_before, requests=tuple( diff --git a/tests/test_litellm_rust/cache/test_python_cache.py b/tests/test_litellm_rust/cache/test_python_cache.py index d88feec7d5e..632a7d788c2 100644 --- a/tests/test_litellm_rust/cache/test_python_cache.py +++ b/tests/test_litellm_rust/cache/test_python_cache.py @@ -6,6 +6,7 @@ from pydantic import TypeAdapter import litellm from litellm.caching.caching import Cache, CacheMode +from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.caching_handler import ( _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request ) @@ -33,6 +34,14 @@ from tests.test_litellm_rust.support.routing import native_route pytestmark = pytest.mark.requires_rust_extension +_STORED: Final = TypeAdapter(dict[str, object]) + + +def _entries(storage: InMemoryCache) -> dict[str, object]: + return _STORED.validate_python( + storage.cache_dict # pyright: ignore[reportUnknownMemberType] # InMemoryCache leaves its dict untyped + ) + @pytest.mark.asyncio @pytest.mark.parametrize( @@ -110,7 +119,9 @@ async def test_legacy_cache_controls_and_backend_credential_key_semantics(record recording_server.expected_requests = 3 litellm.cache = Cache() await invoke("messages", recording_server, {"cache": {"no-store": True}}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) await invoke("messages", recording_server, {}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) await invoke("messages", recording_server, {}) assert len(recording_server.requests) == 2 await invoke("messages", recording_server, {"cache": {"no-cache": True}}) @@ -174,13 +185,18 @@ async def test_rust_messages_uses_a_legacy_cache_without_python_inference( "api_base": recording_server.base_url, } first: Final = await invoke("messages", recording_server, parameters) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) second: Final = await invoke("messages", recording_server, parameters) - assert cache_key(second) - assert cache_key(first) is None assert payload(first) == payload(second) assert len(recording_server.requests) == 1 + assert cache_key(first) is None + assert cache_key(second) is None - await logger.wait_for_async("async_log_success_event", count=2) + successes: Final = await logger.wait_for_async("async_log_success_event", count=2) + assert [TypeAdapter(dict[str, object]).validate_python(event.kwargs).get("cache_hit") for event in successes] == [ + None, + True, + ] assert logger.names.count("async_log_success_event") == 2 assert "log_failure_event" not in logger.names assert "async_log_failure_event" not in logger.names @@ -210,7 +226,9 @@ async def test_legacy_cache_default_off_requires_opt_in_even_for_existing_entrie @pytest.mark.asyncio -async def test_legacy_cache_lookup_uses_backend_request_callback_semantics(recording_server: RecordingServer) -> None: +async def test_a_legacy_cache_hit_skips_the_pre_call_callback_that_would_rewrite_the_request( + recording_server: RecordingServer, +) -> None: class Rewrite(RecordingLogger): temperature = 0.1 @@ -222,15 +240,16 @@ async def test_legacy_cache_lookup_uses_backend_request_callback_semantics(recor litellm.cache = Cache() recording_server.expected_requests = 1 await invoke("messages", recording_server, {"callbacks": [logger]}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) first_hit: Final = await invoke("messages", recording_server, {"callbacks": [logger]}) logger.temperature = 0.8 await invoke("messages", recording_server, {"callbacks": [logger]}) second_hit: Final = await invoke("messages", recording_server, {"callbacks": [logger]}) - assert logger.names.count("log_pre_api_call") == 4 + assert logger.names.count("log_pre_api_call") == 1 + assert cache_key(first_hit) is None + assert cache_key(second_hit) is None assert len(recording_server.requests) == recording_server.expected_requests assert recording_server.requests[0].body["temperature"] == 0.1 - assert isinstance(cache_key(first_hit), str) - assert isinstance(cache_key(second_hit), str) @pytest.mark.asyncio @@ -314,7 +333,11 @@ async def test_python_cache_operations_stay_in_the_rust_callers_task_except_the_ second: Final = await invoke("messages", recording_server, {"stream": stream}) second_payload: Final = await collect(second) if stream else payload(second) assert first_payload == second_payload - assert cache_key(second) + assert cache_key(first) is None + if stream: + assert cache_key(second) in _entries(storage) + else: + assert cache_key(second) is None assert len(recording_server.requests) == 1 @@ -389,6 +412,8 @@ async def test_messages_cache_stores_the_response_a_yielding_deployment_hook_ret async def test_rust_messages_legacy_cache_honors_request_namespaces( recording_server: RecordingServer, namespace_source: str ) -> None: + from litellm.caching.in_memory_cache import InMemoryCache + litellm.cache = Cache() recording_server.expected_requests = 2 first_options: Final = ( @@ -399,13 +424,20 @@ async def test_rust_messages_legacy_cache_honors_request_namespaces( if namespace_source == "cache" else {"metadata": {"redis_namespace": "second"}} ) + storage: Final = litellm.cache.cache + assert isinstance(storage, InMemoryCache) await invoke("messages", recording_server, first_options) - second: Final = await invoke("messages", recording_server, second_options) - first_hit: Final = await invoke("messages", recording_server, first_options) - second_hit: Final = await invoke("messages", recording_server, second_options) - assert cache_key(second) is None - assert cache_key(first_hit) is not None - assert cache_key(second_hit) is not None + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + first_keys: Final = set(_entries(storage)) + await invoke("messages", recording_server, second_options) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + second_keys: Final = set(_entries(storage)) - first_keys + assert len(recording_server.requests) == 2 + assert [key.split(":", 1)[0] for key in first_keys] == ["first"] + assert [key.split(":", 1)[0] for key in second_keys] == ["second"] + assert {key.split(":", 1)[1] for key in first_keys} == {key.split(":", 1)[1] for key in second_keys} + await invoke("messages", recording_server, first_options) + await invoke("messages", recording_server, second_options) assert len(recording_server.requests) == 2 @@ -420,9 +452,8 @@ async def test_rust_messages_legacy_semantic_cache_preserves_python_scope( first_options: Final = {"messages": [{"role": "user", "content": "hello"}]} second_options: Final = {"messages": [{"role": "user", "content": "hi"}]} first: Final = await invoke("messages", recording_server, first_options) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) second: Final = await invoke("messages", recording_server, second_options) - assert cache_key(first) is None - assert cache_key(second) is not None assert payload(first) == payload(second) assert len(recording_server.requests) == 1 @@ -443,3 +474,62 @@ async def test_legacy_cache_keeps_public_messages_responses_compatible( second_payload: Final = await collect(second) if stream else payload(second) assert second_payload == first_payload assert len(recording_server.requests) == 1 + + +_PER_CALL_HIDDEN_PARAMS: Final = frozenset( + {"_response_ms", "litellm_overhead_time_ms", "callback_duration_ms", "litellm_call_id"} +) + + +async def _write_then_read( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + writer: bool, + reader: bool, + stream: bool, +) -> tuple[object, object, object, int]: + from litellm.caching.in_memory_cache import InMemoryCache + + storage: Final = InMemoryCache() + monkeypatch.setattr(litellm, "cache", Cache(_backend=storage)) + requests_before: Final = len(recording_server.requests) + monkeypatch.setenv("LITELLM_RUST", "1" if writer else "0") + first: Final = await invoke("messages", recording_server, {"stream": stream}, native=writer) + if stream: + await collect(first) + else: + add_retry_headers_to_response( + _add_headers_to_response(first, {"x-litellm-model-group": "group"}), attempted_retries=0 + ) + get_hidden_params_dict(first)["model_id"] = "deployment" + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + (entry,) = _entries(storage).values() + stored: Final = TypeAdapter(dict[str, object]).validate_python(entry)["response"] + monkeypatch.setenv("LITELLM_RUST", "1" if reader else "0") + hit: Final = await invoke("messages", recording_server, {"stream": stream}, native=reader) + hidden: Final = { + name: value for name, value in get_hidden_params_dict(hit).items() if name not in _PER_CALL_HIDDEN_PARAMS + } + headers: Final = TypeAdapter(dict[str, object]).validate_python(hidden.pop("additional_headers", {})) + kept: Final = { + name: value for name, value in headers.items() if name not in {"x-litellm-rust", "x-litellm-cache-key"} + } + served: Final = await collect(hit) if stream else payload(hit) + return stored, served, {**hidden, "additional_headers": kept}, len(recording_server.requests) - requests_before + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", (False, True), ids=("response", "stream")) +@pytest.mark.parametrize(("writer", "reader"), ((True, False), (False, True)), ids=("rust_to_python", "python_to_rust")) +async def test_a_cache_entry_written_by_one_leg_is_stored_and_served_like_the_readers_own( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + writer: bool, + reader: bool, + stream: bool, +) -> None: + recording_server.expected_requests = 2 + crossed: Final = await _write_then_read(recording_server, monkeypatch, writer, reader, stream) + own: Final = await _write_then_read(recording_server, monkeypatch, reader, reader, stream) + assert crossed == own + assert crossed[3] == 1 diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 6ce91df957e..ac803294ab7 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -18,7 +18,7 @@ from litellm.messages.dispatch import ( ) from litellm.proxy import proxy_server from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import RULES, Feature, Route, RouteRule, Rules, Target +from litellm.rust_bridge.catalog import MESSAGES_PYTHON_FEATURES, RULES, Feature, Route, RouteRule, Rules, Target from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES, NativeAmessages, NativeMessages from litellm.rust_bridge.public_call import NativeCall @@ -464,10 +464,6 @@ ADVISOR_TOOL: Final = {"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor", " STRICT_TOOL: Final = {"name": "a", "input_schema": {"type": "object"}, "strict": True} -def _handler_cache() -> Cache: - return Cache(supported_call_types=["anthropic_messages_handler"]) # pyright: ignore[reportArgumentType] # the sync handler name the cache gate keys on - - @pytest.mark.parametrize( ("feature", "model", "kwargs", "settings"), ( @@ -480,12 +476,6 @@ def _handler_cache() -> Cache: ), (Feature.CALLER_CLIENT, "anthropic/claude-sonnet-4-5", {"client": object()}, {}), (Feature.AGENTIC_LOOP_HOOK, "anthropic/claude-sonnet-4-5", {}, {"callbacks": [_Agentic()]}), - ( - Feature.RESPONSE_CACHE, - "anthropic/claude-sonnet-4-5", - {}, - {"cache": _handler_cache()}, - ), (Feature.TLS_POLICY, "anthropic/claude-sonnet-4-5", {}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}), (Feature.MODIFY_PARAMS, "anthropic/claude-sonnet-4-5", {}, {"modify_params": True}), ( @@ -648,6 +638,45 @@ async def test_strict_tools_stay_native_unless_python_could_drop_strict( assert served is native_result +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kwargs", + ( + pytest.param({}, id="reads-and-writes"), + pytest.param({"cache": {"no-store": True}}, id="no-store"), + pytest.param({"cache": {"ttl": 60, "namespace": "team"}}, id="ttl-and-namespace"), + pytest.param({"stream": True}, id="stream"), + ), +) +async def test_a_configured_response_cache_stays_native( + monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object] +) -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=MESSAGES_PYTHON_FEATURES), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + native_result: Final = response() + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "cache", Cache()) + + async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + pytest.fail("Rust reads and writes the configured response cache") + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + served: Final = await _ADISPATCH.arun( + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + kwargs, + python=python, + binding=amessages_binding(native), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=rules, + ) + + assert served is native_result + + ASYNC_ONLY_FEATURES: Final = ( pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"), pytest.param( diff --git a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py index 78648842271..6b69b774adb 100644 --- a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py @@ -73,7 +73,7 @@ def test_cache_hit_finalization_preserves_execution_provider_attribution(explici ) prepared.logger.model_call_details.update({"cache_hit": True, "cache_key": "cached-response"}) response: Final = ModelResponse(model="cache-test-model") - legacy.finalize(response, prepared.logger, prepared.kwargs, now, now) + legacy.finalize(response, prepared.logger, prepared.kwargs, now, now, False) assert prepared.logger.model_call_details["custom_llm_provider"] == "azure" assert prepared.logger.model_call_details["model"] == "resolved-cache-model" assert prepared.logger.litellm_params["metadata"]["user_api_key"] == "key-hash" @@ -404,6 +404,7 @@ def test_stream_head_is_stamped_in_place_with_the_call_hidden_params() -> None: {"model": "anthropic/claude-sonnet-4-5", "litellm_call_id": "messages"}, started, started + datetime.timedelta(milliseconds=5), + False, ) assert head["additional_headers"] == {"request-id": "req_native", "llm_provider-request-id": "req_native"} diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 000141f5714..1b7bc1c848e 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -9,7 +9,7 @@ import httpx import pytest import litellm -from litellm.caching.caching import Cache, CacheMode +from litellm.caching.caching import Cache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.responses.utils import ResponsesAPIRequestUtils @@ -505,53 +505,6 @@ def test_response_cache_is_skipped_only_by_no_cache( assert detectors.response_cache(call(**kwargs)) is expected -@pytest.mark.parametrize( - ("cache", "kwargs", "expected"), - ( - pytest.param(None, {}, False, id="no-cache-configured"), - pytest.param(Cache(), {}, True, id="default-cache"), - pytest.param(Cache(), {"cache": {"no-cache": True}}, False, id="no-cache-control"), - pytest.param(Cache(mode=CacheMode.default_off), {}, False, id="default-off"), - pytest.param( - Cache(mode=CacheMode.default_off), {"cache": {"use-cache": True}}, True, id="default-off-opted-in" - ), - pytest.param(Cache(supported_call_types=["acompletion"]), {}, False, id="messages-not-cached"), - pytest.param(Cache(supported_call_types=["anthropic_messages"]), {}, True, id="messages-cached"), - pytest.param(Cache(), {"caching": False}, False, id="caching-off"), - pytest.param(Cache(), {"model": "databricks_agent/agent"}, False, id="excluded-provider"), - ), -) -def test_messages_response_cache_fires_only_when_python_touches_the_cache( - monkeypatch: pytest.MonkeyPatch, cache: Cache | None, kwargs: Mapping[str, object], expected: bool -) -> None: - monkeypatch.setattr(litellm, "cache", cache) - assert messages_detectors.response_cache(call(**kwargs)) is expected - - -def _handler_cache() -> Cache: - return Cache(supported_call_types=["anthropic_messages_handler"]) # pyright: ignore[reportArgumentType] # the sync handler name the cache gate keys on - - -@pytest.mark.parametrize( - ("cache", "kwargs", "expected"), - ( - pytest.param(_handler_cache(), {}, True, id="handler-cached"), - pytest.param(Cache(supported_call_types=["anthropic_messages"]), {}, False, id="only-async-cached"), - pytest.param( - _handler_cache(), - {"cache": {"no-cache": True}}, - True, - id="sync-lookup-ignores-no-cache", - ), - ), -) -def test_sync_messages_response_cache_keys_on_the_handler_name( - monkeypatch: pytest.MonkeyPatch, cache: Cache, kwargs: Mapping[str, object], expected: bool -) -> None: - monkeypatch.setattr(litellm, "cache", cache) - assert messages_detectors.sync_response_cache(call(**kwargs)) is expected - - @pytest.mark.parametrize( ("kwargs", "expected"), ( diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index 6ceea94d843..44b5876a078 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -1223,6 +1223,13 @@ ALIAS_CASES: Final = ( False, ), ("litellm.caching.caching_handler", "LLMCachingHandler", "_async_get_cache", "async_get_cache", False), + ( + "litellm.caching.caching_handler", + "LLMCachingHandler", + "_async_log_cache_hit_on_callbacks", + "async_log_cache_hit_on_callbacks", + False, + ), ( "litellm.caching.caching_handler", "LLMCachingHandler", @@ -1957,7 +1964,13 @@ ALIAS_CASES: Final = ( "get_tags_from_request_kwargs", False, ), - ("litellm.litellm_core_utils.add_retry_fallback_headers", "HiddenParamsAsyncIteratorWrapper", "_inner", "inner", True), + ( + "litellm.litellm_core_utils.add_retry_fallback_headers", + "HiddenParamsAsyncIteratorWrapper", + "_inner", + "inner", + True, + ), ("litellm.litellm_core_utils.add_retry_fallback_headers", "", "_HiddenParamsHost", "HiddenParamsHost", False), ( "litellm.litellm_core_utils.batch_utils", @@ -2301,1837 +2314,1837 @@ LLMS_ALIAS_CASES: Final = ( ) PROXY_ALIAS_CASES: Final = ( ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - '', - '_is_mcp_admitted_user_subject', - 'is_mcp_admitted_user_subject', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "", + "_is_mcp_admitted_user_subject", + "is_mcp_admitted_user_subject", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_auth_header_from_headers', - 'get_mcp_auth_header_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_auth_header_from_headers", + "get_mcp_auth_header_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_server_auth_headers_from_headers', - 'get_mcp_server_auth_headers_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_server_auth_headers_from_headers", + "get_mcp_server_auth_headers_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_servers_from_access_groups', - 'get_mcp_servers_from_access_groups', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_servers_from_access_groups", + "get_mcp_servers_from_access_groups", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_oauth2_headers_from_headers', - 'get_oauth2_headers_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_oauth2_headers_from_headers", + "get_oauth2_headers_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_safe_get_headers_from_scope', - 'safe_get_headers_from_scope', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_safe_get_headers_from_scope", + "safe_get_headers_from_scope", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_bridge_mint_error_response', - 'bridge_mint_error_response', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_bridge_mint_error_response", + "bridge_mint_error_response", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_extract_user_id_from_request', - 'extract_user_id_from_request', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_extract_user_id_from_request", + "extract_user_id_from_request", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_finish_bridge_mint', - 'finish_bridge_mint', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_finish_bridge_mint", + "finish_bridge_mint", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_prepare_bridge_mint', - 'prepare_bridge_mint', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_prepare_bridge_mint", + "prepare_bridge_mint", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_prepare_bridge_refresh', - 'prepare_bridge_refresh', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_prepare_bridge_refresh", + "prepare_bridge_refresh", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_reload_active_user_by_id', - 'reload_active_user_by_id', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_reload_active_user_by_id", + "reload_active_user_by_id", False, ), ( - 'litellm.proxy.mcp_server.byok_oauth_endpoints', - '', - '_user_id_from_session_cookie', - 'user_id_from_session_cookie', + "litellm.proxy.mcp_server.byok_oauth_endpoints", + "", + "_user_id_from_session_cookie", + "user_id_from_session_cookie", False, ), ( - 'litellm.proxy.mcp_server.db', - '', - '_decode_oauth_payload', - 'decode_oauth_payload', + "litellm.proxy.mcp_server.db", + "", + "_decode_oauth_payload", + "decode_oauth_payload", False, ), ( - 'litellm.proxy.mcp_server.discoverable_endpoints', - '', - '_raise_if_not_oauth2', - 'raise_if_not_oauth2', + "litellm.proxy.mcp_server.discoverable_endpoints", + "", + "_raise_if_not_oauth2", + "raise_if_not_oauth2", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_active_toolset_id', - 'mcp_active_toolset_id', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_active_toolset_id", + "mcp_active_toolset_id", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_gateway_initialize_instructions', - 'mcp_gateway_initialize_instructions', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_gateway_initialize_instructions", + "mcp_gateway_initialize_instructions", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_gateway_server_name', - 'mcp_gateway_server_name', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_gateway_server_name", + "mcp_gateway_server_name", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES', - 'UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES", + "UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_caller_authorization_fans_out', - 'caller_authorization_fans_out', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_caller_authorization_fans_out", + "caller_authorization_fans_out", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_client_forwarded_authorization_headers', - 'client_forwarded_authorization_headers', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_client_forwarded_authorization_headers", + "client_forwarded_authorization_headers", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_resolve_openapi_tool_auth', - 'resolve_openapi_tool_auth', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_resolve_openapi_tool_auth", + "resolve_openapi_tool_auth", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_should_strip_caller_authorization', - 'should_strip_caller_authorization', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_should_strip_caller_authorization", + "should_strip_caller_authorization", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_build_mcp_server_table', - 'build_mcp_server_table', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_build_mcp_server_table", + "build_mcp_server_table", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_build_stdio_env', - 'build_stdio_env', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_build_stdio_env", + "build_stdio_env", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_create_mcp_client', - 'create_mcp_client', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_create_mcp_client", + "create_mcp_client", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_ensure_upstream_initialize_instructions_cached', - 'ensure_upstream_initialize_instructions_cached', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_ensure_upstream_initialize_instructions_cached", + "ensure_upstream_initialize_instructions_cached", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_extract_subject_token', - 'extract_subject_token', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_extract_subject_token", + "extract_subject_token", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_get_mcp_server_from_tool_name', - 'get_mcp_server_from_tool_name', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_get_mcp_server_from_tool_name", + "get_mcp_server_from_tool_name", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_get_tools_from_server', - 'get_tools_from_server', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_get_tools_from_server", + "get_tools_from_server", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_is_server_accessible_from_ip', - 'is_server_accessible_from_ip', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_is_server_accessible_from_ip", + "is_server_accessible_from_ip", False, ), ( - 'litellm.proxy.mcp_server.oauth2_token_cache', - '', - '_compute_per_user_token_ttl', - 'compute_per_user_token_ttl', + "litellm.proxy.mcp_server.oauth2_token_cache", + "", + "_compute_per_user_token_ttl", + "compute_per_user_token_ttl", False, ), ( - 'litellm.proxy.mcp_server.oauth_utils', - '', - '_redact_mcp_resource_url', - 'redact_mcp_resource_url', + "litellm.proxy.mcp_server.oauth_utils", + "", + "_redact_mcp_resource_url", + "redact_mcp_resource_url", False, ), ( - 'litellm.proxy.mcp_server.server', - '', - '_redact_mcp_resource_url', - 'redact_mcp_resource_url', + "litellm.proxy.mcp_server.server", + "", + "_redact_mcp_resource_url", + "redact_mcp_resource_url", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_OPENAPI_TOOL_NAME_MAX_LEN', - 'OPENAPI_TOOL_NAME_MAX_LEN', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_OPENAPI_TOOL_NAME_MAX_LEN", + "OPENAPI_TOOL_NAME_MAX_LEN", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_auth_header', - 'request_auth_header', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_auth_header", + "request_auth_header", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_extra_headers', - 'request_extra_headers', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_extra_headers", + "request_extra_headers", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_resolved_auth_headers', - 'request_resolved_auth_headers', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_resolved_auth_headers", + "request_resolved_auth_headers", False, ), ( - 'litellm.proxy.mcp_server.semantic_tool_filter', - 'SemanticMCPToolFilter', - '_extract_tool_info', - 'extract_tool_info', + "litellm.proxy.mcp_server.semantic_tool_filter", + "SemanticMCPToolFilter", + "_extract_tool_info", + "extract_tool_info", False, ), ( - 'litellm.proxy.mcp_server.server', - '', - '_apply_toolset_scope', - 'apply_toolset_scope', + "litellm.proxy.mcp_server.server", + "", + "_apply_toolset_scope", + "apply_toolset_scope", False, ), ( - 'litellm.proxy.mcp_server.server_resolution', - 'MCPServerRegistry', - '_build_mcp_server_table', - 'build_mcp_server_table', + "litellm.proxy.mcp_server.server_resolution", + "MCPServerRegistry", + "_build_mcp_server_table", + "build_mcp_server_table", False, ), ( - 'litellm.proxy.mcp_server.server_resolution', - 'MCPServerRegistry', - '_is_server_accessible_from_ip', - 'is_server_accessible_from_ip', + "litellm.proxy.mcp_server.server_resolution", + "MCPServerRegistry", + "_is_server_accessible_from_ip", + "is_server_accessible_from_ip", False, ), ( - 'litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace', - '', - '_get_prisma_client', - 'get_prisma_client', + "litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace", + "", + "_get_prisma_client", + "get_prisma_client", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_access_object', - 'cache_access_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_access_object", + "cache_access_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_key_object', - 'cache_key_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_key_object", + "cache_key_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_team_object', - 'cache_team_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_team_object", + "cache_team_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_can_object_call_model', - 'can_object_call_model', + "litellm.proxy.auth.auth_checks", + "", + "_can_object_call_model", + "can_object_call_model", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_end_user_budget', - 'check_end_user_budget', + "litellm.proxy.auth.auth_checks", + "", + "_check_end_user_budget", + "check_end_user_budget", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_model_access_helper', - 'check_model_access_helper', + "litellm.proxy.auth.auth_checks", + "", + "_check_model_access_helper", + "check_model_access_helper", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_team_member_model_access', - 'check_team_member_model_access', + "litellm.proxy.auth.auth_checks", + "", + "_check_team_member_model_access", + "check_team_member_model_access", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_copy_user_api_key_auth_for_cache', - 'copy_user_api_key_auth_for_cache', + "litellm.proxy.auth.auth_checks", + "", + "_copy_user_api_key_auth_for_cache", + "copy_user_api_key_auth_for_cache", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_delete_cache_access_object', - 'delete_cache_access_object', + "litellm.proxy.auth.auth_checks", + "", + "_delete_cache_access_object", + "delete_cache_access_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_delete_cache_key_object', - 'delete_cache_key_object', + "litellm.proxy.auth.auth_checks", + "", + "_delete_cache_key_object", + "delete_cache_key_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_fetch_key_object_from_db_with_reconnect', - 'fetch_key_object_from_db_with_reconnect', + "litellm.proxy.auth.auth_checks", + "", + "_fetch_key_object_from_db_with_reconnect", + "fetch_key_object_from_db_with_reconnect", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_agent_ids_from_access_groups', - 'get_agent_ids_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_agent_ids_from_access_groups", + "get_agent_ids_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_mcp_server_ids_from_access_groups', - 'get_mcp_server_ids_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_mcp_server_ids_from_access_groups", + "get_mcp_server_ids_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_models_from_access_groups', - 'get_models_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_models_from_access_groups", + "get_models_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_team_object_from_cache', - 'get_team_object_from_cache', + "litellm.proxy.auth.auth_checks", + "", + "_get_team_object_from_cache", + "get_team_object_from_cache", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_user_role', - 'get_user_role', + "litellm.proxy.auth.auth_checks", + "", + "_get_user_role", + "get_user_role", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_is_model_cost_zero', - 'is_model_cost_zero', + "litellm.proxy.auth.auth_checks", + "", + "_is_model_cost_zero", + "is_model_cost_zero", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_is_user_proxy_admin', - 'is_user_proxy_admin', + "litellm.proxy.auth.auth_checks", + "", + "_is_user_proxy_admin", + "is_user_proxy_admin", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_key_access_group_grants_model', - 'key_access_group_grants_model', + "litellm.proxy.auth.auth_checks", + "", + "_key_access_group_grants_model", + "key_access_group_grants_model", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_organization_max_budget_check', - 'organization_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_organization_max_budget_check", + "organization_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_team_max_budget_check', - 'team_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_team_max_budget_check", + "team_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_team_member_max_budget_alert_check', - 'team_member_max_budget_alert_check', + "litellm.proxy.auth.auth_checks", + "", + "_team_member_max_budget_alert_check", + "team_member_max_budget_alert_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_max_budget_alert_check', - 'virtual_key_max_budget_alert_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_max_budget_alert_check", + "virtual_key_max_budget_alert_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_max_budget_check', - 'virtual_key_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_max_budget_check", + "virtual_key_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_soft_budget_check', - 'virtual_key_soft_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_soft_budget_check", + "virtual_key_soft_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks_organization', - '', - '_user_is_org_admin', - 'user_is_org_admin', + "litellm.proxy.auth.auth_checks_organization", + "", + "_user_is_org_admin", + "user_is_org_admin", False, ), ( - 'litellm.proxy.auth.auth_exception_handler', - 'UserAPIKeyAuthExceptionHandler', - '_handle_authentication_error', - 'handle_authentication_error', + "litellm.proxy.auth.auth_exception_handler", + "UserAPIKeyAuthExceptionHandler", + "_handle_authentication_error", + "handle_authentication_error", False, ), ( - 'litellm.proxy.auth.auth_utils', - '', - '_get_request_ip_address', - 'get_request_ip_address', + "litellm.proxy.auth.auth_utils", + "", + "_get_request_ip_address", + "get_request_ip_address", False, ), ( - 'litellm.proxy.auth.resolvers.store', - 'IdentityStore', - '_principal_from_key', - 'principal_from_key', + "litellm.proxy.auth.resolvers.store", + "IdentityStore", + "_principal_from_key", + "principal_from_key", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_get_request_method', - 'get_request_method', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_get_request_method", + "get_request_method", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_is_assistants_api_request', - 'is_assistants_api_request', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_is_assistants_api_request", + "is_assistants_api_request", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_is_wildcard_pattern', - 'is_wildcard_pattern', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_is_wildcard_pattern", + "is_wildcard_pattern", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_enforce_key_and_fallback_model_access', - 'enforce_key_and_fallback_model_access', + "litellm.proxy.auth.user_api_key_auth", + "", + "_enforce_key_and_fallback_model_access", + "enforce_key_and_fallback_model_access", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_fetch_global_spend_with_event_coordination', - 'fetch_global_spend_with_event_coordination', + "litellm.proxy.auth.user_api_key_auth", + "", + "_fetch_global_spend_with_event_coordination", + "fetch_global_spend_with_event_coordination", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_get_bearer_token', - 'get_bearer_token', + "litellm.proxy.auth.user_api_key_auth", + "", + "_get_bearer_token", + "get_bearer_token", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_run_centralized_common_checks', - 'run_centralized_common_checks', + "litellm.proxy.auth.user_api_key_auth", + "", + "_run_centralized_common_checks", + "run_centralized_common_checks", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_user_api_key_auth_builder', - 'user_api_key_auth_builder', + "litellm.proxy.auth.user_api_key_auth", + "", + "_user_api_key_auth_builder", + "user_api_key_auth_builder", False, ), ( - 'litellm.proxy.common_request_processing', - '', - '_is_azure_model_router_request', - 'is_azure_model_router_request', + "litellm.proxy.common_request_processing", + "", + "_is_azure_model_router_request", + "is_azure_model_router_request", False, ), ( - 'litellm.proxy.common_request_processing', - '', - '_should_return_raw_model_name', - 'should_return_raw_model_name', + "litellm.proxy.common_request_processing", + "", + "_should_return_raw_model_name", + "should_return_raw_model_name", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_finalize_streaming_generator_cleanup', - 'finalize_streaming_generator_cleanup', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_finalize_streaming_generator_cleanup", + "finalize_streaming_generator_cleanup", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_handle_llm_api_exception', - 'handle_llm_api_exception', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_handle_llm_api_exception", + "handle_llm_api_exception", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_process_chunk_with_cost_injection', - 'process_chunk_with_cost_injection', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_process_chunk_with_cost_injection", + "process_chunk_with_cost_injection", False, ), ( - 'litellm.proxy.common_utils.callback_utils', - '', - '_CALLBACK_VAR_ENCRYPTED_PREFIX', - 'CALLBACK_VAR_ENCRYPTED_PREFIX', + "litellm.proxy.common_utils.callback_utils", + "", + "_CALLBACK_VAR_ENCRYPTED_PREFIX", + "CALLBACK_VAR_ENCRYPTED_PREFIX", False, ), ( - 'litellm.proxy.common_utils.config_sync_pubsub', - '', - '_ConfigSyncPubSub', - 'ConfigSyncPubSub', + "litellm.proxy.common_utils.config_sync_pubsub", + "", + "_ConfigSyncPubSub", + "ConfigSyncPubSub", False, ), ( - 'litellm.proxy.common_utils.config_sync_pubsub', - '', - '_pubsub_capable_client', - 'pubsub_capable_client', + "litellm.proxy.common_utils.config_sync_pubsub", + "", + "_pubsub_capable_client", + "pubsub_capable_client", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_ALGO_AES_GCM', - 'ALGO_AES_GCM', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_ALGO_AES_GCM", + "ALGO_AES_GCM", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_ENCRYPTION_ALGORITHM_SETTING', - 'ENCRYPTION_ALGORITHM_SETTING', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_ENCRYPTION_ALGORITHM_SETTING", + "ENCRYPTION_ALGORITHM_SETTING", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_V2_GCM_PREFIX', - 'V2_GCM_PREFIX', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_V2_GCM_PREFIX", + "V2_GCM_PREFIX", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_get_salt_key', - 'get_salt_key', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_get_salt_key", + "get_salt_key", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_read_request_body', - 'read_request_body', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_read_request_body", + "read_request_body", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_get_request_headers', - 'safe_get_request_headers', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_get_request_headers", + "safe_get_request_headers", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_get_request_query_params', - 'safe_get_request_query_params', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_get_request_query_params", + "safe_get_request_query_params", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_set_request_parsed_body', - 'safe_set_request_parsed_body', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_set_request_parsed_body", + "safe_set_request_parsed_body", False, ), ( - 'litellm.proxy.common_utils.realtime_utils', - '', - '_realtime_request_body', - 'realtime_request_body', + "litellm.proxy.common_utils.realtime_utils", + "", + "_realtime_request_body", + "realtime_request_body", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_commit_daily_tag_spend_to_db', - 'commit_daily_tag_spend_to_db', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_commit_daily_tag_spend_to_db", + "commit_daily_tag_spend_to_db", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_commit_daily_tag_spend_to_db_with_redis', - 'commit_daily_tag_spend_to_db_with_redis', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_commit_daily_tag_spend_to_db_with_redis", + "commit_daily_tag_spend_to_db_with_redis", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_handle_spend_update_failure', - 'handle_spend_update_failure', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_handle_spend_update_failure", + "handle_spend_update_failure", False, ), ( - 'litellm.proxy.db.db_transaction_queue.redis_update_buffer', - 'RedisUpdateBuffer', - '_should_commit_spend_updates_to_redis', - 'should_commit_spend_updates_to_redis', + "litellm.proxy.db.db_transaction_queue.redis_update_buffer", + "RedisUpdateBuffer", + "_should_commit_spend_updates_to_redis", + "should_commit_spend_updates_to_redis", False, ), ( - 'litellm.proxy.db.log_db_metrics', - '', - '_is_exception_related_to_db', - 'is_exception_related_to_db', + "litellm.proxy.db.log_db_metrics", + "", + "_is_exception_related_to_db", + "is_exception_related_to_db", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.azure.base', - '', - '_RESPONSES_API_CALL_TYPES', - 'RESPONSES_API_CALL_TYPES', + "litellm.proxy.guardrails.guardrail_hooks.azure.base", + "", + "_RESPONSES_API_CALL_TYPES", + "RESPONSES_API_CALL_TYPES", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.cisco_ai_defense.cisco_ai_defense_mcp', - '', - '_CiscoAIDefenseMcpMixin', - 'CiscoAIDefenseMcpMixin', + "litellm.proxy.guardrails.guardrail_hooks.cisco_ai_defense.cisco_ai_defense_mcp", + "", + "_CiscoAIDefenseMcpMixin", + "CiscoAIDefenseMcpMixin", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_compile_marker', - 'compile_marker', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_compile_marker", + "compile_marker", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_count_signals', - 'count_signals', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_count_signals", + "count_signals", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_word_boundary_match', - 'word_boundary_match', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_word_boundary_match", + "word_boundary_match", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.presidio', - '', - '_OPTIONAL_PresidioPIIMasking', - 'OPTIONAL_PresidioPIIMasking', + "litellm.proxy.guardrails.guardrail_hooks.presidio", + "", + "_OPTIONAL_PresidioPIIMasking", + "OPTIONAL_PresidioPIIMasking", False, ), ( - 'litellm.proxy.health_check', - '', - '_clean_endpoint_data', - 'clean_endpoint_data', + "litellm.proxy.health_check", + "", + "_clean_endpoint_data", + "clean_endpoint_data", False, ), ( - 'litellm.proxy.health_check', - '', - '_update_litellm_params_for_health_check', - 'update_litellm_params_for_health_check', + "litellm.proxy.health_check", + "", + "_update_litellm_params_for_health_check", + "update_litellm_params_for_health_check", False, ), ( - 'litellm.proxy.health_endpoints._health_endpoints', - '', - '_convert_health_check_to_dict', - 'convert_health_check_to_dict', + "litellm.proxy.health_endpoints._health_endpoints", + "", + "_convert_health_check_to_dict", + "convert_health_check_to_dict", False, ), ( - 'litellm.proxy.health_endpoints._health_endpoints', - '', - '_save_background_health_checks_to_db', - 'save_background_health_checks_to_db', + "litellm.proxy.health_endpoints._health_endpoints", + "", + "_save_background_health_checks_to_db", + "save_background_health_checks_to_db", False, ), ( - 'litellm.proxy.hooks.azure_content_safety', - '', - '_PROXY_AzureContentSafety', - 'PROXY_AzureContentSafety', + "litellm.proxy.hooks.azure_content_safety", + "", + "_PROXY_AzureContentSafety", + "PROXY_AzureContentSafety", False, ), ( - 'litellm.proxy.hooks.batch_rate_limiter', - '', - '_PROXY_BatchRateLimiter', - 'PROXY_BatchRateLimiter', + "litellm.proxy.hooks.batch_rate_limiter", + "", + "_PROXY_BatchRateLimiter", + "PROXY_BatchRateLimiter", False, ), ( - 'litellm.proxy.hooks.batch_redis_get', - '', - '_PROXY_BatchRedisRequests', - 'PROXY_BatchRedisRequests', + "litellm.proxy.hooks.batch_redis_get", + "", + "_PROXY_BatchRedisRequests", + "PROXY_BatchRedisRequests", False, ), ( - 'litellm.proxy.hooks.cache_control_check', - '', - '_PROXY_CacheControlCheck', - 'PROXY_CacheControlCheck', + "litellm.proxy.hooks.cache_control_check", + "", + "_PROXY_CacheControlCheck", + "PROXY_CacheControlCheck", False, ), ( - 'litellm.proxy.hooks.dynamic_rate_limiter', - '', - '_PROXY_DynamicRateLimitHandler', - 'PROXY_DynamicRateLimitHandler', + "litellm.proxy.hooks.dynamic_rate_limiter", + "", + "_PROXY_DynamicRateLimitHandler", + "PROXY_DynamicRateLimitHandler", False, ), ( - 'litellm.proxy.hooks.dynamic_rate_limiter_v3', - '', - '_PROXY_DynamicRateLimitHandlerV3', - 'PROXY_DynamicRateLimitHandlerV3', + "litellm.proxy.hooks.dynamic_rate_limiter_v3", + "", + "_PROXY_DynamicRateLimitHandlerV3", + "PROXY_DynamicRateLimitHandlerV3", False, ), ( - 'litellm.proxy.hooks.max_budget_per_session_limiter', - '', - '_PROXY_MaxBudgetPerSessionHandler', - 'PROXY_MaxBudgetPerSessionHandler', + "litellm.proxy.hooks.max_budget_per_session_limiter", + "", + "_PROXY_MaxBudgetPerSessionHandler", + "PROXY_MaxBudgetPerSessionHandler", False, ), ( - 'litellm.proxy.hooks.max_iterations_limiter', - '', - '_PROXY_MaxIterationsHandler', - 'PROXY_MaxIterationsHandler', + "litellm.proxy.hooks.max_iterations_limiter", + "", + "_PROXY_MaxIterationsHandler", + "PROXY_MaxIterationsHandler", False, ), ( - 'litellm.proxy.hooks.model_max_budget_limiter', - '', - '_PROXY_VirtualKeyModelMaxBudgetLimiter', - 'PROXY_VirtualKeyModelMaxBudgetLimiter', + "litellm.proxy.hooks.model_max_budget_limiter", + "", + "_PROXY_VirtualKeyModelMaxBudgetLimiter", + "PROXY_VirtualKeyModelMaxBudgetLimiter", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter', - '', - '_PROXY_MaxParallelRequestsHandler', - 'PROXY_MaxParallelRequestsHandler', + "litellm.proxy.hooks.parallel_request_limiter", + "", + "_PROXY_MaxParallelRequestsHandler", + "PROXY_MaxParallelRequestsHandler", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter_v3', - '', - '_PROXY_MaxParallelRequestsHandler_v3', - 'PROXY_MaxParallelRequestsHandler_v3', + "litellm.proxy.hooks.parallel_request_limiter_v3", + "", + "_PROXY_MaxParallelRequestsHandler_v3", + "PROXY_MaxParallelRequestsHandler_v3", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter_v3', - '_PROXY_MaxParallelRequestsHandler_v3', - '_create_rate_limit_descriptors', - 'create_rate_limit_descriptors', + "litellm.proxy.hooks.parallel_request_limiter_v3", + "_PROXY_MaxParallelRequestsHandler_v3", + "_create_rate_limit_descriptors", + "create_rate_limit_descriptors", False, ), ( - 'litellm.proxy.hooks.prompt_injection_detection', - '', - '_OPTIONAL_PromptInjectionDetection', - 'OPTIONAL_PromptInjectionDetection', + "litellm.proxy.hooks.prompt_injection_detection", + "", + "_OPTIONAL_PromptInjectionDetection", + "OPTIONAL_PromptInjectionDetection", False, ), ( - 'litellm.proxy.hooks.proxy_track_cost_callback', - '', - '_ProxyDBLogger', - 'ProxyDBLogger', + "litellm.proxy.hooks.proxy_track_cost_callback", + "", + "_ProxyDBLogger", + "ProxyDBLogger", False, ), ( - 'litellm.proxy.hooks.sensitive_data_routing', - '', - '_PROXY_SensitiveDataRoutingHandler', - 'PROXY_SensitiveDataRoutingHandler', + "litellm.proxy.hooks.sensitive_data_routing", + "", + "_PROXY_SensitiveDataRoutingHandler", + "PROXY_SensitiveDataRoutingHandler", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_add_guardrails_from_key_or_team_metadata', - 'add_guardrails_from_key_or_team_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_add_guardrails_from_key_or_team_metadata", + "add_guardrails_from_key_or_team_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_dynamic_logging_metadata', - 'get_dynamic_logging_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_dynamic_logging_metadata", + "get_dynamic_logging_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_metadata_variable_name', - 'get_metadata_variable_name', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_metadata_variable_name", + "get_metadata_variable_name", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_validated_callback_metadata', - 'get_validated_callback_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_validated_callback_metadata", + "get_validated_callback_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - 'LiteLLMProxyRequestSetup', - '_merge_tags', - 'merge_tags', + "litellm.proxy.litellm_pre_call_utils", + "LiteLLMProxyRequestSetup", + "_merge_tags", + "merge_tags", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_check_disable_global_guardrails_caller_permission', - 'check_disable_global_guardrails_caller_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_check_disable_global_guardrails_caller_permission", + "check_disable_global_guardrails_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_check_passthrough_routes_caller_permission', - 'check_passthrough_routes_caller_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_check_passthrough_routes_caller_permission", + "check_passthrough_routes_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_set_object_metadata_field', - 'set_object_metadata_field', + "litellm.proxy.management_endpoints.common_utils", + "", + "_set_object_metadata_field", + "set_object_metadata_field", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_team_member_has_permission', - 'team_member_has_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_team_member_has_permission", + "team_member_has_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_update_metadata_fields', - 'update_metadata_fields', + "litellm.proxy.management_endpoints.common_utils", + "", + "_update_metadata_fields", + "update_metadata_fields", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_upsert_budget_and_membership', - 'upsert_budget_and_membership', + "litellm.proxy.management_endpoints.common_utils", + "", + "_upsert_budget_and_membership", + "upsert_budget_and_membership", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_user_has_admin_privileges', - 'user_has_admin_privileges', + "litellm.proxy.management_endpoints.common_utils", + "", + "_user_has_admin_privileges", + "user_has_admin_privileges", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_user_has_admin_view', - 'user_api_key_has_admin_view', + "litellm.proxy.management_endpoints.common_utils", + "", + "_user_has_admin_view", + "user_api_key_has_admin_view", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_clear_hashicorp_vault_state', - 'clear_hashicorp_vault_state', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_clear_hashicorp_vault_state", + "clear_hashicorp_vault_state", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_get_current_env_values', - 'get_current_env_values', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_get_current_env_values", + "get_current_env_values", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_parse_config_value', - 'parse_config_value', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_parse_config_value", + "parse_config_value", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_set_env_vars', - 'set_env_vars', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_set_env_vars", + "set_env_vars", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_calculate_key_rotation_time', - 'calculate_key_rotation_time', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_calculate_key_rotation_time", + "calculate_key_rotation_time", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_check_permissions_caller_permission', - 'check_permissions_caller_permission', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_check_permissions_caller_permission", + "check_permissions_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_get_caller_team_role', - 'get_caller_team_role', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_get_caller_team_role", + "get_caller_team_role", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_list_key_helper', - 'list_key_helper', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_list_key_helper", + "list_key_helper", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_persist_deleted_verification_tokens', - 'persist_deleted_verification_tokens', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_persist_deleted_verification_tokens", + "persist_deleted_verification_tokens", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_rotate_master_key', - 'rotate_master_key', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_rotate_master_key", + "rotate_master_key", False, ), ( - 'litellm.proxy.management_endpoints.mcp_management_endpoints', - '', - '_inherit_credentials_from_existing_server', - 'inherit_credentials_from_existing_server', + "litellm.proxy.management_endpoints.mcp_management_endpoints", + "", + "_inherit_credentials_from_existing_server", + "inherit_credentials_from_existing_server", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_add_model_to_db', - 'add_model_to_db', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_add_model_to_db", + "add_model_to_db", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_add_team_model_to_db', - 'add_team_model_to_db', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_add_team_model_to_db", + "add_team_model_to_db", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_deduplicate_litellm_router_models', - 'deduplicate_litellm_router_models', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_deduplicate_litellm_router_models", + "deduplicate_litellm_router_models", False, ), ( - 'litellm.proxy.management_endpoints.organization_endpoints', - '', - '_verify_org_access', - 'verify_org_access', + "litellm.proxy.management_endpoints.organization_endpoints", + "", + "_verify_org_access", + "verify_org_access", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_all_names_per_competitor', - 'build_all_names_per_competitor', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_all_names_per_competitor", + "build_all_names_per_competitor", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_comparison_blocked_words', - 'build_comparison_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_comparison_blocked_words", + "build_comparison_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_competitor_guardrail_definitions', - 'build_competitor_guardrail_definitions', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_competitor_guardrail_definitions", + "build_competitor_guardrail_definitions", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_name_blocked_words', - 'build_name_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_name_blocked_words", + "build_name_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_recommendation_blocked_words', - 'build_recommendation_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_recommendation_blocked_words", + "build_recommendation_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_refinement_prompt', - 'build_refinement_prompt', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_refinement_prompt", + "build_refinement_prompt", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_clean_competitor_line', - 'clean_competitor_line', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_clean_competitor_line", + "clean_competitor_line", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_parse_variations_response', - 'parse_variations_response', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_parse_variations_response", + "parse_variations_response", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - '', - '_cleanup_members_with_roles', - 'cleanup_members_with_roles', + "litellm.proxy.management_endpoints.team_endpoints", + "", + "_cleanup_members_with_roles", + "cleanup_members_with_roles", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - '', - '_refresh_cached_team', - 'refresh_cached_team', + "litellm.proxy.management_endpoints.team_endpoints", + "", + "_refresh_cached_team", + "refresh_cached_team", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - 'TeamMemberBudgetHandler', - '_clean_team_member_fields', - 'clean_team_member_fields', + "litellm.proxy.management_endpoints.team_endpoints", + "TeamMemberBudgetHandler", + "_clean_team_member_fields", + "clean_team_member_fields", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - '', - '_sso_return_to_redirect', - 'sso_return_to_redirect', + "litellm.proxy.management_endpoints.ui_sso", + "", + "_sso_return_to_redirect", + "sso_return_to_redirect", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_delete_pkce_verifier', - 'delete_pkce_verifier', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_delete_pkce_verifier", + "delete_pkce_verifier", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_get_cli_state', - 'get_cli_state', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_get_cli_state", + "get_cli_state", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_get_user_email_and_id_from_result', - 'get_user_email_and_id_from_result', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_get_user_email_and_id_from_result", + "get_user_email_and_id_from_result", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_pkce_token_exchange', - 'pkce_token_exchange', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_pkce_token_exchange", + "pkce_token_exchange", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_validate_return_to', - 'validate_return_to', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_validate_return_to", + "validate_return_to", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_get_allow_all_keys_server_ids', - 'get_allow_all_keys_server_ids', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_get_allow_all_keys_server_ids", + "get_allow_all_keys_server_ids", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_get_team_allowed_mcp_servers', - 'get_team_allowed_mcp_servers', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_get_team_allowed_mcp_servers", + "get_team_allowed_mcp_servers", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_set_object_permission', - 'set_object_permission', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_set_object_permission", + "set_object_permission", False, ), ( - 'litellm.proxy.openai_files_endpoints.common_utils', - '', - '_is_base64_encoded_unified_file_id', - 'is_base64_encoded_unified_file_id', + "litellm.proxy.openai_files_endpoints.common_utils", + "", + "_is_base64_encoded_unified_file_id", + "is_base64_encoded_unified_file_id", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints', - '', - '_extract_model_from_bedrock_endpoint', - 'extract_model_from_bedrock_endpoint', + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints", + "", + "_extract_model_from_bedrock_endpoint", + "extract_model_from_bedrock_endpoint", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints', - 'BaseOpenAIPassThroughHandler', - '_base_openai_pass_through_handler', - 'base_openai_pass_through_handler', + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints", + "BaseOpenAIPassThroughHandler", + "_base_openai_pass_through_handler", + "base_openai_pass_through_handler", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_build_complete_streaming_response', - 'build_complete_streaming_response', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_build_complete_streaming_response", + "build_complete_streaming_response", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_build_usage_only_response_from_chunks', - 'build_usage_only_response_from_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_build_usage_only_response_from_chunks", + "build_usage_only_response_from_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_handle_logging_anthropic_collected_chunks', - 'handle_logging_anthropic_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_handle_logging_anthropic_collected_chunks", + "handle_logging_anthropic_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_get_assembly_base_url_from_region', - 'get_assembly_base_url_from_region', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_get_assembly_base_url_from_region", + "get_assembly_base_url_from_region", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_get_assembly_region_from_url', - 'get_assembly_region_from_url', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_get_assembly_region_from_url", + "get_assembly_region_from_url", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_should_log_request', - 'should_log_request', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_should_log_request", + "should_log_request", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler', - '', - '_is_openai_compatible_url', - 'is_openai_compatible_url', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler", + "", + "_is_openai_compatible_url", + "is_openai_compatible_url", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler', - 'OpenAIPassthroughLoggingHandler', - '_handle_logging_openai_collected_chunks', - 'handle_logging_openai_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler", + "OpenAIPassthroughLoggingHandler", + "_handle_logging_openai_collected_chunks", + "handle_logging_openai_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler', - 'VertexPassthroughLoggingHandler', - '_handle_logging_vertex_collected_chunks', - 'handle_logging_vertex_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler", + "VertexPassthroughLoggingHandler", + "_handle_logging_vertex_collected_chunks", + "handle_logging_vertex_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.pass_through_endpoints', - 'HttpPassThroughEndpointHelpers', - '_init_kwargs_for_pass_through_endpoint', - 'init_kwargs_for_pass_through_endpoint', + "litellm.proxy.pass_through_endpoints.pass_through_endpoints", + "HttpPassThroughEndpointHelpers", + "_init_kwargs_for_pass_through_endpoint", + "init_kwargs_for_pass_through_endpoint", False, ), ( - 'litellm.proxy.pass_through_endpoints.pass_through_endpoints', - 'HttpPassThroughEndpointHelpers', - '_update_stream_param_based_on_request_body', - 'update_stream_param_based_on_request_body', + "litellm.proxy.pass_through_endpoints.pass_through_endpoints", + "HttpPassThroughEndpointHelpers", + "_update_stream_param_based_on_request_body", + "update_stream_param_based_on_request_body", False, ), ( - 'litellm.proxy.pass_through_endpoints.streaming_handler', - 'PassThroughStreamingHandler', - '_route_streaming_logging_to_handler', - 'route_streaming_logging_to_handler', + "litellm.proxy.pass_through_endpoints.streaming_handler", + "PassThroughStreamingHandler", + "_route_streaming_logging_to_handler", + "route_streaming_logging_to_handler", False, ), ( - 'litellm.proxy.pass_through_endpoints.success_handler', - 'PassThroughEndpointLogging', - '_handle_logging', - 'handle_logging', + "litellm.proxy.pass_through_endpoints.success_handler", + "PassThroughEndpointLogging", + "_handle_logging", + "handle_logging", False, ), ( - 'litellm.proxy.policy_engine.policy_registry', - 'PolicyRegistry', - '_parse_policy', - 'parse_policy', + "litellm.proxy.policy_engine.policy_registry", + "PolicyRegistry", + "_parse_policy", + "parse_policy", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_apply_uvicorn_max_requests_jitter', - 'apply_uvicorn_max_requests_jitter', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_apply_uvicorn_max_requests_jitter", + "apply_uvicorn_max_requests_jitter", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_configure_dev_reload', - 'configure_dev_reload', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_configure_dev_reload", + "configure_dev_reload", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_echo_litellm_version', - 'echo_litellm_version', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_echo_litellm_version", + "echo_litellm_version", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_get_default_unvicorn_init_args', - 'get_default_unvicorn_init_args', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_get_default_unvicorn_init_args", + "get_default_unvicorn_init_args", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_get_loop_type', - 'get_loop_type', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_get_loop_type", + "get_loop_type", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_init_granian_server', - 'init_granian_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_init_granian_server", + "init_granian_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_init_hypercorn_server', - 'init_hypercorn_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_init_hypercorn_server", + "init_hypercorn_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_is_port_in_use', - 'is_port_in_use', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_is_port_in_use", + "is_port_in_use", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_maybe_setup_prometheus_multiproc_dir', - 'maybe_setup_prometheus_multiproc_dir', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_maybe_setup_prometheus_multiproc_dir", + "maybe_setup_prometheus_multiproc_dir", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_config_validation', - 'run_config_validation', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_config_validation", + "run_config_validation", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_gunicorn_server', - 'run_gunicorn_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_gunicorn_server", + "run_gunicorn_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_health_check', - 'run_health_check', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_health_check", + "run_health_check", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_ollama_serve', - 'run_ollama_serve', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_ollama_serve", + "run_ollama_serve", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_test_chat_completion', - 'run_test_chat_completion', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_test_chat_completion", + "run_test_chat_completion", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_build_redis_usage_cache', - 'build_redis_usage_cache', + "litellm.proxy.proxy_server", + "", + "_build_redis_usage_cache", + "build_redis_usage_cache", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_ensure_spend_counter_initialized', - 'ensure_spend_counter_initialized', + "litellm.proxy.proxy_server", + "", + "_ensure_spend_counter_initialized", + "ensure_spend_counter_initialized", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_ensure_window_spend_counter_initialized', - 'ensure_window_spend_counter_initialized', + "litellm.proxy.proxy_server", + "", + "_ensure_window_spend_counter_initialized", + "ensure_window_spend_counter_initialized", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_environment_has_redis_connection_target', - 'environment_has_redis_connection_target', + "litellm.proxy.proxy_server", + "", + "_environment_has_redis_connection_target", + "environment_has_redis_connection_target", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_get_model_group_info', - 'get_model_group_info', + "litellm.proxy.proxy_server", + "", + "_get_model_group_info", + "get_model_group_info", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_increment_spend_counter_cache', - 'increment_spend_counter_cache', + "litellm.proxy.proxy_server", + "", + "_increment_spend_counter_cache", + "increment_spend_counter_cache", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_initialize_shared_aiohttp_session', - 'initialize_shared_aiohttp_session', + "litellm.proxy.proxy_server", + "", + "_initialize_shared_aiohttp_session", + "initialize_shared_aiohttp_session", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_invalidate_spend_counter', - 'invalidate_spend_counter', + "litellm.proxy.proxy_server", + "", + "_invalidate_spend_counter", + "invalidate_spend_counter", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_title', - 'title', + "litellm.proxy.proxy_server", + "", + "_title", + "title", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_add_deployment_locked', - 'add_deployment_locked', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_add_deployment_locked", + "add_deployment_locked", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_decrypt_and_set_db_env_variables', - 'decrypt_and_set_db_env_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_decrypt_and_set_db_env_variables", + "decrypt_and_set_db_env_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_decrypt_db_variables', - 'decrypt_db_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_decrypt_db_variables", + "decrypt_db_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_encrypt_env_variables', - 'encrypt_env_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_encrypt_env_variables", + "encrypt_env_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_encrypt_env_variables_for_db', - 'encrypt_env_variables_for_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_encrypt_env_variables_for_db", + "encrypt_env_variables_for_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_get_models_from_db', - 'get_models_from_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_get_models_from_db", + "get_models_from_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_init_cache', - 'init_cache', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_init_cache", + "init_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_init_semantic_filter_settings_in_db', - 'init_semantic_filter_settings_in_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_init_semantic_filter_settings_in_db", + "init_semantic_filter_settings_in_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_serve_pass_through_endpoints', - 'serve_pass_through_endpoints', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_serve_pass_through_endpoints", + "serve_pass_through_endpoints", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_add_proxy_budget_to_db', - 'add_proxy_budget_to_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_add_proxy_budget_to_db", + "add_proxy_budget_to_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_attach_router_to_prompt_injection_detectors', - 'attach_router_to_prompt_injection_detectors', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_attach_router_to_prompt_injection_detectors", + "attach_router_to_prompt_injection_detectors", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_get_transaction_buffer_redis_cache', - 'get_transaction_buffer_redis_cache', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_get_transaction_buffer_redis_cache", + "get_transaction_buffer_redis_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_coordination_redis_from_db', - 'init_coordination_redis_from_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_coordination_redis_from_db", + "init_coordination_redis_from_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_dd_tracer', - 'init_dd_tracer', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_dd_tracer", + "init_dd_tracer", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_pyroscope', - 'init_pyroscope', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_pyroscope", + "init_pyroscope", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_jwt_auth', - 'initialize_jwt_auth', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_jwt_auth", + "initialize_jwt_auth", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_semantic_tool_filter', - 'initialize_semantic_tool_filter', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_semantic_tool_filter", + "initialize_semantic_tool_filter", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_startup_logging', - 'initialize_startup_logging', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_startup_logging", + "initialize_startup_logging", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_setup_prisma_client', - 'setup_prisma_client', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_setup_prisma_client", + "setup_prisma_client", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_sync_ui_settings_to_general_settings', - 'sync_ui_settings_to_general_settings', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_sync_ui_settings_to_general_settings", + "sync_ui_settings_to_general_settings", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_update_default_team_member_budget', - 'update_default_team_member_budget', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_update_default_team_member_budget", + "update_default_team_member_budget", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_validate_redis_transaction_buffer_config', - 'validate_redis_transaction_buffer_config', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_validate_redis_transaction_buffer_config", + "validate_redis_transaction_buffer_config", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warm_global_spend_cache', - 'warm_global_spend_cache', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warm_global_spend_cache", + "warm_global_spend_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_budget_without_db', - 'warn_budget_without_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_budget_without_db", + "warn_budget_without_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_fail_closed_rate_limits_without_redis', - 'warn_fail_closed_rate_limits_without_redis', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_fail_closed_rate_limits_without_redis", + "warn_fail_closed_rate_limits_without_redis", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_if_mock_testing_params_enabled', - 'warn_if_mock_testing_params_enabled', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_if_mock_testing_params_enabled", + "warn_if_mock_testing_params_enabled", False, ), ( - 'litellm.proxy.spend_tracking.spend_management_endpoints', - '', - '_get_spend_report_for_time_range', - 'get_spend_report_for_time_range', + "litellm.proxy.spend_tracking.spend_management_endpoints", + "", + "_get_spend_report_for_time_range", + "get_spend_report_for_time_range", False, ), ( - 'litellm.proxy.spend_tracking.spend_management_endpoints', - '', - '_is_admin_view_safe', - 'is_admin_view_safe', + "litellm.proxy.spend_tracking.spend_management_endpoints", + "", + "_is_admin_view_safe", + "is_admin_view_safe", False, ), ( - 'litellm.proxy.spend_tracking.spend_tracking_utils', - '', - '_is_master_key', - 'is_master_key', + "litellm.proxy.spend_tracking.spend_tracking_utils", + "", + "_is_master_key", + "is_master_key", False, ), ( - 'litellm.proxy.spend_tracking.spend_tracking_utils', - '', - '_sanitize_error_information_for_spend_logs', - 'sanitize_error_information_for_spend_logs', + "litellm.proxy.spend_tracking.spend_tracking_utils", + "", + "_sanitize_error_information_for_spend_logs", + "sanitize_error_information_for_spend_logs", False, ), ( - 'litellm.proxy.utils', - '', - '_cache_user_row', - 'cache_user_row', + "litellm.proxy.utils", + "", + "_cache_user_row", + "cache_user_row", False, ), ( - 'litellm.proxy.utils', - '', - '_check_and_merge_model_level_guardrails', - 'check_and_merge_model_level_guardrails', + "litellm.proxy.utils", + "", + "_check_and_merge_model_level_guardrails", + "check_and_merge_model_level_guardrails", False, ), ( - 'litellm.proxy.utils', - '', - '_get_docs_url', - 'get_docs_url', + "litellm.proxy.utils", + "", + "_get_docs_url", + "get_docs_url", False, ), ( - 'litellm.proxy.utils', - '', - '_get_openapi_url', - 'get_openapi_url', + "litellm.proxy.utils", + "", + "_get_openapi_url", + "get_openapi_url", False, ), ( - 'litellm.proxy.utils', - '', - '_get_projected_spend_over_limit', - 'get_projected_spend_over_limit', + "litellm.proxy.utils", + "", + "_get_projected_spend_over_limit", + "get_projected_spend_over_limit", False, ), ( - 'litellm.proxy.utils', - '', - '_get_redoc_url', - 'get_redoc_url', + "litellm.proxy.utils", + "", + "_get_redoc_url", + "get_redoc_url", False, ), ( - 'litellm.proxy.utils', - '', - '_hash_token_if_needed', - 'hash_token_if_needed', + "litellm.proxy.utils", + "", + "_hash_token_if_needed", + "hash_token_if_needed", False, ), ( - 'litellm.proxy.utils', - '', - '_is_projected_spend_over_limit', - 'is_projected_spend_over_limit', + "litellm.proxy.utils", + "", + "_is_projected_spend_over_limit", + "is_projected_spend_over_limit", False, ), ( - 'litellm.proxy.utils', - '', - '_is_valid_team_configs', - 'is_valid_team_configs', + "litellm.proxy.utils", + "", + "_is_valid_team_configs", + "is_valid_team_configs", False, ), ( - 'litellm.proxy.utils', - '', - '_monitor_spend_logs_queue', - 'monitor_spend_logs_queue', + "litellm.proxy.utils", + "", + "_monitor_spend_logs_queue", + "monitor_spend_logs_queue", False, ), ( - 'litellm.proxy.utils', - '', - '_premium_user_check', - 'premium_user_check', + "litellm.proxy.utils", + "", + "_premium_user_check", + "premium_user_check", False, ), ( - 'litellm.proxy.utils', - '', - '_raise_failed_update_spend_exception', - 'raise_failed_update_spend_exception', + "litellm.proxy.utils", + "", + "_raise_failed_update_spend_exception", + "raise_failed_update_spend_exception", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_arelease_max_parallel_requests_on_disconnect', - 'arelease_max_parallel_requests_on_disconnect', + "litellm.proxy.utils", + "ProxyLogging", + "_arelease_max_parallel_requests_on_disconnect", + "arelease_max_parallel_requests_on_disconnect", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_callback_capabilities', - 'callback_capabilities', + "litellm.proxy.utils", + "ProxyLogging", + "_callback_capabilities", + "callback_capabilities", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_convert_mcp_hook_response_to_kwargs', - 'convert_mcp_hook_response_to_kwargs', + "litellm.proxy.utils", + "ProxyLogging", + "_convert_mcp_hook_response_to_kwargs", + "convert_mcp_hook_response_to_kwargs", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_create_mcp_request_object_from_kwargs', - 'create_mcp_request_object_from_kwargs', + "litellm.proxy.utils", + "ProxyLogging", + "_create_mcp_request_object_from_kwargs", + "create_mcp_request_object_from_kwargs", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_fire_deferred_stream_logging', - 'fire_deferred_stream_logging', + "litellm.proxy.utils", + "ProxyLogging", + "_fire_deferred_stream_logging", + "fire_deferred_stream_logging", False, ), ) @@ -5188,17 +5201,16 @@ LLMS_FORWARDER_CASES: Final = ( ) PROXY_FORWARDER_CASES: Final = ( ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_convert_mcp_to_llm_format', - 'convert_mcp_to_llm_format', - 'instance', + "litellm.proxy.utils", + "ProxyLogging", + "_convert_mcp_to_llm_format", + "convert_mcp_to_llm_format", + "instance", False, ), ) - def _get_owner(module_path: str, owner_name: str) -> object: module: Final = import_module(module_path or "litellm") if module_path == "litellm.llms.openai_like.dynamic_config" and owner_name == "JSONProviderConfig": @@ -5292,14 +5304,10 @@ def _make_private_override( def recorded_args(args: tuple[object, ...]) -> tuple[object, ...]: return args[1:] if descriptor in ("instance", "classmethod") else args - def sync_override( - *args: object, **kwargs: object - ) -> tuple[object, tuple[object, ...], dict[str, object]]: + def sync_override(*args: object, **kwargs: object) -> tuple[object, tuple[object, ...], dict[str, object]]: return result, recorded_args(args), kwargs - async def async_override( - *args: object, **kwargs: object - ) -> tuple[object, tuple[object, ...], dict[str, object]]: + async def async_override(*args: object, **kwargs: object) -> tuple[object, tuple[object, ...], dict[str, object]]: return result, recorded_args(args), kwargs implementation: Final = async_override if is_async else sync_override