diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 62bde5806a4..2dfb53b1634 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2067,9 +2067,13 @@ dependencies = [ name = "litellm-callbacks-legacy-python" version = "0.1.0" dependencies = [ + "base64 0.22.1", "litellm-auth", + "litellm-cache", + "litellm-core-utils", "litellm-host", "litellm-host-python", + "litellm-token-counter", "proptest", "pyo3", "rstest", @@ -2087,10 +2091,13 @@ dependencies = [ "litellm-auth", "litellm-auth-aws", "litellm-auth-gcp", + "litellm-cache", + "litellm-cache-memory", "litellm-core-utils", "litellm-host", "litellm-http", "litellm-llms", + "litellm-token-counter", "litellm-types", "mime_guess", "moka", diff --git a/litellm-rust/crates/cache/src/handler.rs b/litellm-rust/crates/cache/src/handler.rs new file mode 100644 index 00000000000..ef14501fc3a --- /dev/null +++ b/litellm-rust/crates/cache/src/handler.rs @@ -0,0 +1,48 @@ +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde_json::Value; + +use crate::{CacheBackend, CacheControls, CacheEntry, CacheKwargs}; + +#[derive(Clone)] +pub struct LLMCachingHandler { + pub backend: CacheBackend, + pub controls: CacheControls, + pub key: String, + pub kwargs: CacheKwargs, + pub max_age: Option, +} + +impl LLMCachingHandler { + pub async fn get(&self) -> Option { + if !self.controls.reads() { + return None; + } + let entry = self + .backend + .async_get_cache(&self.key, &self.kwargs) + .await + .ok()??; + entry.fresh(now(), self.max_age).then_some(entry.response) + } + + pub async fn set(&self, response: Value) { + if !self.controls.writes() { + return; + } + let entry = CacheEntry { + timestamp: now().as_secs_f64(), + response, + }; + let _ = self + .backend + .async_set_cache(&self.key, entry, self.kwargs.clone()) + .await; + } +} + +fn now() -> Duration { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() +} diff --git a/litellm-rust/crates/cache/src/lib.rs b/litellm-rust/crates/cache/src/lib.rs index d0fe3de15cd..5f24755b9f7 100644 --- a/litellm-rust/crates/cache/src/lib.rs +++ b/litellm-rust/crates/cache/src/lib.rs @@ -1,6 +1,8 @@ mod base_cache; mod caching; mod error; +mod handler; +pub use handler::LLMCachingHandler; pub use base_cache::{ BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheFuture, CacheKwargs, diff --git a/litellm-rust/crates/callbacks-legacy-python/Cargo.toml b/litellm-rust/crates/callbacks-legacy-python/Cargo.toml index fe19578e04d..5863224be72 100644 --- a/litellm-rust/crates/callbacks-legacy-python/Cargo.toml +++ b/litellm-rust/crates/callbacks-legacy-python/Cargo.toml @@ -9,8 +9,12 @@ autotests = false [dependencies] litellm-host.workspace = true litellm-host-python.workspace = true +litellm-core-utils.workspace = true +litellm-cache.workspace = true +litellm-token-counter.workspace = true pyo3.workspace = true +base64.workspace = true strum.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index 8a7f3b98f47..007342d8703 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -6,9 +6,6 @@ "start_time", "asynchronous" ], - "check_limits": [ - "kwargs" - ], "finalize": [ "response", "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index e1742190205..ad697b0bece 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -49,6 +49,9 @@ struct DeliveredStream { } enum Pending { + CacheRead, + CacheWrite, + StreamCacheWrite, DeploymentPreCall, DeploymentPostCall, DeploymentFailure, @@ -56,6 +59,9 @@ enum Pending { } pub struct LegacyLogging { + cache: Option, + cache_hit: bool, + accounted: bool, surface: LegacySurface, call: PublicCall, logger: Option, @@ -88,6 +94,9 @@ impl LegacyLogging { asynchronous: bool, ) -> Self { Self { + cache: None, + cache_hit: false, + accounted: false, surface, call, logger: None, @@ -120,9 +129,123 @@ impl LegacyLogging { fn prepare(&mut self, py: Python<'_>) -> PyResult { let prepared = prepare(py, self.call.kwargs().bind(py), self.logger()?)?.unbind(); self.call.set_kwargs(prepared); + if self.surface.call_type == "anthropic_messages" { + self.cache = crate::caching::Caching::prepare( + py, + &self.call, + self.surface.call_type, + self.asynchronous, + )?; + if let Some(cache) = &self.cache + && cache.controls.reads() + { + match cache.read(py, self.asynchronous) { + Ok(value) if self.asynchronous => { + self.pending = Some(Pending::CacheRead); + return Ok(LifecycleStep::Await(value)); + } + result => return self.cache_read(py, result), + } + } + } + self.arguments(py) + } + + fn arguments(&mut self, py: Python<'_>) -> PyResult { + if self.surface.call_type == "anthropic_messages" { + if let Err(error) = + crate::limits::adjust_max_tokens(py, &self.call, self.surface.call_type) + && is_cancellation(py, &error) + { + return Err(error); + } + } Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py))) } + fn cache_read( + &mut self, + py: Python<'_>, + result: PyResult>, + ) -> PyResult { + let decoded = result.and_then(|value| { + self.cache + .as_ref() + .ok_or_else(missing_state)? + .decode(value.bind(py)) + }); + let response = match decoded { + Ok(Some(response)) => response, + Err(error) if is_cancellation(py, &error) => return Err(error), + _ => return self.arguments(py), + }; + self.cache_hit = true; + self.logger()? + .object(py) + .getattr("model_call_details")? + .set_item("cache_hit", true)?; + let cache = self.cache.as_ref().ok_or_else(missing_state)?; + self.logger()? + .object(py) + .getattr("litellm_params")? + .set_item("preset_cache_key", &cache.key)?; + if response + .get("litellm_cached_anthropic_sse_events") + .is_some() + { + self.call + .kwargs() + .bind(py) + .set_item("_rust_messages_cached_response", to_py(py, &response)?)?; + return Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py))); + } + self.end = Some(datetime(py, epoch_seconds())?); + self.response = Some(crate::caching::cached_public(py, &response)?); + self.finalize(py)?; + Ok(LifecycleStep::Cached( + self.response + .as_ref() + .ok_or_else(missing_state)? + .clone_ref(py), + )) + } + + fn cache_write(&mut self, py: Python<'_>) -> PyResult { + if let (Some(cache), Some(response)) = (&self.cache, &self.response) + && cache.controls.writes() + && !self.cache_hit + { + match cache.write(py, response.bind(py), self.asynchronous) { + Ok(awaitable) if self.asynchronous => { + self.pending = Some(Pending::CacheWrite); + return Ok(LifecycleStep::Await(awaitable)); + } + Err(error) if is_cancellation(py, &error) => return Err(error), + _ => {} + } + } + self.finalize(py) + } + + fn dispatch_cache_hit(&self, py: Python<'_>) -> PyResult<()> { + let kwargs = PyDict::new(py); + kwargs.set_item("result", &self.response)?; + kwargs.set_item("start_time", &self.start)?; + kwargs.set_item("end_time", &self.end)?; + kwargs.set_item("cache_hit", true)?; + let logger = self.logger()?.object(py); + if self.asynchronous { + let coroutine = logger.call_method("async_success_handler", (), Some(&kwargs))?; + crate::python::Logging::Enqueue.call(py, (coroutine,))?; + } + logger.call_method( + "handle_sync_success_callbacks_for_async_calls", + (), + Some(&kwargs), + )?; + Ok(()) + } + fn finalize(&mut self, py: Python<'_>) -> PyResult { finalize( py, @@ -138,6 +261,68 @@ impl LegacyLogging { .ok_or_else(missing_state) } + fn record_usage(&mut self, py: Python<'_>, usage: &Value) -> PyResult<()> { + if self.accounted || self.surface.call_type != "anthropic_messages" { + return Ok(()); + } + self.accounted = true; + let model = self + .call + .lookup(py, "model")? + .map(|value| value.extract::()) + .transpose()?; + let info = model + .map(|model| crate::limits::model_info(py, &model)) + .transpose()? + .flatten(); + let cost = if self.cache_hit { + Some(0.0) + } else { + info.as_ref() + .and_then(|info| litellm_core_utils::cost_calculator::messages_cost(info, usage)) + }; + if let Some(cost) = cost { + let logger = self.logger()?.object(py); + logger.setattr("_native_response_cost", cost)?; + logger + .getattr("model_call_details")? + .set_item("response_cost", cost)?; + let sdk = py.import("litellm")?; + if !self.cache_hit && sdk.getattr("max_budget")?.is_truthy()? { + let current: f64 = sdk.getattr("_current_cost")?.extract()?; + sdk.setattr("_current_cost", current + cost)?; + } + } + Ok(()) + } + + fn finish_stream(&mut self, py: Python<'_>) -> PyResult { + let stream = self.stream.as_ref().ok_or_else(missing_state)?; + let mut bytes = Vec::new(); + for chunk in stream.chunks.bind(py).iter() { + bytes.extend_from_slice(chunk.cast::()?.as_bytes()); + } + if let Some(parsed) = litellm_core_utils::messages_stream::MessagesStream::parse(&bytes) { + self.record_usage(py, &parsed.usage)?; + if !self.cache_hit + && let (Some(cache), Some(response)) = (&self.cache, parsed.cached_response) + && cache.controls.writes() + { + let response = to_py(py, &response)?; + match cache.write(py, response.bind(py), self.asynchronous) { + Ok(awaitable) if self.asynchronous => { + self.pending = Some(Pending::StreamCacheWrite); + return Ok(LifecycleStep::Await(awaitable)); + } + Err(error) if is_cancellation(py, &error) => return Err(error), + _ => {} + } + } + } + self.stream_success(py, self.stream.as_ref().ok_or_else(missing_state)?)?; + Ok(LifecycleStep::Done) + } + fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> { match self.try_dispatch_success(py) { Err(error) if error.is_instance_of::(py) => { @@ -149,6 +334,9 @@ impl LegacyLogging { } fn try_dispatch_success(&self, py: Python<'_>) -> PyResult<()> { + if self.cache_hit { + return self.dispatch_cache_hit(py); + } let logger = self.logger()?; let pending = || PendingSuccess { logger: logger.clone_ref(py), @@ -361,7 +549,7 @@ impl PythonLifecycle for LegacyLogging { self.surface.call_type, )?)); } - self.finalize(py) + self.cache_write(py) } fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult { @@ -385,10 +573,14 @@ impl PythonLifecycle for LegacyLogging { LifecycleEvent::Succeeded { timing, response } => { self.end = Some(datetime(py, timing.end_time)?); self.response = Some(response.clone_ref(py)); - match &self.stream { - Some(stream) => self.stream_success(py, stream)?, - None => self.dispatch_success(py)?, + if self.stream.is_some() { + return self.finish_stream(py); } + if self.surface.call_type == "anthropic_messages" { + let value: Value = from_py(response.bind(py))?; + self.record_usage(py, value.get("usage").unwrap_or(&Value::Null))?; + } + self.dispatch_success(py)?; Ok(LifecycleStep::Done) } LifecycleEvent::Failed { @@ -448,8 +640,28 @@ impl PythonLifecycle for LegacyLogging { } Pending::DeploymentPostCall => { self.response = Some(result?); + self.cache_write(py) + } + Pending::CacheRead => self.cache_read(py, result), + Pending::CacheWrite => { + if let Err(error) = result + && is_cancellation(py, &error) + { + return Err(error); + } self.finalize(py) } + Pending::StreamCacheWrite => { + if let Err(error) = result + && is_cancellation(py, &error) + { + return Err(error); + } + if let Some(stream) = &self.stream { + self.stream_success(py, stream)?; + } + Ok(LifecycleStep::Done) + } Pending::DeploymentFailure => self.dispatch_failure(py), Pending::AsyncFailure => match result { Err(failure) if is_cancellation(py, &failure) => Err(failure), @@ -467,10 +679,14 @@ impl PythonLifecycle for LegacyLogging { self.body = None; self.context = None; self.stream = None; + self.cache = None; } fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { self.call.traverse(visit)?; + if let Some(cache) = &self.cache { + cache.traverse(visit)?; + } if let Some(logger) = &self.logger { logger.traverse(visit)?; } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/caching.rs b/litellm-rust/crates/callbacks-legacy-python/src/caching.rs new file mode 100644 index 00000000000..a2f03f3be7c --- /dev/null +++ b/litellm-rust/crates/callbacks-legacy-python/src/caching.rs @@ -0,0 +1,291 @@ +use litellm_cache::{CacheControls, CacheKeyField, CacheKeyInput, cache_key}; +use litellm_host_python::{from_py, to_py}; +use pyo3::{ + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::PyDict, +}; +use serde_json::Value; + +use crate::PublicCall; + +pub(crate) struct Caching { + backend: Py, + pub key: String, + pub controls: CacheControls, + kwargs: Py, + max_age: Option, +} + +impl Caching { + pub fn prepare( + py: Python<'_>, + call: &PublicCall, + call_type: &str, + asynchronous: bool, + ) -> PyResult> { + let sdk = py.import("litellm")?; + let cache = sdk.getattr("cache")?; + if cache.is_none() { + return Ok(None); + } + let supported = cache.getattr("supported_call_types")?; + if supported.is_none() || !supported.contains(call_type)? { + return Ok(None); + } + let kwargs = call.kwargs().bind(py).copy()?; + let names = [ + "max_tokens", + "messages", + "model", + "metadata", + "stop_sequences", + "stream", + "system", + "temperature", + "thinking", + "tool_choice", + "tools", + "top_k", + "top_p", + "container", + "api_key", + "api_base", + "client", + "custom_llm_provider", + ]; + for (name, value) in names + .iter() + .filter(|name| !asynchronous || **name != "container") + .zip(call.args().bind(py).iter()) + { + kwargs.set_item(name, value)?; + } + let directives = kwargs + .get_item("cache")? + .unwrap_or_else(|| PyDict::new(py).into_any()); + let directive = |name: &str| directives.call_method1("get", (name,)); + let is_true = + |value: Bound<'_, PyAny>| value.is(&true.into_pyobject(py).expect("bool conversion")); + let controls = CacheControls { + supported_call_type: true, + configured: true, + native_backend: false, + default_on: cache.getattr("mode")?.eq("default_on")?, + caching: kwargs + .get_item("caching")? + .filter(|v| !v.is_none()) + .map(is_true), + no_cache: is_true(directive("no-cache")?), + no_store: is_true(directive("no-store")?), + use_cache: is_true(directive("use-cache")?), + }; + if !controls.reads() && !controls.writes() { + return Ok(None); + } + let metadata = kwargs + .get_item("metadata")? + .filter(|value| !value.is_none()) + .unwrap_or_else(|| PyDict::new(py).into_any()); + let params = kwargs + .get_item("litellm_params")? + .filter(|value| !value.is_none()) + .unwrap_or_else(|| PyDict::new(py).into_any()); + let preset = kwargs + .get_item("cache_key")? + .filter(|value| !value.is_none()) + .or_else(|| { + params + .call_method1("get", ("preset_cache_key",)) + .ok() + .filter(|value| !value.is_none()) + }); + let key = if let Some(preset) = preset { + preset.extract()? + } else { + let api_params = py + .import("litellm.litellm_core_utils.model_param_helper")? + .getattr("ModelParamHelper")? + .call_method0("_get_all_llm_api_params")?; + let internal = py + .import("litellm.types.utils")? + .getattr("all_litellm_params")?; + let include_provider_parameters = sdk + .getattr("enable_caching_on_provider_specific_optional_params")? + .is_truthy()?; + let model_group = metadata.call_method1("get", ("model_group",))?; + let nested_metadata = params.call_method1("get", ("metadata", PyDict::new(py)))?; + let model_group = if model_group.is_truthy()? { + model_group + } else if nested_metadata.is_none() { + py.None().into_bound(py) + } else { + nested_metadata.call_method1("get", ("model_group",))? + }; + let groups = metadata.call_method1("get", ("caching_groups",))?; + let mut group = None; + if groups.is_truthy()? { + for item in groups.try_iter()? { + let item = item?; + if item.contains(&model_group)? { + group = Some(item.str()?.to_string()); + break; + } + } + } + let fields = kwargs + .iter() + .map(|(name, value)| { + let name: String = name.extract()?; + let api_parameter = api_params.contains(&name)?; + let internal_parameter = internal.contains(&name)?; + let value = if value.is_none() + || (!api_parameter && (!include_provider_parameters || internal_parameter)) + { + None + } else if name == "model" && group.is_some() { + group.clone() + } else if name == "model" && model_group.is_truthy()? { + Some(model_group.str()?.to_string()) + } else { + Some(value.str()?.to_string()) + }; + Ok(CacheKeyField { + name, + value, + api_parameter, + internal_parameter, + }) + }) + .collect::>>()?; + let namespace = [ + directive("namespace")?, + metadata.call_method1("get", ("redis_namespace",))?, + cache.getattr("namespace")?, + ] + .into_iter() + .find(|value| value.is_truthy().unwrap_or(false)) + .map(|value| value.extract()) + .transpose()?; + cache_key(&CacheKeyInput { + fields, + preset: None, + namespace, + include_provider_parameters, + }) + }; + let backend_kwargs = PyDict::new(py); + for name in [ + "messages", + "input", + "metadata", + "litellm_params", + "parent_otel_span", + ] { + if let Some(value) = kwargs.get_item(name)? { + backend_kwargs.set_item(name, value)?; + } + } + let ttl = directive("ttl")?; + let ttl = if !ttl.is_none() { + Some(ttl) + } else { + let configured = cache.getattr("ttl")?; + if configured.is_none() { + kwargs.get_item("ttl")? + } else { + Some(configured) + } + }; + if let Some(ttl) = ttl { + backend_kwargs.set_item("ttl", ttl)?; + } + let max_age = if asynchronous { + directives.call_method1("get", ("s-max-age", directive("s-maxage")?))? + } else { + let first = directive("s-maxage")?; + if first.is_truthy()? { + first + } else { + directive("s-max-age")? + } + }; + Ok(Some(Self { + backend: cache.getattr("cache")?.unbind(), + key, + controls, + kwargs: backend_kwargs.unbind(), + max_age: max_age.extract()?, + })) + } + + pub fn read(&self, py: Python<'_>, asynchronous: bool) -> PyResult> { + self.backend + .bind(py) + .call_method( + if asynchronous { + "async_get_cache" + } else { + "get_cache" + }, + (&self.key,), + Some(self.kwargs.bind(py)), + ) + .map(Bound::unbind) + } + + pub fn decode(&self, value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + let value: Value = from_py(value)?; + let response = if let Some(timestamp) = value.get("timestamp").and_then(Value::as_f64) { + if self + .max_age + .is_some_and(|max| litellm_host::event::epoch_seconds() - timestamp > max) + { + return Ok(None); + } + value.get("response").cloned().unwrap_or(Value::Null) + } else { + value + }; + let response = match response { + Value::String(text) => serde_json::from_str(&text).unwrap_or(Value::Null), + value => value, + }; + Ok(response.is_object().then_some(response)) + } + + pub fn write( + &self, + py: Python<'_>, + response: &Bound<'_, PyAny>, + asynchronous: bool, + ) -> PyResult> { + let entry = PyDict::new(py); + entry.set_item("timestamp", litellm_host::event::epoch_seconds())?; + entry.set_item("response", response)?; + self.backend + .bind(py) + .call_method( + if asynchronous { + "async_set_cache" + } else { + "set_cache" + }, + (&self.key, entry), + Some(self.kwargs.bind(py)), + ) + .map(Bound::unbind) + } + + pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.backend)?; + visit.call(&self.kwargs) + } +} + +pub(crate) fn cached_public(py: Python<'_>, value: &Value) -> PyResult> { + to_py(py, value) +} diff --git a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs index 44393792d1f..2f7071d0c64 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs @@ -10,9 +10,11 @@ //! without keeping a copy. mod adapter; +mod caching; mod call; mod callbacks; mod deferred; +mod limits; mod logger; mod preparation; mod python; diff --git a/litellm-rust/crates/callbacks-legacy-python/src/limits.rs b/litellm-rust/crates/callbacks-legacy-python/src/limits.rs new file mode 100644 index 00000000000..52df8cb1e36 --- /dev/null +++ b/litellm-rust/crates/callbacks-legacy-python/src/limits.rs @@ -0,0 +1,107 @@ +use std::sync::OnceLock; + +use litellm_core_utils::token_counter::get_modified_max_tokens; +use litellm_host_python::from_py; +use litellm_token_counter::{CountableRequest, TokenCounter}; +use pyo3::{prelude::*, types::PyDict}; +use serde_json::Value; + +use crate::PublicCall; + +pub(crate) fn adjust_max_tokens( + py: Python<'_>, + call: &PublicCall, + call_type: &str, +) -> PyResult<()> { + if !matches!( + call_type, + "completion" | "acompletion" | "anthropic_messages" + ) { + return Ok(()); + } + let sdk = py.import("litellm")?; + if !sdk.getattr("modify_params")?.is(&true.into_pyobject(py)?) { + return Ok(()); + } + let Some(requested) = call + .lookup(py, "max_tokens")? + .filter(|value| !value.is_none()) + else { + return Ok(()); + }; + let requested: i64 = requested.extract()?; + let Some(model) = call.lookup(py, "model")? else { + return Ok(()); + }; + let model: String = model.extract()?; + let info = model_info(py, &model)?; + let Some(info) = info else { + return Ok(()); + }; + let counter = counter(py, &model)?; + let messages = if call.args().bind(py).len() > 1 { + call.args().bind(py).get_item(1)? + } else { + call.lookup(py, "messages")? + .unwrap_or_else(|| py.None().into_bound(py)) + }; + let request = PyDict::new(py); + request.set_item("model", &model)?; + request.set_item("messages", messages)?; + let request: CountableRequest = from_py(&request)?; + let count = py + .detach(|| counter.count_request(&request)) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + let adjusted = get_modified_max_tokens( + requested, + info.get("max_input_tokens").and_then(Value::as_i64), + info.get("max_output_tokens") + .or_else(|| info.get("max_tokens")) + .and_then(Value::as_i64), + count.input_tokens, + None, + None, + ); + call.kwargs().bind(py).set_item("max_tokens", adjusted) +} + +pub(crate) fn model_info(py: Python<'_>, model: &str) -> PyResult> { + let sdk = py.import("litellm")?; + let costs = sdk.getattr("model_cost")?; + let aliases = sdk.getattr("model_alias_map")?; + let resolved = aliases.call_method1("get", (model, model))?; + let resolved: String = resolved.extract()?; + let stripped = resolved.strip_prefix("anthropic/").unwrap_or(&resolved); + for key in [&resolved, stripped, &format!("anthropic/{stripped}")] { + let value = costs.call_method1("get", (key,))?; + if !value.is_none() { + return from_py(&value).map(Some); + } + } + Ok(None) +} + +fn counter(py: Python<'_>, model: &str) -> PyResult<&'static TokenCounter> { + static CLAUDE: OnceLock = OnceLock::new(); + static CL100K: OnceLock = OnceLock::new(); + let sdk = py.import("litellm")?; + let anthropic = sdk.getattr("anthropic_models")?.contains(model)? + && !model.contains("claude-3") + && !sdk.getattr("disable_hf_tokenizer_download")?.is_truthy()?; + let cell = if anthropic { &CLAUDE } else { &CL100K }; + if let Some(counter) = cell.get() { + return Ok(counter); + } + let loaded = if anthropic { + let json: String = py + .import("litellm.utils")? + .getattr("claude_json_str")? + .extract()?; + py.detach(|| TokenCounter::from_json(&json)) + } else { + py.detach(|| TokenCounter::from_tiktoken("cl100k_base")) + } + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + let _ = cell.set(loaded); + cell.get().ok_or_else(litellm_host_python::missing_state) +} diff --git a/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs b/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs index aab654c9893..83d13b22030 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/preparation.rs @@ -29,10 +29,60 @@ pub fn prepare<'py>( .call(py, ())? .cast_into::()?) })?; - Wrapper::CheckLimits.call(py, (&arguments,))?; + check_limits(py, &arguments)?; Ok(arguments) } +fn check_limits(py: Python<'_>, kwargs: &Bound<'_, PyDict>) -> PyResult<()> { + let sdk = py.import("litellm")?; + check_limits_with_settings(py, kwargs, sdk.as_any()) +} + +fn check_limits_with_settings( + py: Python<'_>, + kwargs: &Bound<'_, PyDict>, + sdk: &Bound<'_, PyAny>, +) -> PyResult<()> { + let max_budget = sdk.getattr("max_budget")?.extract::>()?; + if max_budget.is_some_and(|limit| limit != 0.0) { + let current_cost = sdk.getattr("_current_cost")?.extract::()?; + if let Err(error) = litellm_core_utils::budget::check_budget(current_cost, max_budget) { + let arguments = PyDict::new(py); + arguments.set_item("current_cost", error.current_cost)?; + arguments.set_item("max_budget", error.max_budget)?; + return Err(PyErr::from_value( + sdk.getattr("BudgetExceededError")? + .call((), Some(&arguments))?, + )); + } + } + let limit = sdk.getattr("num_retries_per_request")?; + if limit.is_none() { + return Ok(()); + } + let key = if kwargs.contains("litellm_metadata")? { + "litellm_metadata" + } else { + "metadata" + }; + let Some(metadata) = kwargs.get_item(key)? else { + return Ok(()); + }; + if !metadata.is_instance(&py.import("collections.abc")?.getattr("Mapping")?)? { + return Ok(()); + } + let retry = metadata.call_method1("get", ("request_retry_count",))?; + if retry.get_type().is(&py.get_type::()) + && retry.gt(0)? + && retry.ge(limit)? + { + return Err(pyo3::exceptions::PyRuntimeError::new_err( + "Max retries per request hit!", + )); + } + Ok(()) +} + fn inherit_credentials<'py>( py: Python<'py>, arguments: &Bound<'py, PyDict>, @@ -73,6 +123,58 @@ fn inherit_credentials<'py>( mod tests { use super::*; + #[test] + fn native_limits_preserve_budget_and_retry_boundaries() { + Python::initialize(); + Python::attach(|py| { + let locals = eval( + py, + c" +import types +class BudgetExceededError(Exception): + def __init__(self, current_cost, max_budget): + self.current_cost = current_cost + self.max_budget = max_budget +settings = types.SimpleNamespace(max_budget=2.0, _current_cost=2.0, num_retries_per_request=5, + BudgetExceededError=BudgetExceededError) +", + ); + let settings = locals.get_item("settings").unwrap().unwrap(); + let arguments = PyDict::new(py); + assert!(check_limits_with_settings(py, &arguments, &settings).is_ok()); + settings.setattr("_current_cost", 3.0).unwrap(); + let error = check_limits_with_settings(py, &arguments, &settings).unwrap_err(); + assert_eq!( + error + .value(py) + .getattr("current_cost") + .unwrap() + .extract::() + .unwrap(), + 3.0 + ); + settings.setattr("max_budget", 0.0).unwrap(); + for key in ["metadata", "litellm_metadata"] { + for (count, refused) in [(0, false), (4, false), (5, true), (6, true)] { + let metadata = PyDict::new(py); + metadata.set_item("request_retry_count", count).unwrap(); + arguments.set_item(key, metadata).unwrap(); + assert_eq!( + check_limits_with_settings(py, &arguments, &settings).is_err(), + refused + ); + } + let metadata = PyDict::new(py); + metadata.set_item("request_retry_count", true).unwrap(); + settings.setattr("num_retries_per_request", 0).unwrap(); + arguments.set_item(key, metadata).unwrap(); + assert!(check_limits_with_settings(py, &arguments, &settings).is_ok()); + arguments.del_item(key).unwrap(); + settings.setattr("num_retries_per_request", 5).unwrap(); + } + }); + } + fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { let locals = PyDict::new(py); py.run(source, Some(&locals), Some(&locals)).unwrap(); diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index cb609d52878..f2be700b2ee 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -25,8 +25,6 @@ pub(crate) enum LegacyPython { pub(crate) enum Wrapper { #[strum(serialize = "setup")] Setup, - #[strum(serialize = "check_limits")] - CheckLimits, #[strum(serialize = "credential_list")] CredentialList, #[strum(serialize = "warn_unknown_credential")] diff --git a/litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs b/litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs index 52c5e47f83f..d4bfb50fbde 100644 --- a/litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs +++ b/litellm-rust/crates/callbacks-legacy-python/tests/deployment_hooks.rs @@ -90,8 +90,6 @@ assert prepared['document'] is replacement assert prepared['pages'] is replaced_kwargs['pages'] assert prepared['litellm_logging_obj'] is logger assert 'litellm_logging_obj' not in replaced_kwargs -[checked] = [value for name, value in logger.calls if name == 'check_limits'] -assert checked is prepared ", ); }); @@ -126,8 +124,6 @@ kwargs = {'logger': logger, 'vendor_extension': opaque} &locals, c" assert prepared['vendor_extension'] is opaque -[checked] = [value for name, value in logger.calls if name == 'check_limits'] -assert checked['vendor_extension'] is opaque assert hooked == ([opaque] if asynchronous else []), hooked ", ); @@ -247,7 +243,7 @@ assert all(value is failure for name, value in logger.calls if name.endswith('_h #[rstest] #[case::synchronous(false)] #[case::asynchronous(true)] -fn a_limit_rejected_before_the_call_surfaces_as_the_callers_error(#[case] asynchronous: bool) { +fn preparation_failure_before_the_call_preserves_exception_identity(#[case] asynchronous: bool) { Python::initialize(); Python::attach(|py| { let locals = namespace( @@ -258,13 +254,13 @@ class BudgetExceeded(Exception): rejection = BudgetExceeded('over budget') -class LimitedLogger(StubLogger): - def check_limits(self, arguments): +class RejectedCredential(str): + def __bool__(self): raise rejection -logger = LimitedLogger() +logger = StubLogger() logger.hooks = {'pre': lambda kwargs: kwargs} -kwargs = {'logger': logger} +kwargs = {'logger': logger, 'litellm_credential_name': RejectedCredential('named')} ", ); let mut logging = legacy_call(py, &locals, asynchronous); diff --git a/litellm-rust/crates/callbacks-legacy-python/tests/support.rs b/litellm-rust/crates/callbacks-legacy-python/tests/support.rs index d0c02fa6da5..cf7334a8b16 100644 --- a/litellm-rust/crates/callbacks-legacy-python/tests/support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/tests/support.rs @@ -27,6 +27,9 @@ for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_le sys.modules.setdefault(name, types.ModuleType(name)) legacy = sys.modules['litellm.rust_bridge.callbacks_legacy_python'] +sdk = sys.modules['litellm'] +sdk.max_budget = None +sdk.num_retries_per_request = None CONTRACT = json.loads(python_contract) @@ -50,7 +53,6 @@ FAKES = { logger=kwargs['logger_factory'](kwargs) if 'logger_factory' in kwargs else kwargs['logger'], kwargs=kwargs, ), - 'check_limits': lambda arguments: arguments['logger'].check_limits(arguments), 'finalize': lambda response, logger, kwargs, start, end: logger.record('finalize', response), 'update_logging': lambda logger, kwargs, model, optional_params, litellm_params, provider: logger.update_from_kwargs( kwargs=kwargs, diff --git a/litellm-rust/crates/core-utils/src/budget.rs b/litellm-rust/crates/core-utils/src/budget.rs new file mode 100644 index 00000000000..87292368215 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/budget.rs @@ -0,0 +1,82 @@ +use std::sync::Mutex; + +#[derive(Clone, Copy, Debug, PartialEq, thiserror::Error)] +#[error("Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}")] +pub struct BudgetExceeded { + pub current_cost: f64, + pub max_budget: f64, +} + +pub fn check_budget(current_cost: f64, max_budget: Option) -> Result<(), BudgetExceeded> { + if let Some(max_budget) = max_budget.filter(|limit| *limit != 0.0) + && current_cost > max_budget + { + return Err(BudgetExceeded { + current_cost, + max_budget, + }); + } + Ok(()) +} + +#[derive(Default)] +pub struct Budget { + current_cost: Mutex, + pub max_budget: Option, +} + +impl Budget { + pub fn new(max_budget: Option, current_cost: f64) -> Self { + Self { + current_cost: Mutex::new(current_cost), + max_budget, + } + } + + pub fn current_cost(&self) -> f64 { + *self + .current_cost + .lock() + .unwrap_or_else(|error| error.into_inner()) + } + + pub fn check(&self) -> Result<(), BudgetExceeded> { + check_budget(self.current_cost(), self.max_budget) + } + + pub fn record(&self, cost: f64) { + *self + .current_cost + .lock() + .unwrap_or_else(|error| error.into_inner()) += cost; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn matches_python_truthiness_and_strict_boundary() { + assert!(check_budget(100.0, None).is_ok()); + assert!(check_budget(100.0, Some(0.0)).is_ok()); + assert!(check_budget(1.0, Some(1.0)).is_ok()); + assert_eq!(check_budget(1.5, Some(1.0)).unwrap_err().current_cost, 1.5); + } + + #[test] + fn concurrent_completions_do_not_lose_spend() { + let budget = Budget::new(Some(10.0), 0.0); + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + for _ in 0..100 { + budget.record(0.25); + } + }); + } + }); + assert_eq!(budget.current_cost(), 200.0); + assert!(budget.check().is_err()); + } +} diff --git a/litellm-rust/crates/core-utils/src/cost_calculator.rs b/litellm-rust/crates/core-utils/src/cost_calculator.rs new file mode 100644 index 00000000000..1043754355a --- /dev/null +++ b/litellm-rust/crates/core-utils/src/cost_calculator.rs @@ -0,0 +1,156 @@ +use serde_json::Value; + +#[derive(Clone, Debug, Default)] +pub struct TokenPrices { + pub input: f64, + pub output: f64, + pub cache_read: f64, + pub cache_creation: f64, + pub cache_creation_1hr: f64, +} + +impl TokenPrices { + pub fn from_model_info(info: &Value, usage: &Value) -> Option { + let input = info.get("input_cost_per_token")?.as_f64()?; + let output = info.get("output_cost_per_token")?.as_f64()?; + let total_input = [ + "input_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ] + .iter() + .map(|key| count(usage, key)) + .sum::(); + let threshold = info + .as_object()? + .keys() + .filter_map(|key| { + let threshold = key + .strip_prefix("input_cost_per_token_above_")? + .strip_suffix("_tokens")?; + let number = threshold + .strip_suffix('k') + .unwrap_or(threshold) + .parse::() + .ok()? + * if threshold.ends_with('k') { + 1000.0 + } else { + 1.0 + }; + (total_input > number).then_some((number, threshold)) + }) + .max_by(|a, b| a.0.total_cmp(&b.0)) + .map(|(_, suffix)| suffix); + let tier = usage + .get("service_tier") + .and_then(Value::as_str) + .filter(|tier| matches!(*tier, "priority" | "flex")); + let rate = |key: &str| -> Option { + let base = tier + .and_then(|tier| info.get(format!("{key}_{tier}")).and_then(Value::as_f64)) + .or_else(|| info.get(key).and_then(Value::as_f64)); + threshold + .and_then(|threshold| { + let key = format!("{key}_above_{threshold}_tokens"); + tier.and_then(|tier| info.get(format!("{key}_{tier}")).and_then(Value::as_f64)) + .or_else(|| info.get(key).and_then(Value::as_f64)) + }) + .or(base) + }; + let input = rate("input_cost_per_token").unwrap_or(input); + let creation = rate("cache_creation_input_token_cost").unwrap_or(input); + Some(Self { + input, + output: rate("output_cost_per_token").unwrap_or(output), + cache_read: rate("cache_read_input_token_cost").unwrap_or(input), + cache_creation: creation, + cache_creation_1hr: rate("cache_creation_input_token_cost_above_1hr") + .unwrap_or(creation), + }) + } + + pub fn messages_cost(&self, usage: &Value) -> f64 { + let creation = match usage + .get("cache_creation") + .filter(|value| value.is_object()) + { + Some(details) => { + count(details, "ephemeral_5m_input_tokens") * self.cache_creation + + count(details, "ephemeral_1h_input_tokens") * self.cache_creation_1hr + } + None => count(usage, "cache_creation_input_tokens") * self.cache_creation, + }; + count(usage, "input_tokens") * self.input + + count(usage, "output_tokens") * self.output + + count(usage, "cache_read_input_tokens") * self.cache_read + + creation + } +} + +fn count(value: &Value, key: &str) -> f64 { + value.get(key).and_then(Value::as_f64).unwrap_or_default() +} + +pub fn messages_cost(info: &Value, usage: &Value) -> Option { + let prices = TokenPrices::from_model_info(info, usage)?; + let provider = info.get("provider_specific_entry").unwrap_or(&Value::Null); + let speed = if usage.get("speed").and_then(Value::as_str) == Some("fast") { + provider.get("fast").and_then(Value::as_f64).unwrap_or(1.0) + } else { + 1.0 + }; + let geo = usage + .get("inference_geo") + .and_then(Value::as_str) + .and_then(|geo| provider.get(geo)) + .and_then(Value::as_f64) + .unwrap_or(1.0); + let searches = usage + .pointer("/server_tool_use/web_search_requests") + .and_then(Value::as_f64) + .unwrap_or_default(); + let search_rate = info + .pointer("/search_context_cost_per_query/search_context_size_medium") + .and_then(Value::as_f64) + .unwrap_or_default(); + Some(prices.messages_cost(usage) * speed * geo + searches * search_rate) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn accounts_for_each_input_category_once_and_preserves_zero_prices() { + let info = json!({"input_cost_per_token": 2.0, "output_cost_per_token": 3.0, + "cache_read_input_token_cost": 0.0, "cache_creation_input_token_cost": 4.0, + "cache_creation_input_token_cost_above_1hr": 5.0}); + let usage = json!({"input_tokens": 10, "output_tokens": 3, + "cache_read_input_tokens": 100, "cache_creation_input_tokens": 7, + "cache_creation": {"ephemeral_5m_input_tokens": 2, "ephemeral_1h_input_tokens": 5}}); + assert_eq!(messages_cost(&info, &usage), Some(62.0)); + } + + #[test] + fn threshold_uses_total_input_and_is_strict() { + let info = json!({"input_cost_per_token": 1.0, "output_cost_per_token": 2.0, + "input_cost_per_token_above_100_tokens": 4.0}); + assert_eq!( + messages_cost(&info, &json!({"input_tokens": 100})), + Some(100.0) + ); + assert_eq!( + messages_cost( + &info, + &json!({"input_tokens": 100, "cache_read_input_tokens": 1}) + ), + Some(404.0) + ); + assert_eq!( + messages_cost(&json!({}), &json!({"input_tokens": 100})), + None + ); + } +} diff --git a/litellm-rust/crates/core-utils/src/lib.rs b/litellm-rust/crates/core-utils/src/lib.rs index ceb0e9eb3f2..d965b49bd45 100644 --- a/litellm-rust/crates/core-utils/src/lib.rs +++ b/litellm-rust/crates/core-utils/src/lib.rs @@ -1,10 +1,14 @@ +pub mod budget; pub mod call_arguments; pub mod core_helpers; +pub mod cost_calculator; pub mod exception_mapping_utils; pub mod get_llm_provider_logic; +pub mod messages_stream; pub mod params; pub mod prompt_templates; pub mod secret_redaction; pub mod serde_compat; pub mod settings; +pub mod token_counter; pub mod url_utils; diff --git a/litellm-rust/crates/core-utils/src/messages_stream.rs b/litellm-rust/crates/core-utils/src/messages_stream.rs new file mode 100644 index 00000000000..9567f358ef4 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/messages_stream.rs @@ -0,0 +1,40 @@ +use serde_json::{Map, Value}; + +pub struct MessagesStream { + pub usage: Value, + pub cached_response: Option, +} + +impl MessagesStream { + pub fn parse(bytes: &[u8]) -> Option { + let text = std::str::from_utf8(bytes).ok()?; + let mut usage = Map::new(); + let mut stopped = false; + let mut failed = false; + let events: Vec<_> = text.split_inclusive("\n\n").collect(); + for event in &events { + for line in event.lines().filter_map(|line| line.strip_prefix("data:")) { + let Ok(value) = serde_json::from_str::(line.trim()) else { + continue; + }; + match value.get("type").and_then(Value::as_str) { + Some("message_stop") => stopped = true, + Some("error") => failed = true, + _ => {} + } + if let Some(fields) = value + .pointer("/message/usage") + .or_else(|| value.get("usage")) + .and_then(Value::as_object) + { + usage.extend(fields.clone()); + } + } + } + Some(Self { + usage: Value::Object(usage), + cached_response: (stopped && !failed) + .then(|| serde_json::json!({"litellm_cached_anthropic_sse_events": events})), + }) + } +} diff --git a/litellm-rust/crates/core-utils/src/token_counter.rs b/litellm-rust/crates/core-utils/src/token_counter.rs new file mode 100644 index 00000000000..b1242beed9b --- /dev/null +++ b/litellm-rust/crates/core-utils/src/token_counter.rs @@ -0,0 +1,55 @@ +pub fn get_modified_max_tokens( + user_max_tokens: i64, + max_input_tokens: Option, + max_output_tokens: Option, + input_tokens: usize, + buffer_perc: Option, + buffer_num: Option, +) -> i64 { + let Some(output) = max_output_tokens else { + return user_max_tokens; + }; + let buffer = + (buffer_perc.unwrap_or(0.1) * input_tokens as f64).max(buffer_num.unwrap_or(10.0)) as i64; + let input = (input_tokens as i64).saturating_add(buffer); + if max_input_tokens == Some(output) { + if input <= output && user_max_tokens.saturating_add(input) > output { + return output - input; + } + return user_max_tokens; + } + user_max_tokens.min(output) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shared_context_includes_buffer_and_does_not_hide_overflow() { + assert_eq!( + get_modified_max_tokens(80, Some(100), Some(100), 40, None, None), + 50 + ); + assert_eq!( + get_modified_max_tokens(80, Some(100), Some(100), 100, None, None), + 80 + ); + assert_eq!( + get_modified_max_tokens(80, Some(100), Some(100), 90, None, None), + 0 + ); + assert_eq!( + get_modified_max_tokens(80, Some(1000), Some(100), 900, None, None), + 80 + ); + assert_eq!( + get_modified_max_tokens(180, Some(1000), Some(100), 900, None, None), + 100 + ); + assert_eq!( + get_modified_max_tokens(180, None, None, 900, None, None), + 180 + ); + } +} diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 69ae8004d46..fa3d9f63787 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -9,6 +9,8 @@ autotests = false [dependencies] litellm-types.workspace = true litellm-core-utils.workspace = true +litellm-cache.workspace = true +litellm-token-counter.workspace = true litellm-host.workspace = true bytes.workspace = true futures-util.workspace = true @@ -36,6 +38,7 @@ url.workspace = true veil.workspace = true [dev-dependencies] +litellm-cache-memory.workspace = true litellm-auth-gcp.workspace = true litellm-llms = { workspace = true, features = ["test-support"] } rstest.workspace = true diff --git a/litellm-rust/crates/core/src/client.rs b/litellm-rust/crates/core/src/client.rs new file mode 100644 index 00000000000..cee9535f1e5 --- /dev/null +++ b/litellm-rust/crates/core/src/client.rs @@ -0,0 +1,67 @@ +use std::sync::Arc; + +use litellm_cache::LLMCachingHandler; +use litellm_core_utils::{ + budget::Budget, cost_calculator::messages_cost, token_counter::get_modified_max_tokens, +}; +use litellm_token_counter::{CountableRequest, TokenCounter}; +use serde_json::{Map, Value}; + +#[derive(Clone, Default)] +pub struct ClientOptions { + pub cache: Option, + pub budget: Option>, + pub model_info: Option, + pub token_counter: Option>, + pub modify_params: bool, +} + +impl ClientOptions { + pub fn adjust_max_tokens(&self, body: &mut Map) { + if !self.modify_params { + return; + } + let (Some(counter), Some(info), Some(requested)) = ( + &self.token_counter, + &self.model_info, + body.get("max_tokens").and_then(Value::as_i64), + ) else { + return; + }; + let request = + serde_json::json!({"model": body.get("model"), "messages": body.get("messages")}); + let Ok(request) = serde_json::from_value::(request) else { + return; + }; + let Ok(count) = counter.count_request(&request) else { + return; + }; + let output = info + .get("max_output_tokens") + .or_else(|| info.get("max_tokens")) + .and_then(Value::as_i64); + let adjusted = get_modified_max_tokens( + requested, + info.get("max_input_tokens").and_then(Value::as_i64), + output, + count.input_tokens, + None, + None, + ); + body.insert("max_tokens".into(), Value::from(adjusted)); + } + + pub fn record_messages_usage(&self, usage: &Value, cache_hit: bool) -> Option { + let cost = if cache_hit { + Some(0.0) + } else { + self.model_info + .as_ref() + .and_then(|info| messages_cost(info, usage)) + }; + if !cache_hit && let (Some(budget), Some(cost)) = (&self.budget, cost) { + budget.record(cost); + } + cost + } +} diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index afe5ea595aa..42135724854 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -1,5 +1,6 @@ pub mod audio_transcription; pub mod chat_completions; +pub mod client; pub mod constants; pub mod error; pub mod messages; diff --git a/litellm-rust/crates/core/src/messages/error.rs b/litellm-rust/crates/core/src/messages/error.rs index 51fb764032c..48e85d0b5ce 100644 --- a/litellm-rust/crates/core/src/messages/error.rs +++ b/litellm-rust/crates/core/src/messages/error.rs @@ -1,7 +1,9 @@ use litellm_llms::base_llm::chat::transformation::Error as LlmError; -#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[derive(Clone, Debug, PartialEq, thiserror::Error)] pub enum Error { + #[error(transparent)] + BudgetExceeded(#[from] litellm_core_utils::budget::BudgetExceeded), #[error("invalid provider: {0}")] InvalidProvider(String), #[error("missing required field: {0}")] diff --git a/litellm-rust/crates/core/src/messages/lifecycle_tests.rs b/litellm-rust/crates/core/src/messages/lifecycle_tests.rs new file mode 100644 index 00000000000..bcdd32eaad2 --- /dev/null +++ b/litellm-rust/crates/core/src/messages/lifecycle_tests.rs @@ -0,0 +1,147 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_cache::{CacheControls, CacheKwargs, LLMCachingHandler}; +use litellm_cache_memory::InMemoryCache; +use litellm_core_utils::budget::Budget; +use litellm_token_counter::{TokenCounter, Tokenizer}; +use serde_json::{Value, json}; +use tokio::{io::AsyncWriteExt, net::TcpListener}; + +use crate::{ + client::ClientOptions, + messages::{Error, messages_with_options, types::MessagesRequest}, +}; + +fn options() -> ClientOptions { + ClientOptions { + cache: Some(LLMCachingHandler { + backend: Arc::new(InMemoryCache::default()), + controls: CacheControls { + supported_call_type: true, + configured: true, + native_backend: true, + default_on: true, + ..Default::default() + }, + key: "test-key".into(), + kwargs: CacheKwargs::default(), + max_age: None, + }), + budget: Some(Arc::new(Budget::new(Some(100.0), 0.0))), + model_info: Some( + json!({"input_cost_per_token": 2.0, "output_cost_per_token": 3.0, + "cache_read_input_token_cost": 0.5, "cache_creation_input_token_cost": 4.0}), + ), + ..Default::default() + } +} + +fn request(url: &str) -> MessagesRequest<'_> { + MessagesRequest { + model: "anthropic/test-model", + body: json!({"model":"anthropic/test-model", "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}]}), + api_key: Some("test"), + api_base: Some(url), + custom_llm_provider: Some("anthropic"), + extra_headers: None, + timeout: Some(Duration::from_secs(2)), + } +} + +async fn start_server() -> (String, tokio::task::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let request = super::tests::read_http_request(&mut socket).await; + let body = json!({"id":"msg_test", "type":"message", "role":"assistant", "model":"test-model", + "content":[{"type":"text","text":"hello"}], "stop_reason":"end_turn", "stop_sequence":null, + "usage":{"input_tokens":2,"output_tokens":3,"cache_read_input_tokens":4,"cache_creation_input_tokens":5}}); + socket + .write_all(super::tests::write_response(&body.to_string()).as_bytes()) + .await + .unwrap(); + serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() + }); + (url, server) +} + +#[tokio::test] +async fn native_messages_cache_hit_skips_network_and_does_not_double_charge() { + let options = options(); + let (url, server) = start_server().await; + let first = messages_with_options(request(&url), options.clone()) + .await + .unwrap(); + server.await.unwrap(); + let cost = 2.0 * 2.0 + 3.0 * 3.0 + 4.0 * 0.5 + 5.0 * 4.0; + assert_eq!(options.budget.as_ref().unwrap().current_cost(), cost); + let cached = messages_with_options(request(&url), options.clone()) + .await + .unwrap(); + assert_eq!(first, cached); + assert_eq!(options.budget.as_ref().unwrap().current_cost(), cost); + + let exceeded = ClientOptions { + budget: Some(Arc::new(Budget::new(Some(cost - 1.0), cost))), + ..options + }; + assert!(matches!( + messages_with_options(request(&url), exceeded).await, + Err(Error::BudgetExceeded(_)) + )); +} + +#[tokio::test] +async fn no_store_and_no_cache_control_different_operations() { + let mut options = options(); + options.cache.as_mut().unwrap().controls.no_store = true; + let (url, server) = start_server().await; + messages_with_options(request(&url), options.clone()) + .await + .unwrap(); + server.await.unwrap(); + assert!(options.cache.as_ref().unwrap().get().await.is_none()); + + options.cache.as_mut().unwrap().controls.no_store = false; + options + .cache + .as_ref() + .unwrap() + .set(json!({"id":"sentinel"})) + .await; + options.cache.as_mut().unwrap().controls.no_cache = true; + assert!(options.cache.as_ref().unwrap().get().await.is_none()); + let (url, server) = start_server().await; + let response = messages_with_options(request(&url), options.clone()) + .await + .unwrap(); + server.await.unwrap(); + options.cache.as_mut().unwrap().controls.no_cache = false; + assert_eq!( + options.cache.as_ref().unwrap().get().await.unwrap()["id"], + response.id + ); +} + +struct Characters; +impl Tokenizer for Characters { + fn count_tokens(&self, text: &str) -> Result { + Ok(text.chars().count()) + } +} + +#[tokio::test] +async fn opt_in_token_adjustment_reaches_the_wire_with_a_standard_counter_interface() { + let (url, server) = start_server().await; + let options = ClientOptions { + token_counter: Some(Arc::new(TokenCounter::new(Characters))), + modify_params: true, + model_info: Some(json!({"max_input_tokens": 40, "max_output_tokens": 40})), + ..Default::default() + }; + messages_with_options(request(&url), options).await.unwrap(); + let sent = server.await.unwrap(); + assert_eq!(sent["max_tokens"], 40 - (3 + 4 + 2 + 3) - 10); +} diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index 289f79109dd..e9e06371c0a 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -19,12 +19,21 @@ use serde_json::Value; use crate::messages::types::MessagesRequest; pub async fn messages(request: MessagesRequest<'_>) -> Result { + messages_with_options(request, crate::client::ClientOptions::default()).await +} + +pub async fn messages_with_options( + request: MessagesRequest<'_>, + options: crate::client::ClientOptions, +) -> Result { let Value::Object(body) = request.body else { return Err(Error::InvalidRequest( "messages body must be an object".into(), )); }; let call = MessagesCall { + cached_response: None, + options, model: request.model.into(), body, api_key: request.api_key.map(Into::into), @@ -41,5 +50,7 @@ pub async fn messages(request: MessagesRequest<'_>) -> Result, + pub options: crate::client::ClientOptions, pub model: String, pub body: Map, pub api_key: Option, @@ -125,8 +127,48 @@ pub fn messages_machine() -> MessagesMachine { } async fn execute(host: MessagesHost) -> Result { - let MessagesOpResult::Request(call) = host.route(MessagesOp::ProjectRequest).await?; + let MessagesOpResult::Request(mut call) = host.route(MessagesOp::ProjectRequest).await?; + if let Some(budget) = &call.options.budget { + budget.check()?; + } let stream = call.streams(); + let cached = match call.cached_response.take() { + Some(response) => Some(response), + None => match &call.options.cache { + Some(cache) => cache.get().await, + None => None, + }, + }; + if let Some(response) = cached { + if !stream + && let Ok(message) = + serde_json::from_value::(response.clone()) + { + call.options + .record_messages_usage(message.usage.as_ref().unwrap_or(&Value::Null), true); + return Ok(MessagesOutput::Message(Box::new(message))); + } + if stream + && let Some(events) = response + .get("litellm_cached_anthropic_sse_events") + .and_then(Value::as_array) + && events.iter().all(Value::is_string) + { + if host.open(()).await? == Demand::More { + for event in events { + if host + .deliver(Bytes::from(event.as_str().unwrap_or_default().to_owned())) + .await? + == Demand::Detached + { + break; + } + } + } + return Ok(MessagesOutput::Streamed); + } + } + call.options.adjust_max_tokens(&mut call.body); let request = prepare_provider_request(MessagesRequest { model: &call.model, body: Value::Object(call.body.clone()), @@ -167,15 +209,22 @@ async fn execute(host: MessagesHost) -> Result { return Err(provider_error(response).await); } if stream { - return relay(&host, response).await; + return relay(&host, response, &call.options).await; } let text = response.text().await.map_err(network)?; host.emit(MachineEvent::ResponseReceived { raw: RawResponse { body: text.clone() }, }) .await?; - decode_response(request.config, &request.model, &text) - .map(|message| MessagesOutput::Message(Box::new(message))) + let message = decode_response(request.config, &request.model, &text)?; + call.options + .record_messages_usage(message.usage.as_ref().unwrap_or(&Value::Null), false); + if let Some(cache) = &call.options.cache + && let Ok(value) = serde_json::to_value(&message) + { + cache.set(value).await; + } + Ok(MessagesOutput::Message(Box::new(message))) } /// Hands each upstream chunk to the caller as it arrives. A caller that stops reading @@ -183,14 +232,26 @@ async fn execute(host: MessagesHost) -> Result { async fn relay( host: &MessagesHost, mut response: reqwest::Response, + options: &crate::client::ClientOptions, ) -> Result { if host.open(()).await? == Demand::Detached { return Ok(MessagesOutput::Streamed); } + let mut collected = Vec::new(); + let mut detached = false; while let Some(chunk) = response.chunk().await.map_err(network)? { + collected.extend_from_slice(&chunk); if host.deliver(chunk).await? == Demand::Detached { + detached = true; break; } } + if let Some(stream) = litellm_core_utils::messages_stream::MessagesStream::parse(&collected) { + options.record_messages_usage(&stream.usage, false); + if !detached && let (Some(cache), Some(response)) = (&options.cache, stream.cached_response) + { + cache.set(response).await; + } + } Ok(MessagesOutput::Streamed) } diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index 057b42a316c..29280969c5d 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -15,7 +15,7 @@ use super::{ }; use crate::messages::types::MessagesRequest; -async fn read_http_request(socket: &mut TcpStream) -> String { +pub(super) async fn read_http_request(socket: &mut TcpStream) -> String { let mut request = Vec::new(); let mut buffer = [0_u8; 1024]; let header_end = loop { @@ -48,7 +48,7 @@ async fn read_http_request(socket: &mut TcpStream) -> String { String::from_utf8(request).expect("request is utf8") } -fn write_response(body: &str) -> String { +pub(super) fn write_response(body: &str) -> String { format!( "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", body.len(), diff --git a/litellm-rust/crates/host-python/src/adapter.rs b/litellm-rust/crates/host-python/src/adapter.rs index 3a4cb49be4d..68b12eddab2 100644 --- a/litellm-rust/crates/host-python/src/adapter.rs +++ b/litellm-rust/crates/host-python/src/adapter.rs @@ -14,6 +14,7 @@ pub fn missing_state() -> PyErr { pub enum LifecycleStep { Await(Py), Arguments(Py), + Cached(Py), Wire(Box), Response(Py), Done, diff --git a/litellm-rust/crates/host-python/src/driver.rs b/litellm-rust/crates/host-python/src/driver.rs index 77a294d274b..d540c9b4513 100644 --- a/litellm-rust/crates/host-python/src/driver.rs +++ b/litellm-rust/crates/host-python/src/driver.rs @@ -207,6 +207,10 @@ where self.stage = Stage::Call; self.resume_machine(py, None) } + (Expect::Arguments, LifecycleStep::Cached(response)) => { + self.ended_at = Some(epoch_seconds()); + self.succeeded(py, response) + } (Expect::Wire, LifecycleStep::Wire(wire)) => { self.resume_machine(py, Some(Ok(HostResult::BeforeSend(wire)))) } diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 1a9b170f661..5f4aad063d9 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -85,6 +85,11 @@ impl MessagesRouteHost { .transpose()? .flatten(); Ok(MessagesCall { + cached_response: arguments + .get_item("_rust_messages_cached_response")? + .map(|value| from_py(&value)) + .transpose()?, + options: Default::default(), model, body, api_key: string("api_key")?, diff --git a/litellm-rust/crates/python-bridge/tests/messages_lifecycle.py b/litellm-rust/crates/python-bridge/tests/messages_lifecycle.py new file mode 100644 index 00000000000..47838ddbb16 --- /dev/null +++ b/litellm-rust/crates/python-bridge/tests/messages_lifecycle.py @@ -0,0 +1,107 @@ +import asyncio +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import litellm +from litellm.caching.caching import Cache +from litellm.rust_bridge import _native +from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest + + +class Upstream(BaseHTTPRequestHandler): + calls = [] + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["content-length"]))) + self.calls.append(body) + usage = {"input_tokens": 2, "output_tokens": 3, "cache_read_input_tokens": 4, "cache_creation_input_tokens": 5} + message = {"id": "msg_native", "type": "message", "role": "assistant", "model": "test-native-sdk", + "content": [{"type": "text", "text": "hello"}], "stop_reason": "end_turn", "stop_sequence": None, + "usage": usage} + if body.get("stream"): + events = ({"type": "message_start", "message": {**message, "usage": {**usage, "output_tokens": 0}}}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 3}}, + {"type": "message_stop"}) + response = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() + else: + response = json.dumps(message).encode() + self.send_response(200) + self.send_header("content-type", "text/event-stream" if body.get("stream") else "application/json") + self.send_header("content-length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + def log_message(self, *_args): + pass + + +class MessagesLifecycle(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.original = (litellm.cache, litellm.max_budget, litellm._current_cost, litellm.modify_params) + litellm.cache = Cache(type="local", supported_call_types=["anthropic_messages"]) + litellm.max_budget = 100.0 + litellm._current_cost = 0.0 + litellm.modify_params = False + litellm.register_model({"test-native-sdk": {"litellm_provider": "anthropic", "mode": "chat", + "input_cost_per_token": 2.0, "output_cost_per_token": 3.0, + "cache_read_input_token_cost": 0.5, "cache_creation_input_token_cost": 4.0, + "max_input_tokens": 200, "max_output_tokens": 50}}) + self.before = len(Upstream.calls) + + async def asyncTearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join() + litellm.cache, litellm.max_budget, litellm._current_cost, litellm.modify_params = self.original + litellm.model_cost.pop("test-native-sdk", None) + + def call(self, asynchronous=True, **extra): + kwargs = {"max_tokens": 100, "messages": [{"role": "user", "content": "hello"}], + "model": "test-native-sdk", "custom_llm_provider": "anthropic", "api_key": "test", + "api_base": f"http://127.0.0.1:{self.server.server_port}", **extra} + request = LiteLLMMessagesRequest(model=kwargs["model"], messages=kwargs["messages"], max_tokens=100, + stream=kwargs.get("stream"), api_key="test", api_base=kwargs["api_base"], + custom_llm_provider="anthropic", kwargs=kwargs) + return (_native.amessages if asynchronous else _native.messages)(request, (), kwargs) + + async def test_cache_hit_charges_once_and_budget_precedes_cache(self): + first = await self.call() + second = await self.call() + self.assertEqual(first, second) + self.assertEqual(len(Upstream.calls) - self.before, 1) + cost = 2 * 2.0 + 3 * 3.0 + 4 * 0.5 + 5 * 4.0 + self.assertEqual(litellm._current_cost, cost) + litellm.max_budget = cost - 1 + with self.assertRaises(litellm.BudgetExceededError): + await self.call() + self.assertEqual(len(Upstream.calls) - self.before, 1) + + async def test_stream_cache_replays_and_charges_once(self): + first = b"".join([chunk async for chunk in await self.call(stream=True)]) + second = b"".join([chunk async for chunk in await self.call(stream=True)]) + self.assertEqual(first, second) + self.assertEqual(len(Upstream.calls) - self.before, 1) + self.assertEqual(litellm._current_cost, 2 * 2.0 + 3 * 3.0 + 4 * 0.5 + 5 * 4.0) + + async def test_opt_in_output_cap_is_sent_and_cached_under_original_request(self): + litellm.modify_params = True + await self.call() + await self.call() + self.assertEqual(len(Upstream.calls) - self.before, 1) + self.assertEqual(Upstream.calls[-1]["max_tokens"], 50) + + async def test_sync_messages_use_the_same_cache_and_budget(self): + first = self.call(asynchronous=False) + second = await self.call() + self.assertEqual(first, second) + self.assertEqual(len(Upstream.calls) - self.before, 1) + + +if __name__ == "__main__": + print(f"native extension: {_native.__file__}") + unittest.main() diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ba8addbaaa0..6a7c86d488b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -504,6 +504,7 @@ class Logging(LiteLLMLoggingBaseClass): classifier_input: Mapping[str, JsonValue] | None = None baseline_cache_context: "BaselineCacheContext | None" = None baseline_observation: "CapturedBaselineObservation | None" = None + _native_response_cost: float | None = None def __init__( self, @@ -1744,6 +1745,9 @@ class Logging(LiteLLMLoggingBaseClass): if cache_hit is True: return 0.0 + if self._native_response_cost is not None: + return self._native_response_cost + if is_unbilled_non_inference_call( self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result ): diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy_python.py index 30aa1d97bfc..ca681ab14a2 100644 --- a/litellm/rust_bridge/callbacks_legacy_python.py +++ b/litellm/rust_bridge/callbacks_legacy_python.py @@ -64,21 +64,6 @@ def setup( return CallSetup(logger, prepared) -def check_limits(kwargs: Mapping[str, object]) -> None: - from litellm import ( - BudgetExceededError, - _current_cost, # pyright: ignore[reportPrivateUsage] # shared SDK budget counter has no public accessor - max_budget, - num_retries_per_request, - ) - from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit - - if max_budget and _current_cost > max_budget: - raise BudgetExceededError(current_cost=_current_cost, max_budget=max_budget) - if max_retries_per_request_hit(kwargs, num_retries_per_request): - raise RuntimeError("Max retries per request hit!") - - def finalize( response: object, logger: Logging, diff --git a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py b/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py index 1f6a214398a..dcd95c05371 100644 --- a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py @@ -11,7 +11,7 @@ from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.rust_bridge import callbacks_legacy_python as legacy -from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup +from litellm.rust_bridge.callbacks_legacy_python import setup _OCR_KWARGS: Final = MappingProxyType( { @@ -21,33 +21,6 @@ _OCR_KWARGS: Final = MappingProxyType( ) -@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) -@pytest.mark.parametrize( - "cap, request_retry_count, refused", - [(5, 5, True), (5, 4, False), (0, 0, False), (0, 1, True)], - ids=[ - "cap-above-four-reached", - "cap-above-four-not-reached", - "first-attempt-passes-cap-of-zero", - "cap-of-zero-refuses-first-retry", - ], -) -def test_check_limits_reads_request_retry_count( - monkeypatch: pytest.MonkeyPatch, metadata_key: str, cap: int, request_retry_count: int, refused: bool -) -> None: - monkeypatch.setattr(litellm, "num_retries_per_request", cap) - monkeypatch.setattr(litellm, "max_budget", None) - kwargs: Final = { - "model": "mistral/mistral-ocr-latest", - metadata_key: {"request_retry_count": request_retry_count}, - } - if refused: - with pytest.raises(RuntimeError, match="Max retries per request hit!"): - check_limits(kwargs) - else: - check_limits(kwargs) - - def _supplied_logger() -> Logging: return Logging( model="mistral/mistral-ocr-latest",