feat(rust): add messages SDK lifecycle

This commit is contained in:
Yujong Lee 2026-09-20 14:13:45 -07:00
parent 9599cc0cb4
commit cdca831c88
33 changed files with 1550 additions and 70 deletions

View file

@ -2067,9 +2067,13 @@ dependencies = [
name = "litellm-callbacks-legacy-python"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"litellm-auth",
"litellm-cache",
"litellm-core-utils",
"litellm-host",
"litellm-host-python",
"litellm-token-counter",
"proptest",
"pyo3",
"rstest",
@ -2087,10 +2091,13 @@ dependencies = [
"litellm-auth",
"litellm-auth-aws",
"litellm-auth-gcp",
"litellm-cache",
"litellm-cache-memory",
"litellm-core-utils",
"litellm-host",
"litellm-http",
"litellm-llms",
"litellm-token-counter",
"litellm-types",
"mime_guess",
"moka",

View file

@ -0,0 +1,48 @@
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use serde_json::Value;
use crate::{CacheBackend, CacheControls, CacheEntry, CacheKwargs};
#[derive(Clone)]
pub struct LLMCachingHandler {
pub backend: CacheBackend,
pub controls: CacheControls,
pub key: String,
pub kwargs: CacheKwargs,
pub max_age: Option<Duration>,
}
impl LLMCachingHandler {
pub async fn get(&self) -> Option<Value> {
if !self.controls.reads() {
return None;
}
let entry = self
.backend
.async_get_cache(&self.key, &self.kwargs)
.await
.ok()??;
entry.fresh(now(), self.max_age).then_some(entry.response)
}
pub async fn set(&self, response: Value) {
if !self.controls.writes() {
return;
}
let entry = CacheEntry {
timestamp: now().as_secs_f64(),
response,
};
let _ = self
.backend
.async_set_cache(&self.key, entry, self.kwargs.clone())
.await;
}
}
fn now() -> Duration {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
}

View file

@ -1,6 +1,8 @@
mod base_cache;
mod caching;
mod error;
mod handler;
pub use handler::LLMCachingHandler;
pub use base_cache::{
BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheFuture, CacheKwargs,

View file

@ -9,8 +9,12 @@ autotests = false
[dependencies]
litellm-host.workspace = true
litellm-host-python.workspace = true
litellm-core-utils.workspace = true
litellm-cache.workspace = true
litellm-token-counter.workspace = true
pyo3.workspace = true
base64.workspace = true
strum.workspace = true
serde_json.workspace = true

View file

@ -6,9 +6,6 @@
"start_time",
"asynchronous"
],
"check_limits": [
"kwargs"
],
"finalize": [
"response",
"logger",

View file

@ -49,6 +49,9 @@ struct DeliveredStream {
}
enum Pending {
CacheRead,
CacheWrite,
StreamCacheWrite,
DeploymentPreCall,
DeploymentPostCall,
DeploymentFailure,
@ -56,6 +59,9 @@ enum Pending {
}
pub struct LegacyLogging {
cache: Option<crate::caching::Caching>,
cache_hit: bool,
accounted: bool,
surface: LegacySurface,
call: PublicCall,
logger: Option<PythonLogger>,
@ -88,6 +94,9 @@ impl LegacyLogging {
asynchronous: bool,
) -> Self {
Self {
cache: None,
cache_hit: false,
accounted: false,
surface,
call,
logger: None,
@ -120,9 +129,123 @@ impl LegacyLogging {
fn prepare(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
let prepared = prepare(py, self.call.kwargs().bind(py), self.logger()?)?.unbind();
self.call.set_kwargs(prepared);
if self.surface.call_type == "anthropic_messages" {
self.cache = crate::caching::Caching::prepare(
py,
&self.call,
self.surface.call_type,
self.asynchronous,
)?;
if let Some(cache) = &self.cache
&& cache.controls.reads()
{
match cache.read(py, self.asynchronous) {
Ok(value) if self.asynchronous => {
self.pending = Some(Pending::CacheRead);
return Ok(LifecycleStep::Await(value));
}
result => return self.cache_read(py, result),
}
}
}
self.arguments(py)
}
fn arguments(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
if self.surface.call_type == "anthropic_messages" {
if let Err(error) =
crate::limits::adjust_max_tokens(py, &self.call, self.surface.call_type)
&& is_cancellation(py, &error)
{
return Err(error);
}
}
Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py)))
}
fn cache_read(
&mut self,
py: Python<'_>,
result: PyResult<Py<PyAny>>,
) -> PyResult<LifecycleStep> {
let decoded = result.and_then(|value| {
self.cache
.as_ref()
.ok_or_else(missing_state)?
.decode(value.bind(py))
});
let response = match decoded {
Ok(Some(response)) => response,
Err(error) if is_cancellation(py, &error) => return Err(error),
_ => return self.arguments(py),
};
self.cache_hit = true;
self.logger()?
.object(py)
.getattr("model_call_details")?
.set_item("cache_hit", true)?;
let cache = self.cache.as_ref().ok_or_else(missing_state)?;
self.logger()?
.object(py)
.getattr("litellm_params")?
.set_item("preset_cache_key", &cache.key)?;
if response
.get("litellm_cached_anthropic_sse_events")
.is_some()
{
self.call
.kwargs()
.bind(py)
.set_item("_rust_messages_cached_response", to_py(py, &response)?)?;
return Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py)));
}
self.end = Some(datetime(py, epoch_seconds())?);
self.response = Some(crate::caching::cached_public(py, &response)?);
self.finalize(py)?;
Ok(LifecycleStep::Cached(
self.response
.as_ref()
.ok_or_else(missing_state)?
.clone_ref(py),
))
}
fn cache_write(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
if let (Some(cache), Some(response)) = (&self.cache, &self.response)
&& cache.controls.writes()
&& !self.cache_hit
{
match cache.write(py, response.bind(py), self.asynchronous) {
Ok(awaitable) if self.asynchronous => {
self.pending = Some(Pending::CacheWrite);
return Ok(LifecycleStep::Await(awaitable));
}
Err(error) if is_cancellation(py, &error) => return Err(error),
_ => {}
}
}
self.finalize(py)
}
fn dispatch_cache_hit(&self, py: Python<'_>) -> PyResult<()> {
let kwargs = PyDict::new(py);
kwargs.set_item("result", &self.response)?;
kwargs.set_item("start_time", &self.start)?;
kwargs.set_item("end_time", &self.end)?;
kwargs.set_item("cache_hit", true)?;
let logger = self.logger()?.object(py);
if self.asynchronous {
let coroutine = logger.call_method("async_success_handler", (), Some(&kwargs))?;
crate::python::Logging::Enqueue.call(py, (coroutine,))?;
}
logger.call_method(
"handle_sync_success_callbacks_for_async_calls",
(),
Some(&kwargs),
)?;
Ok(())
}
fn finalize(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
finalize(
py,
@ -138,6 +261,68 @@ impl LegacyLogging {
.ok_or_else(missing_state)
}
fn record_usage(&mut self, py: Python<'_>, usage: &Value) -> PyResult<()> {
if self.accounted || self.surface.call_type != "anthropic_messages" {
return Ok(());
}
self.accounted = true;
let model = self
.call
.lookup(py, "model")?
.map(|value| value.extract::<String>())
.transpose()?;
let info = model
.map(|model| crate::limits::model_info(py, &model))
.transpose()?
.flatten();
let cost = if self.cache_hit {
Some(0.0)
} else {
info.as_ref()
.and_then(|info| litellm_core_utils::cost_calculator::messages_cost(info, usage))
};
if let Some(cost) = cost {
let logger = self.logger()?.object(py);
logger.setattr("_native_response_cost", cost)?;
logger
.getattr("model_call_details")?
.set_item("response_cost", cost)?;
let sdk = py.import("litellm")?;
if !self.cache_hit && sdk.getattr("max_budget")?.is_truthy()? {
let current: f64 = sdk.getattr("_current_cost")?.extract()?;
sdk.setattr("_current_cost", current + cost)?;
}
}
Ok(())
}
fn finish_stream(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
let stream = self.stream.as_ref().ok_or_else(missing_state)?;
let mut bytes = Vec::new();
for chunk in stream.chunks.bind(py).iter() {
bytes.extend_from_slice(chunk.cast::<pyo3::types::PyBytes>()?.as_bytes());
}
if let Some(parsed) = litellm_core_utils::messages_stream::MessagesStream::parse(&bytes) {
self.record_usage(py, &parsed.usage)?;
if !self.cache_hit
&& let (Some(cache), Some(response)) = (&self.cache, parsed.cached_response)
&& cache.controls.writes()
{
let response = to_py(py, &response)?;
match cache.write(py, response.bind(py), self.asynchronous) {
Ok(awaitable) if self.asynchronous => {
self.pending = Some(Pending::StreamCacheWrite);
return Ok(LifecycleStep::Await(awaitable));
}
Err(error) if is_cancellation(py, &error) => return Err(error),
_ => {}
}
}
}
self.stream_success(py, self.stream.as_ref().ok_or_else(missing_state)?)?;
Ok(LifecycleStep::Done)
}
fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
match self.try_dispatch_success(py) {
Err(error) if error.is_instance_of::<PyException>(py) => {
@ -149,6 +334,9 @@ impl LegacyLogging {
}
fn try_dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
if self.cache_hit {
return self.dispatch_cache_hit(py);
}
let logger = self.logger()?;
let pending = || PendingSuccess {
logger: logger.clone_ref(py),
@ -361,7 +549,7 @@ impl PythonLifecycle for LegacyLogging {
self.surface.call_type,
)?));
}
self.finalize(py)
self.cache_write(py)
}
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
@ -385,10 +573,14 @@ impl PythonLifecycle for LegacyLogging {
LifecycleEvent::Succeeded { timing, response } => {
self.end = Some(datetime(py, timing.end_time)?);
self.response = Some(response.clone_ref(py));
match &self.stream {
Some(stream) => self.stream_success(py, stream)?,
None => self.dispatch_success(py)?,
if self.stream.is_some() {
return self.finish_stream(py);
}
if self.surface.call_type == "anthropic_messages" {
let value: Value = from_py(response.bind(py))?;
self.record_usage(py, value.get("usage").unwrap_or(&Value::Null))?;
}
self.dispatch_success(py)?;
Ok(LifecycleStep::Done)
}
LifecycleEvent::Failed {
@ -448,8 +640,28 @@ impl PythonLifecycle for LegacyLogging {
}
Pending::DeploymentPostCall => {
self.response = Some(result?);
self.cache_write(py)
}
Pending::CacheRead => self.cache_read(py, result),
Pending::CacheWrite => {
if let Err(error) = result
&& is_cancellation(py, &error)
{
return Err(error);
}
self.finalize(py)
}
Pending::StreamCacheWrite => {
if let Err(error) = result
&& is_cancellation(py, &error)
{
return Err(error);
}
if let Some(stream) = &self.stream {
self.stream_success(py, stream)?;
}
Ok(LifecycleStep::Done)
}
Pending::DeploymentFailure => self.dispatch_failure(py),
Pending::AsyncFailure => match result {
Err(failure) if is_cancellation(py, &failure) => Err(failure),
@ -467,10 +679,14 @@ impl PythonLifecycle for LegacyLogging {
self.body = None;
self.context = None;
self.stream = None;
self.cache = None;
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
self.call.traverse(visit)?;
if let Some(cache) = &self.cache {
cache.traverse(visit)?;
}
if let Some(logger) = &self.logger {
logger.traverse(visit)?;
}

View file

@ -0,0 +1,291 @@
use litellm_cache::{CacheControls, CacheKeyField, CacheKeyInput, cache_key};
use litellm_host_python::{from_py, to_py};
use pyo3::{
gc::{PyTraverseError, PyVisit},
prelude::*,
types::PyDict,
};
use serde_json::Value;
use crate::PublicCall;
pub(crate) struct Caching {
backend: Py<PyAny>,
pub key: String,
pub controls: CacheControls,
kwargs: Py<PyDict>,
max_age: Option<f64>,
}
impl Caching {
pub fn prepare(
py: Python<'_>,
call: &PublicCall,
call_type: &str,
asynchronous: bool,
) -> PyResult<Option<Self>> {
let sdk = py.import("litellm")?;
let cache = sdk.getattr("cache")?;
if cache.is_none() {
return Ok(None);
}
let supported = cache.getattr("supported_call_types")?;
if supported.is_none() || !supported.contains(call_type)? {
return Ok(None);
}
let kwargs = call.kwargs().bind(py).copy()?;
let names = [
"max_tokens",
"messages",
"model",
"metadata",
"stop_sequences",
"stream",
"system",
"temperature",
"thinking",
"tool_choice",
"tools",
"top_k",
"top_p",
"container",
"api_key",
"api_base",
"client",
"custom_llm_provider",
];
for (name, value) in names
.iter()
.filter(|name| !asynchronous || **name != "container")
.zip(call.args().bind(py).iter())
{
kwargs.set_item(name, value)?;
}
let directives = kwargs
.get_item("cache")?
.unwrap_or_else(|| PyDict::new(py).into_any());
let directive = |name: &str| directives.call_method1("get", (name,));
let is_true =
|value: Bound<'_, PyAny>| value.is(&true.into_pyobject(py).expect("bool conversion"));
let controls = CacheControls {
supported_call_type: true,
configured: true,
native_backend: false,
default_on: cache.getattr("mode")?.eq("default_on")?,
caching: kwargs
.get_item("caching")?
.filter(|v| !v.is_none())
.map(is_true),
no_cache: is_true(directive("no-cache")?),
no_store: is_true(directive("no-store")?),
use_cache: is_true(directive("use-cache")?),
};
if !controls.reads() && !controls.writes() {
return Ok(None);
}
let metadata = kwargs
.get_item("metadata")?
.filter(|value| !value.is_none())
.unwrap_or_else(|| PyDict::new(py).into_any());
let params = kwargs
.get_item("litellm_params")?
.filter(|value| !value.is_none())
.unwrap_or_else(|| PyDict::new(py).into_any());
let preset = kwargs
.get_item("cache_key")?
.filter(|value| !value.is_none())
.or_else(|| {
params
.call_method1("get", ("preset_cache_key",))
.ok()
.filter(|value| !value.is_none())
});
let key = if let Some(preset) = preset {
preset.extract()?
} else {
let api_params = py
.import("litellm.litellm_core_utils.model_param_helper")?
.getattr("ModelParamHelper")?
.call_method0("_get_all_llm_api_params")?;
let internal = py
.import("litellm.types.utils")?
.getattr("all_litellm_params")?;
let include_provider_parameters = sdk
.getattr("enable_caching_on_provider_specific_optional_params")?
.is_truthy()?;
let model_group = metadata.call_method1("get", ("model_group",))?;
let nested_metadata = params.call_method1("get", ("metadata", PyDict::new(py)))?;
let model_group = if model_group.is_truthy()? {
model_group
} else if nested_metadata.is_none() {
py.None().into_bound(py)
} else {
nested_metadata.call_method1("get", ("model_group",))?
};
let groups = metadata.call_method1("get", ("caching_groups",))?;
let mut group = None;
if groups.is_truthy()? {
for item in groups.try_iter()? {
let item = item?;
if item.contains(&model_group)? {
group = Some(item.str()?.to_string());
break;
}
}
}
let fields = kwargs
.iter()
.map(|(name, value)| {
let name: String = name.extract()?;
let api_parameter = api_params.contains(&name)?;
let internal_parameter = internal.contains(&name)?;
let value = if value.is_none()
|| (!api_parameter && (!include_provider_parameters || internal_parameter))
{
None
} else if name == "model" && group.is_some() {
group.clone()
} else if name == "model" && model_group.is_truthy()? {
Some(model_group.str()?.to_string())
} else {
Some(value.str()?.to_string())
};
Ok(CacheKeyField {
name,
value,
api_parameter,
internal_parameter,
})
})
.collect::<PyResult<Vec<_>>>()?;
let namespace = [
directive("namespace")?,
metadata.call_method1("get", ("redis_namespace",))?,
cache.getattr("namespace")?,
]
.into_iter()
.find(|value| value.is_truthy().unwrap_or(false))
.map(|value| value.extract())
.transpose()?;
cache_key(&CacheKeyInput {
fields,
preset: None,
namespace,
include_provider_parameters,
})
};
let backend_kwargs = PyDict::new(py);
for name in [
"messages",
"input",
"metadata",
"litellm_params",
"parent_otel_span",
] {
if let Some(value) = kwargs.get_item(name)? {
backend_kwargs.set_item(name, value)?;
}
}
let ttl = directive("ttl")?;
let ttl = if !ttl.is_none() {
Some(ttl)
} else {
let configured = cache.getattr("ttl")?;
if configured.is_none() {
kwargs.get_item("ttl")?
} else {
Some(configured)
}
};
if let Some(ttl) = ttl {
backend_kwargs.set_item("ttl", ttl)?;
}
let max_age = if asynchronous {
directives.call_method1("get", ("s-max-age", directive("s-maxage")?))?
} else {
let first = directive("s-maxage")?;
if first.is_truthy()? {
first
} else {
directive("s-max-age")?
}
};
Ok(Some(Self {
backend: cache.getattr("cache")?.unbind(),
key,
controls,
kwargs: backend_kwargs.unbind(),
max_age: max_age.extract()?,
}))
}
pub fn read(&self, py: Python<'_>, asynchronous: bool) -> PyResult<Py<PyAny>> {
self.backend
.bind(py)
.call_method(
if asynchronous {
"async_get_cache"
} else {
"get_cache"
},
(&self.key,),
Some(self.kwargs.bind(py)),
)
.map(Bound::unbind)
}
pub fn decode(&self, value: &Bound<'_, PyAny>) -> PyResult<Option<Value>> {
if value.is_none() {
return Ok(None);
}
let value: Value = from_py(value)?;
let response = if let Some(timestamp) = value.get("timestamp").and_then(Value::as_f64) {
if self
.max_age
.is_some_and(|max| litellm_host::event::epoch_seconds() - timestamp > max)
{
return Ok(None);
}
value.get("response").cloned().unwrap_or(Value::Null)
} else {
value
};
let response = match response {
Value::String(text) => serde_json::from_str(&text).unwrap_or(Value::Null),
value => value,
};
Ok(response.is_object().then_some(response))
}
pub fn write(
&self,
py: Python<'_>,
response: &Bound<'_, PyAny>,
asynchronous: bool,
) -> PyResult<Py<PyAny>> {
let entry = PyDict::new(py);
entry.set_item("timestamp", litellm_host::event::epoch_seconds())?;
entry.set_item("response", response)?;
self.backend
.bind(py)
.call_method(
if asynchronous {
"async_set_cache"
} else {
"set_cache"
},
(&self.key, entry),
Some(self.kwargs.bind(py)),
)
.map(Bound::unbind)
}
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.backend)?;
visit.call(&self.kwargs)
}
}
pub(crate) fn cached_public(py: Python<'_>, value: &Value) -> PyResult<Py<PyAny>> {
to_py(py, value)
}

View file

@ -10,9 +10,11 @@
//! without keeping a copy.
mod adapter;
mod caching;
mod call;
mod callbacks;
mod deferred;
mod limits;
mod logger;
mod preparation;
mod python;

View file

@ -0,0 +1,107 @@
use std::sync::OnceLock;
use litellm_core_utils::token_counter::get_modified_max_tokens;
use litellm_host_python::from_py;
use litellm_token_counter::{CountableRequest, TokenCounter};
use pyo3::{prelude::*, types::PyDict};
use serde_json::Value;
use crate::PublicCall;
pub(crate) fn adjust_max_tokens(
py: Python<'_>,
call: &PublicCall,
call_type: &str,
) -> PyResult<()> {
if !matches!(
call_type,
"completion" | "acompletion" | "anthropic_messages"
) {
return Ok(());
}
let sdk = py.import("litellm")?;
if !sdk.getattr("modify_params")?.is(&true.into_pyobject(py)?) {
return Ok(());
}
let Some(requested) = call
.lookup(py, "max_tokens")?
.filter(|value| !value.is_none())
else {
return Ok(());
};
let requested: i64 = requested.extract()?;
let Some(model) = call.lookup(py, "model")? else {
return Ok(());
};
let model: String = model.extract()?;
let info = model_info(py, &model)?;
let Some(info) = info else {
return Ok(());
};
let counter = counter(py, &model)?;
let messages = if call.args().bind(py).len() > 1 {
call.args().bind(py).get_item(1)?
} else {
call.lookup(py, "messages")?
.unwrap_or_else(|| py.None().into_bound(py))
};
let request = PyDict::new(py);
request.set_item("model", &model)?;
request.set_item("messages", messages)?;
let request: CountableRequest = from_py(&request)?;
let count = py
.detach(|| counter.count_request(&request))
.map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?;
let adjusted = get_modified_max_tokens(
requested,
info.get("max_input_tokens").and_then(Value::as_i64),
info.get("max_output_tokens")
.or_else(|| info.get("max_tokens"))
.and_then(Value::as_i64),
count.input_tokens,
None,
None,
);
call.kwargs().bind(py).set_item("max_tokens", adjusted)
}
pub(crate) fn model_info(py: Python<'_>, model: &str) -> PyResult<Option<Value>> {
let sdk = py.import("litellm")?;
let costs = sdk.getattr("model_cost")?;
let aliases = sdk.getattr("model_alias_map")?;
let resolved = aliases.call_method1("get", (model, model))?;
let resolved: String = resolved.extract()?;
let stripped = resolved.strip_prefix("anthropic/").unwrap_or(&resolved);
for key in [&resolved, stripped, &format!("anthropic/{stripped}")] {
let value = costs.call_method1("get", (key,))?;
if !value.is_none() {
return from_py(&value).map(Some);
}
}
Ok(None)
}
fn counter(py: Python<'_>, model: &str) -> PyResult<&'static TokenCounter> {
static CLAUDE: OnceLock<TokenCounter> = OnceLock::new();
static CL100K: OnceLock<TokenCounter> = OnceLock::new();
let sdk = py.import("litellm")?;
let anthropic = sdk.getattr("anthropic_models")?.contains(model)?
&& !model.contains("claude-3")
&& !sdk.getattr("disable_hf_tokenizer_download")?.is_truthy()?;
let cell = if anthropic { &CLAUDE } else { &CL100K };
if let Some(counter) = cell.get() {
return Ok(counter);
}
let loaded = if anthropic {
let json: String = py
.import("litellm.utils")?
.getattr("claude_json_str")?
.extract()?;
py.detach(|| TokenCounter::from_json(&json))
} else {
py.detach(|| TokenCounter::from_tiktoken("cl100k_base"))
}
.map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?;
let _ = cell.set(loaded);
cell.get().ok_or_else(litellm_host_python::missing_state)
}

View file

@ -29,10 +29,60 @@ pub fn prepare<'py>(
.call(py, ())?
.cast_into::<PyList>()?)
})?;
Wrapper::CheckLimits.call(py, (&arguments,))?;
check_limits(py, &arguments)?;
Ok(arguments)
}
fn check_limits(py: Python<'_>, kwargs: &Bound<'_, PyDict>) -> PyResult<()> {
let sdk = py.import("litellm")?;
check_limits_with_settings(py, kwargs, sdk.as_any())
}
fn check_limits_with_settings(
py: Python<'_>,
kwargs: &Bound<'_, PyDict>,
sdk: &Bound<'_, PyAny>,
) -> PyResult<()> {
let max_budget = sdk.getattr("max_budget")?.extract::<Option<f64>>()?;
if max_budget.is_some_and(|limit| limit != 0.0) {
let current_cost = sdk.getattr("_current_cost")?.extract::<f64>()?;
if let Err(error) = litellm_core_utils::budget::check_budget(current_cost, max_budget) {
let arguments = PyDict::new(py);
arguments.set_item("current_cost", error.current_cost)?;
arguments.set_item("max_budget", error.max_budget)?;
return Err(PyErr::from_value(
sdk.getattr("BudgetExceededError")?
.call((), Some(&arguments))?,
));
}
}
let limit = sdk.getattr("num_retries_per_request")?;
if limit.is_none() {
return Ok(());
}
let key = if kwargs.contains("litellm_metadata")? {
"litellm_metadata"
} else {
"metadata"
};
let Some(metadata) = kwargs.get_item(key)? else {
return Ok(());
};
if !metadata.is_instance(&py.import("collections.abc")?.getattr("Mapping")?)? {
return Ok(());
}
let retry = metadata.call_method1("get", ("request_retry_count",))?;
if retry.get_type().is(&py.get_type::<pyo3::types::PyInt>())
&& retry.gt(0)?
&& retry.ge(limit)?
{
return Err(pyo3::exceptions::PyRuntimeError::new_err(
"Max retries per request hit!",
));
}
Ok(())
}
fn inherit_credentials<'py>(
py: Python<'py>,
arguments: &Bound<'py, PyDict>,
@ -73,6 +123,58 @@ fn inherit_credentials<'py>(
mod tests {
use super::*;
#[test]
fn native_limits_preserve_budget_and_retry_boundaries() {
Python::initialize();
Python::attach(|py| {
let locals = eval(
py,
c"
import types
class BudgetExceededError(Exception):
def __init__(self, current_cost, max_budget):
self.current_cost = current_cost
self.max_budget = max_budget
settings = types.SimpleNamespace(max_budget=2.0, _current_cost=2.0, num_retries_per_request=5,
BudgetExceededError=BudgetExceededError)
",
);
let settings = locals.get_item("settings").unwrap().unwrap();
let arguments = PyDict::new(py);
assert!(check_limits_with_settings(py, &arguments, &settings).is_ok());
settings.setattr("_current_cost", 3.0).unwrap();
let error = check_limits_with_settings(py, &arguments, &settings).unwrap_err();
assert_eq!(
error
.value(py)
.getattr("current_cost")
.unwrap()
.extract::<f64>()
.unwrap(),
3.0
);
settings.setattr("max_budget", 0.0).unwrap();
for key in ["metadata", "litellm_metadata"] {
for (count, refused) in [(0, false), (4, false), (5, true), (6, true)] {
let metadata = PyDict::new(py);
metadata.set_item("request_retry_count", count).unwrap();
arguments.set_item(key, metadata).unwrap();
assert_eq!(
check_limits_with_settings(py, &arguments, &settings).is_err(),
refused
);
}
let metadata = PyDict::new(py);
metadata.set_item("request_retry_count", true).unwrap();
settings.setattr("num_retries_per_request", 0).unwrap();
arguments.set_item(key, metadata).unwrap();
assert!(check_limits_with_settings(py, &arguments, &settings).is_ok());
arguments.del_item(key).unwrap();
settings.setattr("num_retries_per_request", 5).unwrap();
}
});
}
fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> {
let locals = PyDict::new(py);
py.run(source, Some(&locals), Some(&locals)).unwrap();

View file

@ -25,8 +25,6 @@ pub(crate) enum LegacyPython {
pub(crate) enum Wrapper {
#[strum(serialize = "setup")]
Setup,
#[strum(serialize = "check_limits")]
CheckLimits,
#[strum(serialize = "credential_list")]
CredentialList,
#[strum(serialize = "warn_unknown_credential")]

View file

@ -90,8 +90,6 @@ assert prepared['document'] is replacement
assert prepared['pages'] is replaced_kwargs['pages']
assert prepared['litellm_logging_obj'] is logger
assert 'litellm_logging_obj' not in replaced_kwargs
[checked] = [value for name, value in logger.calls if name == 'check_limits']
assert checked is prepared
",
);
});
@ -126,8 +124,6 @@ kwargs = {'logger': logger, 'vendor_extension': opaque}
&locals,
c"
assert prepared['vendor_extension'] is opaque
[checked] = [value for name, value in logger.calls if name == 'check_limits']
assert checked['vendor_extension'] is opaque
assert hooked == ([opaque] if asynchronous else []), hooked
",
);
@ -247,7 +243,7 @@ assert all(value is failure for name, value in logger.calls if name.endswith('_h
#[rstest]
#[case::synchronous(false)]
#[case::asynchronous(true)]
fn a_limit_rejected_before_the_call_surfaces_as_the_callers_error(#[case] asynchronous: bool) {
fn preparation_failure_before_the_call_preserves_exception_identity(#[case] asynchronous: bool) {
Python::initialize();
Python::attach(|py| {
let locals = namespace(
@ -258,13 +254,13 @@ class BudgetExceeded(Exception):
rejection = BudgetExceeded('over budget')
class LimitedLogger(StubLogger):
def check_limits(self, arguments):
class RejectedCredential(str):
def __bool__(self):
raise rejection
logger = LimitedLogger()
logger = StubLogger()
logger.hooks = {'pre': lambda kwargs: kwargs}
kwargs = {'logger': logger}
kwargs = {'logger': logger, 'litellm_credential_name': RejectedCredential('named')}
",
);
let mut logging = legacy_call(py, &locals, asynchronous);

View file

@ -27,6 +27,9 @@ for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_le
sys.modules.setdefault(name, types.ModuleType(name))
legacy = sys.modules['litellm.rust_bridge.callbacks_legacy_python']
sdk = sys.modules['litellm']
sdk.max_budget = None
sdk.num_retries_per_request = None
CONTRACT = json.loads(python_contract)
@ -50,7 +53,6 @@ FAKES = {
logger=kwargs['logger_factory'](kwargs) if 'logger_factory' in kwargs else kwargs['logger'],
kwargs=kwargs,
),
'check_limits': lambda arguments: arguments['logger'].check_limits(arguments),
'finalize': lambda response, logger, kwargs, start, end: logger.record('finalize', response),
'update_logging': lambda logger, kwargs, model, optional_params, litellm_params, provider: logger.update_from_kwargs(
kwargs=kwargs,

View file

@ -0,0 +1,82 @@
use std::sync::Mutex;
#[derive(Clone, Copy, Debug, PartialEq, thiserror::Error)]
#[error("Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}")]
pub struct BudgetExceeded {
pub current_cost: f64,
pub max_budget: f64,
}
pub fn check_budget(current_cost: f64, max_budget: Option<f64>) -> Result<(), BudgetExceeded> {
if let Some(max_budget) = max_budget.filter(|limit| *limit != 0.0)
&& current_cost > max_budget
{
return Err(BudgetExceeded {
current_cost,
max_budget,
});
}
Ok(())
}
#[derive(Default)]
pub struct Budget {
current_cost: Mutex<f64>,
pub max_budget: Option<f64>,
}
impl Budget {
pub fn new(max_budget: Option<f64>, current_cost: f64) -> Self {
Self {
current_cost: Mutex::new(current_cost),
max_budget,
}
}
pub fn current_cost(&self) -> f64 {
*self
.current_cost
.lock()
.unwrap_or_else(|error| error.into_inner())
}
pub fn check(&self) -> Result<(), BudgetExceeded> {
check_budget(self.current_cost(), self.max_budget)
}
pub fn record(&self, cost: f64) {
*self
.current_cost
.lock()
.unwrap_or_else(|error| error.into_inner()) += cost;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn matches_python_truthiness_and_strict_boundary() {
assert!(check_budget(100.0, None).is_ok());
assert!(check_budget(100.0, Some(0.0)).is_ok());
assert!(check_budget(1.0, Some(1.0)).is_ok());
assert_eq!(check_budget(1.5, Some(1.0)).unwrap_err().current_cost, 1.5);
}
#[test]
fn concurrent_completions_do_not_lose_spend() {
let budget = Budget::new(Some(10.0), 0.0);
std::thread::scope(|scope| {
for _ in 0..8 {
scope.spawn(|| {
for _ in 0..100 {
budget.record(0.25);
}
});
}
});
assert_eq!(budget.current_cost(), 200.0);
assert!(budget.check().is_err());
}
}

View file

@ -0,0 +1,156 @@
use serde_json::Value;
#[derive(Clone, Debug, Default)]
pub struct TokenPrices {
pub input: f64,
pub output: f64,
pub cache_read: f64,
pub cache_creation: f64,
pub cache_creation_1hr: f64,
}
impl TokenPrices {
pub fn from_model_info(info: &Value, usage: &Value) -> Option<Self> {
let input = info.get("input_cost_per_token")?.as_f64()?;
let output = info.get("output_cost_per_token")?.as_f64()?;
let total_input = [
"input_tokens",
"cache_read_input_tokens",
"cache_creation_input_tokens",
]
.iter()
.map(|key| count(usage, key))
.sum::<f64>();
let threshold = info
.as_object()?
.keys()
.filter_map(|key| {
let threshold = key
.strip_prefix("input_cost_per_token_above_")?
.strip_suffix("_tokens")?;
let number = threshold
.strip_suffix('k')
.unwrap_or(threshold)
.parse::<f64>()
.ok()?
* if threshold.ends_with('k') {
1000.0
} else {
1.0
};
(total_input > number).then_some((number, threshold))
})
.max_by(|a, b| a.0.total_cmp(&b.0))
.map(|(_, suffix)| suffix);
let tier = usage
.get("service_tier")
.and_then(Value::as_str)
.filter(|tier| matches!(*tier, "priority" | "flex"));
let rate = |key: &str| -> Option<f64> {
let base = tier
.and_then(|tier| info.get(format!("{key}_{tier}")).and_then(Value::as_f64))
.or_else(|| info.get(key).and_then(Value::as_f64));
threshold
.and_then(|threshold| {
let key = format!("{key}_above_{threshold}_tokens");
tier.and_then(|tier| info.get(format!("{key}_{tier}")).and_then(Value::as_f64))
.or_else(|| info.get(key).and_then(Value::as_f64))
})
.or(base)
};
let input = rate("input_cost_per_token").unwrap_or(input);
let creation = rate("cache_creation_input_token_cost").unwrap_or(input);
Some(Self {
input,
output: rate("output_cost_per_token").unwrap_or(output),
cache_read: rate("cache_read_input_token_cost").unwrap_or(input),
cache_creation: creation,
cache_creation_1hr: rate("cache_creation_input_token_cost_above_1hr")
.unwrap_or(creation),
})
}
pub fn messages_cost(&self, usage: &Value) -> f64 {
let creation = match usage
.get("cache_creation")
.filter(|value| value.is_object())
{
Some(details) => {
count(details, "ephemeral_5m_input_tokens") * self.cache_creation
+ count(details, "ephemeral_1h_input_tokens") * self.cache_creation_1hr
}
None => count(usage, "cache_creation_input_tokens") * self.cache_creation,
};
count(usage, "input_tokens") * self.input
+ count(usage, "output_tokens") * self.output
+ count(usage, "cache_read_input_tokens") * self.cache_read
+ creation
}
}
fn count(value: &Value, key: &str) -> f64 {
value.get(key).and_then(Value::as_f64).unwrap_or_default()
}
pub fn messages_cost(info: &Value, usage: &Value) -> Option<f64> {
let prices = TokenPrices::from_model_info(info, usage)?;
let provider = info.get("provider_specific_entry").unwrap_or(&Value::Null);
let speed = if usage.get("speed").and_then(Value::as_str) == Some("fast") {
provider.get("fast").and_then(Value::as_f64).unwrap_or(1.0)
} else {
1.0
};
let geo = usage
.get("inference_geo")
.and_then(Value::as_str)
.and_then(|geo| provider.get(geo))
.and_then(Value::as_f64)
.unwrap_or(1.0);
let searches = usage
.pointer("/server_tool_use/web_search_requests")
.and_then(Value::as_f64)
.unwrap_or_default();
let search_rate = info
.pointer("/search_context_cost_per_query/search_context_size_medium")
.and_then(Value::as_f64)
.unwrap_or_default();
Some(prices.messages_cost(usage) * speed * geo + searches * search_rate)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn accounts_for_each_input_category_once_and_preserves_zero_prices() {
let info = json!({"input_cost_per_token": 2.0, "output_cost_per_token": 3.0,
"cache_read_input_token_cost": 0.0, "cache_creation_input_token_cost": 4.0,
"cache_creation_input_token_cost_above_1hr": 5.0});
let usage = json!({"input_tokens": 10, "output_tokens": 3,
"cache_read_input_tokens": 100, "cache_creation_input_tokens": 7,
"cache_creation": {"ephemeral_5m_input_tokens": 2, "ephemeral_1h_input_tokens": 5}});
assert_eq!(messages_cost(&info, &usage), Some(62.0));
}
#[test]
fn threshold_uses_total_input_and_is_strict() {
let info = json!({"input_cost_per_token": 1.0, "output_cost_per_token": 2.0,
"input_cost_per_token_above_100_tokens": 4.0});
assert_eq!(
messages_cost(&info, &json!({"input_tokens": 100})),
Some(100.0)
);
assert_eq!(
messages_cost(
&info,
&json!({"input_tokens": 100, "cache_read_input_tokens": 1})
),
Some(404.0)
);
assert_eq!(
messages_cost(&json!({}), &json!({"input_tokens": 100})),
None
);
}
}

View file

@ -1,10 +1,14 @@
pub mod budget;
pub mod call_arguments;
pub mod core_helpers;
pub mod cost_calculator;
pub mod exception_mapping_utils;
pub mod get_llm_provider_logic;
pub mod messages_stream;
pub mod params;
pub mod prompt_templates;
pub mod secret_redaction;
pub mod serde_compat;
pub mod settings;
pub mod token_counter;
pub mod url_utils;

View file

@ -0,0 +1,40 @@
use serde_json::{Map, Value};
pub struct MessagesStream {
pub usage: Value,
pub cached_response: Option<Value>,
}
impl MessagesStream {
pub fn parse(bytes: &[u8]) -> Option<Self> {
let text = std::str::from_utf8(bytes).ok()?;
let mut usage = Map::new();
let mut stopped = false;
let mut failed = false;
let events: Vec<_> = text.split_inclusive("\n\n").collect();
for event in &events {
for line in event.lines().filter_map(|line| line.strip_prefix("data:")) {
let Ok(value) = serde_json::from_str::<Value>(line.trim()) else {
continue;
};
match value.get("type").and_then(Value::as_str) {
Some("message_stop") => stopped = true,
Some("error") => failed = true,
_ => {}
}
if let Some(fields) = value
.pointer("/message/usage")
.or_else(|| value.get("usage"))
.and_then(Value::as_object)
{
usage.extend(fields.clone());
}
}
}
Some(Self {
usage: Value::Object(usage),
cached_response: (stopped && !failed)
.then(|| serde_json::json!({"litellm_cached_anthropic_sse_events": events})),
})
}
}

View file

@ -0,0 +1,55 @@
pub fn get_modified_max_tokens(
user_max_tokens: i64,
max_input_tokens: Option<i64>,
max_output_tokens: Option<i64>,
input_tokens: usize,
buffer_perc: Option<f64>,
buffer_num: Option<f64>,
) -> i64 {
let Some(output) = max_output_tokens else {
return user_max_tokens;
};
let buffer =
(buffer_perc.unwrap_or(0.1) * input_tokens as f64).max(buffer_num.unwrap_or(10.0)) as i64;
let input = (input_tokens as i64).saturating_add(buffer);
if max_input_tokens == Some(output) {
if input <= output && user_max_tokens.saturating_add(input) > output {
return output - input;
}
return user_max_tokens;
}
user_max_tokens.min(output)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn shared_context_includes_buffer_and_does_not_hide_overflow() {
assert_eq!(
get_modified_max_tokens(80, Some(100), Some(100), 40, None, None),
50
);
assert_eq!(
get_modified_max_tokens(80, Some(100), Some(100), 100, None, None),
80
);
assert_eq!(
get_modified_max_tokens(80, Some(100), Some(100), 90, None, None),
0
);
assert_eq!(
get_modified_max_tokens(80, Some(1000), Some(100), 900, None, None),
80
);
assert_eq!(
get_modified_max_tokens(180, Some(1000), Some(100), 900, None, None),
100
);
assert_eq!(
get_modified_max_tokens(180, None, None, 900, None, None),
180
);
}
}

View file

@ -9,6 +9,8 @@ autotests = false
[dependencies]
litellm-types.workspace = true
litellm-core-utils.workspace = true
litellm-cache.workspace = true
litellm-token-counter.workspace = true
litellm-host.workspace = true
bytes.workspace = true
futures-util.workspace = true
@ -36,6 +38,7 @@ url.workspace = true
veil.workspace = true
[dev-dependencies]
litellm-cache-memory.workspace = true
litellm-auth-gcp.workspace = true
litellm-llms = { workspace = true, features = ["test-support"] }
rstest.workspace = true

View file

@ -0,0 +1,67 @@
use std::sync::Arc;
use litellm_cache::LLMCachingHandler;
use litellm_core_utils::{
budget::Budget, cost_calculator::messages_cost, token_counter::get_modified_max_tokens,
};
use litellm_token_counter::{CountableRequest, TokenCounter};
use serde_json::{Map, Value};
#[derive(Clone, Default)]
pub struct ClientOptions {
pub cache: Option<LLMCachingHandler>,
pub budget: Option<Arc<Budget>>,
pub model_info: Option<Value>,
pub token_counter: Option<Arc<TokenCounter>>,
pub modify_params: bool,
}
impl ClientOptions {
pub fn adjust_max_tokens(&self, body: &mut Map<String, Value>) {
if !self.modify_params {
return;
}
let (Some(counter), Some(info), Some(requested)) = (
&self.token_counter,
&self.model_info,
body.get("max_tokens").and_then(Value::as_i64),
) else {
return;
};
let request =
serde_json::json!({"model": body.get("model"), "messages": body.get("messages")});
let Ok(request) = serde_json::from_value::<CountableRequest>(request) else {
return;
};
let Ok(count) = counter.count_request(&request) else {
return;
};
let output = info
.get("max_output_tokens")
.or_else(|| info.get("max_tokens"))
.and_then(Value::as_i64);
let adjusted = get_modified_max_tokens(
requested,
info.get("max_input_tokens").and_then(Value::as_i64),
output,
count.input_tokens,
None,
None,
);
body.insert("max_tokens".into(), Value::from(adjusted));
}
pub fn record_messages_usage(&self, usage: &Value, cache_hit: bool) -> Option<f64> {
let cost = if cache_hit {
Some(0.0)
} else {
self.model_info
.as_ref()
.and_then(|info| messages_cost(info, usage))
};
if !cache_hit && let (Some(budget), Some(cost)) = (&self.budget, cost) {
budget.record(cost);
}
cost
}
}

View file

@ -1,5 +1,6 @@
pub mod audio_transcription;
pub mod chat_completions;
pub mod client;
pub mod constants;
pub mod error;
pub mod messages;

View file

@ -1,7 +1,9 @@
use litellm_llms::base_llm::chat::transformation::Error as LlmError;
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
#[derive(Clone, Debug, PartialEq, thiserror::Error)]
pub enum Error {
#[error(transparent)]
BudgetExceeded(#[from] litellm_core_utils::budget::BudgetExceeded),
#[error("invalid provider: {0}")]
InvalidProvider(String),
#[error("missing required field: {0}")]

View file

@ -0,0 +1,147 @@
use std::{sync::Arc, time::Duration};
use litellm_cache::{CacheControls, CacheKwargs, LLMCachingHandler};
use litellm_cache_memory::InMemoryCache;
use litellm_core_utils::budget::Budget;
use litellm_token_counter::{TokenCounter, Tokenizer};
use serde_json::{Value, json};
use tokio::{io::AsyncWriteExt, net::TcpListener};
use crate::{
client::ClientOptions,
messages::{Error, messages_with_options, types::MessagesRequest},
};
fn options() -> ClientOptions {
ClientOptions {
cache: Some(LLMCachingHandler {
backend: Arc::new(InMemoryCache::default()),
controls: CacheControls {
supported_call_type: true,
configured: true,
native_backend: true,
default_on: true,
..Default::default()
},
key: "test-key".into(),
kwargs: CacheKwargs::default(),
max_age: None,
}),
budget: Some(Arc::new(Budget::new(Some(100.0), 0.0))),
model_info: Some(
json!({"input_cost_per_token": 2.0, "output_cost_per_token": 3.0,
"cache_read_input_token_cost": 0.5, "cache_creation_input_token_cost": 4.0}),
),
..Default::default()
}
}
fn request(url: &str) -> MessagesRequest<'_> {
MessagesRequest {
model: "anthropic/test-model",
body: json!({"model":"anthropic/test-model", "max_tokens": 100,
"messages": [{"role": "user", "content": "hi"}]}),
api_key: Some("test"),
api_base: Some(url),
custom_llm_provider: Some("anthropic"),
extra_headers: None,
timeout: Some(Duration::from_secs(2)),
}
}
async fn start_server() -> (String, tokio::task::JoinHandle<Value>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = super::tests::read_http_request(&mut socket).await;
let body = json!({"id":"msg_test", "type":"message", "role":"assistant", "model":"test-model",
"content":[{"type":"text","text":"hello"}], "stop_reason":"end_turn", "stop_sequence":null,
"usage":{"input_tokens":2,"output_tokens":3,"cache_read_input_tokens":4,"cache_creation_input_tokens":5}});
socket
.write_all(super::tests::write_response(&body.to_string()).as_bytes())
.await
.unwrap();
serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap()
});
(url, server)
}
#[tokio::test]
async fn native_messages_cache_hit_skips_network_and_does_not_double_charge() {
let options = options();
let (url, server) = start_server().await;
let first = messages_with_options(request(&url), options.clone())
.await
.unwrap();
server.await.unwrap();
let cost = 2.0 * 2.0 + 3.0 * 3.0 + 4.0 * 0.5 + 5.0 * 4.0;
assert_eq!(options.budget.as_ref().unwrap().current_cost(), cost);
let cached = messages_with_options(request(&url), options.clone())
.await
.unwrap();
assert_eq!(first, cached);
assert_eq!(options.budget.as_ref().unwrap().current_cost(), cost);
let exceeded = ClientOptions {
budget: Some(Arc::new(Budget::new(Some(cost - 1.0), cost))),
..options
};
assert!(matches!(
messages_with_options(request(&url), exceeded).await,
Err(Error::BudgetExceeded(_))
));
}
#[tokio::test]
async fn no_store_and_no_cache_control_different_operations() {
let mut options = options();
options.cache.as_mut().unwrap().controls.no_store = true;
let (url, server) = start_server().await;
messages_with_options(request(&url), options.clone())
.await
.unwrap();
server.await.unwrap();
assert!(options.cache.as_ref().unwrap().get().await.is_none());
options.cache.as_mut().unwrap().controls.no_store = false;
options
.cache
.as_ref()
.unwrap()
.set(json!({"id":"sentinel"}))
.await;
options.cache.as_mut().unwrap().controls.no_cache = true;
assert!(options.cache.as_ref().unwrap().get().await.is_none());
let (url, server) = start_server().await;
let response = messages_with_options(request(&url), options.clone())
.await
.unwrap();
server.await.unwrap();
options.cache.as_mut().unwrap().controls.no_cache = false;
assert_eq!(
options.cache.as_ref().unwrap().get().await.unwrap()["id"],
response.id
);
}
struct Characters;
impl Tokenizer for Characters {
fn count_tokens(&self, text: &str) -> Result<usize, litellm_token_counter::Error> {
Ok(text.chars().count())
}
}
#[tokio::test]
async fn opt_in_token_adjustment_reaches_the_wire_with_a_standard_counter_interface() {
let (url, server) = start_server().await;
let options = ClientOptions {
token_counter: Some(Arc::new(TokenCounter::new(Characters))),
modify_params: true,
model_info: Some(json!({"max_input_tokens": 40, "max_output_tokens": 40})),
..Default::default()
};
messages_with_options(request(&url), options).await.unwrap();
let sent = server.await.unwrap();
assert_eq!(sent["max_tokens"], 40 - (3 + 4 + 2 + 3) - 10);
}

View file

@ -19,12 +19,21 @@ use serde_json::Value;
use crate::messages::types::MessagesRequest;
pub async fn messages(request: MessagesRequest<'_>) -> Result<AnthropicMessagesResponse, Error> {
messages_with_options(request, crate::client::ClientOptions::default()).await
}
pub async fn messages_with_options(
request: MessagesRequest<'_>,
options: crate::client::ClientOptions,
) -> Result<AnthropicMessagesResponse, Error> {
let Value::Object(body) = request.body else {
return Err(Error::InvalidRequest(
"messages body must be an object".into(),
));
};
let call = MessagesCall {
cached_response: None,
options,
model: request.model.into(),
body,
api_key: request.api_key.map(Into::into),
@ -41,5 +50,7 @@ pub async fn messages(request: MessagesRequest<'_>) -> Result<AnthropicMessagesR
}
}
#[cfg(test)]
mod lifecycle_tests;
#[cfg(test)]
mod tests;

View file

@ -32,6 +32,8 @@ pub enum MessagesOpResult {
/// The caller's request as the host projects it.
pub struct MessagesCall {
pub cached_response: Option<Value>,
pub options: crate::client::ClientOptions,
pub model: String,
pub body: Map<String, Value>,
pub api_key: Option<String>,
@ -125,8 +127,48 @@ pub fn messages_machine() -> MessagesMachine {
}
async fn execute(host: MessagesHost) -> Result<MessagesOutput, Error> {
let MessagesOpResult::Request(call) = host.route(MessagesOp::ProjectRequest).await?;
let MessagesOpResult::Request(mut call) = host.route(MessagesOp::ProjectRequest).await?;
if let Some(budget) = &call.options.budget {
budget.check()?;
}
let stream = call.streams();
let cached = match call.cached_response.take() {
Some(response) => Some(response),
None => match &call.options.cache {
Some(cache) => cache.get().await,
None => None,
},
};
if let Some(response) = cached {
if !stream
&& let Ok(message) =
serde_json::from_value::<AnthropicMessagesResponse>(response.clone())
{
call.options
.record_messages_usage(message.usage.as_ref().unwrap_or(&Value::Null), true);
return Ok(MessagesOutput::Message(Box::new(message)));
}
if stream
&& let Some(events) = response
.get("litellm_cached_anthropic_sse_events")
.and_then(Value::as_array)
&& events.iter().all(Value::is_string)
{
if host.open(()).await? == Demand::More {
for event in events {
if host
.deliver(Bytes::from(event.as_str().unwrap_or_default().to_owned()))
.await?
== Demand::Detached
{
break;
}
}
}
return Ok(MessagesOutput::Streamed);
}
}
call.options.adjust_max_tokens(&mut call.body);
let request = prepare_provider_request(MessagesRequest {
model: &call.model,
body: Value::Object(call.body.clone()),
@ -167,15 +209,22 @@ async fn execute(host: MessagesHost) -> Result<MessagesOutput, Error> {
return Err(provider_error(response).await);
}
if stream {
return relay(&host, response).await;
return relay(&host, response, &call.options).await;
}
let text = response.text().await.map_err(network)?;
host.emit(MachineEvent::ResponseReceived {
raw: RawResponse { body: text.clone() },
})
.await?;
decode_response(request.config, &request.model, &text)
.map(|message| MessagesOutput::Message(Box::new(message)))
let message = decode_response(request.config, &request.model, &text)?;
call.options
.record_messages_usage(message.usage.as_ref().unwrap_or(&Value::Null), false);
if let Some(cache) = &call.options.cache
&& let Ok(value) = serde_json::to_value(&message)
{
cache.set(value).await;
}
Ok(MessagesOutput::Message(Box::new(message)))
}
/// Hands each upstream chunk to the caller as it arrives. A caller that stops reading
@ -183,14 +232,26 @@ async fn execute(host: MessagesHost) -> Result<MessagesOutput, Error> {
async fn relay(
host: &MessagesHost,
mut response: reqwest::Response,
options: &crate::client::ClientOptions,
) -> Result<MessagesOutput, Error> {
if host.open(()).await? == Demand::Detached {
return Ok(MessagesOutput::Streamed);
}
let mut collected = Vec::new();
let mut detached = false;
while let Some(chunk) = response.chunk().await.map_err(network)? {
collected.extend_from_slice(&chunk);
if host.deliver(chunk).await? == Demand::Detached {
detached = true;
break;
}
}
if let Some(stream) = litellm_core_utils::messages_stream::MessagesStream::parse(&collected) {
options.record_messages_usage(&stream.usage, false);
if !detached && let (Some(cache), Some(response)) = (&options.cache, stream.cached_response)
{
cache.set(response).await;
}
}
Ok(MessagesOutput::Streamed)
}

View file

@ -15,7 +15,7 @@ use super::{
};
use crate::messages::types::MessagesRequest;
async fn read_http_request(socket: &mut TcpStream) -> String {
pub(super) async fn read_http_request(socket: &mut TcpStream) -> String {
let mut request = Vec::new();
let mut buffer = [0_u8; 1024];
let header_end = loop {
@ -48,7 +48,7 @@ async fn read_http_request(socket: &mut TcpStream) -> String {
String::from_utf8(request).expect("request is utf8")
}
fn write_response(body: &str) -> String {
pub(super) fn write_response(body: &str) -> String {
format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),

View file

@ -14,6 +14,7 @@ pub fn missing_state() -> PyErr {
pub enum LifecycleStep {
Await(Py<PyAny>),
Arguments(Py<PyDict>),
Cached(Py<PyAny>),
Wire(Box<WireRequest>),
Response(Py<PyAny>),
Done,

View file

@ -207,6 +207,10 @@ where
self.stage = Stage::Call;
self.resume_machine(py, None)
}
(Expect::Arguments, LifecycleStep::Cached(response)) => {
self.ended_at = Some(epoch_seconds());
self.succeeded(py, response)
}
(Expect::Wire, LifecycleStep::Wire(wire)) => {
self.resume_machine(py, Some(Ok(HostResult::BeforeSend(wire))))
}

View file

@ -85,6 +85,11 @@ impl MessagesRouteHost {
.transpose()?
.flatten();
Ok(MessagesCall {
cached_response: arguments
.get_item("_rust_messages_cached_response")?
.map(|value| from_py(&value))
.transpose()?,
options: Default::default(),
model,
body,
api_key: string("api_key")?,

View file

@ -0,0 +1,107 @@
import asyncio
import json
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import litellm
from litellm.caching.caching import Cache
from litellm.rust_bridge import _native
from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest
class Upstream(BaseHTTPRequestHandler):
calls = []
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["content-length"])))
self.calls.append(body)
usage = {"input_tokens": 2, "output_tokens": 3, "cache_read_input_tokens": 4, "cache_creation_input_tokens": 5}
message = {"id": "msg_native", "type": "message", "role": "assistant", "model": "test-native-sdk",
"content": [{"type": "text", "text": "hello"}], "stop_reason": "end_turn", "stop_sequence": None,
"usage": usage}
if body.get("stream"):
events = ({"type": "message_start", "message": {**message, "usage": {**usage, "output_tokens": 0}}},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 3}},
{"type": "message_stop"})
response = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode()
else:
response = json.dumps(message).encode()
self.send_response(200)
self.send_header("content-type", "text/event-stream" if body.get("stream") else "application/json")
self.send_header("content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def log_message(self, *_args):
pass
class MessagesLifecycle(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.server = ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.original = (litellm.cache, litellm.max_budget, litellm._current_cost, litellm.modify_params)
litellm.cache = Cache(type="local", supported_call_types=["anthropic_messages"])
litellm.max_budget = 100.0
litellm._current_cost = 0.0
litellm.modify_params = False
litellm.register_model({"test-native-sdk": {"litellm_provider": "anthropic", "mode": "chat",
"input_cost_per_token": 2.0, "output_cost_per_token": 3.0,
"cache_read_input_token_cost": 0.5, "cache_creation_input_token_cost": 4.0,
"max_input_tokens": 200, "max_output_tokens": 50}})
self.before = len(Upstream.calls)
async def asyncTearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join()
litellm.cache, litellm.max_budget, litellm._current_cost, litellm.modify_params = self.original
litellm.model_cost.pop("test-native-sdk", None)
def call(self, asynchronous=True, **extra):
kwargs = {"max_tokens": 100, "messages": [{"role": "user", "content": "hello"}],
"model": "test-native-sdk", "custom_llm_provider": "anthropic", "api_key": "test",
"api_base": f"http://127.0.0.1:{self.server.server_port}", **extra}
request = LiteLLMMessagesRequest(model=kwargs["model"], messages=kwargs["messages"], max_tokens=100,
stream=kwargs.get("stream"), api_key="test", api_base=kwargs["api_base"],
custom_llm_provider="anthropic", kwargs=kwargs)
return (_native.amessages if asynchronous else _native.messages)(request, (), kwargs)
async def test_cache_hit_charges_once_and_budget_precedes_cache(self):
first = await self.call()
second = await self.call()
self.assertEqual(first, second)
self.assertEqual(len(Upstream.calls) - self.before, 1)
cost = 2 * 2.0 + 3 * 3.0 + 4 * 0.5 + 5 * 4.0
self.assertEqual(litellm._current_cost, cost)
litellm.max_budget = cost - 1
with self.assertRaises(litellm.BudgetExceededError):
await self.call()
self.assertEqual(len(Upstream.calls) - self.before, 1)
async def test_stream_cache_replays_and_charges_once(self):
first = b"".join([chunk async for chunk in await self.call(stream=True)])
second = b"".join([chunk async for chunk in await self.call(stream=True)])
self.assertEqual(first, second)
self.assertEqual(len(Upstream.calls) - self.before, 1)
self.assertEqual(litellm._current_cost, 2 * 2.0 + 3 * 3.0 + 4 * 0.5 + 5 * 4.0)
async def test_opt_in_output_cap_is_sent_and_cached_under_original_request(self):
litellm.modify_params = True
await self.call()
await self.call()
self.assertEqual(len(Upstream.calls) - self.before, 1)
self.assertEqual(Upstream.calls[-1]["max_tokens"], 50)
async def test_sync_messages_use_the_same_cache_and_budget(self):
first = self.call(asynchronous=False)
second = await self.call()
self.assertEqual(first, second)
self.assertEqual(len(Upstream.calls) - self.before, 1)
if __name__ == "__main__":
print(f"native extension: {_native.__file__}")
unittest.main()

View file

@ -504,6 +504,7 @@ class Logging(LiteLLMLoggingBaseClass):
classifier_input: Mapping[str, JsonValue] | None = None
baseline_cache_context: "BaselineCacheContext | None" = None
baseline_observation: "CapturedBaselineObservation | None" = None
_native_response_cost: float | None = None
def __init__(
self,
@ -1744,6 +1745,9 @@ class Logging(LiteLLMLoggingBaseClass):
if cache_hit is True:
return 0.0
if self._native_response_cost is not None:
return self._native_response_cost
if is_unbilled_non_inference_call(
self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
):

View file

@ -64,21 +64,6 @@ def setup(
return CallSetup(logger, prepared)
def check_limits(kwargs: Mapping[str, object]) -> None:
from litellm import (
BudgetExceededError,
_current_cost, # pyright: ignore[reportPrivateUsage] # shared SDK budget counter has no public accessor
max_budget,
num_retries_per_request,
)
from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit
if max_budget and _current_cost > max_budget:
raise BudgetExceededError(current_cost=_current_cost, max_budget=max_budget)
if max_retries_per_request_hit(kwargs, num_retries_per_request):
raise RuntimeError("Max retries per request hit!")
def finalize(
response: object,
logger: Logging,

View file

@ -11,7 +11,7 @@ from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.rust_bridge import callbacks_legacy_python as legacy
from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup
from litellm.rust_bridge.callbacks_legacy_python import setup
_OCR_KWARGS: Final = MappingProxyType(
{
@ -21,33 +21,6 @@ _OCR_KWARGS: Final = MappingProxyType(
)
@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"])
@pytest.mark.parametrize(
"cap, request_retry_count, refused",
[(5, 5, True), (5, 4, False), (0, 0, False), (0, 1, True)],
ids=[
"cap-above-four-reached",
"cap-above-four-not-reached",
"first-attempt-passes-cap-of-zero",
"cap-of-zero-refuses-first-retry",
],
)
def test_check_limits_reads_request_retry_count(
monkeypatch: pytest.MonkeyPatch, metadata_key: str, cap: int, request_retry_count: int, refused: bool
) -> None:
monkeypatch.setattr(litellm, "num_retries_per_request", cap)
monkeypatch.setattr(litellm, "max_budget", None)
kwargs: Final = {
"model": "mistral/mistral-ocr-latest",
metadata_key: {"request_retry_count": request_retry_count},
}
if refused:
with pytest.raises(RuntimeError, match="Max retries per request hit!"):
check_limits(kwargs)
else:
check_limits(kwargs)
def _supplied_logger() -> Logging:
return Logging(
model="mistral/mistral-ocr-latest",