mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
feat(rust): add messages SDK lifecycle
This commit is contained in:
parent
9599cc0cb4
commit
cdca831c88
33 changed files with 1550 additions and 70 deletions
7
litellm-rust/Cargo.lock
generated
7
litellm-rust/Cargo.lock
generated
|
|
@ -2067,9 +2067,13 @@ dependencies = [
|
|||
name = "litellm-callbacks-legacy-python"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"litellm-auth",
|
||||
"litellm-cache",
|
||||
"litellm-core-utils",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
"litellm-token-counter",
|
||||
"proptest",
|
||||
"pyo3",
|
||||
"rstest",
|
||||
|
|
@ -2087,10 +2091,13 @@ dependencies = [
|
|||
"litellm-auth",
|
||||
"litellm-auth-aws",
|
||||
"litellm-auth-gcp",
|
||||
"litellm-cache",
|
||||
"litellm-cache-memory",
|
||||
"litellm-core-utils",
|
||||
"litellm-host",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
"litellm-token-counter",
|
||||
"litellm-types",
|
||||
"mime_guess",
|
||||
"moka",
|
||||
|
|
|
|||
48
litellm-rust/crates/cache/src/handler.rs
vendored
Normal file
48
litellm-rust/crates/cache/src/handler.rs
vendored
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{CacheBackend, CacheControls, CacheEntry, CacheKwargs};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LLMCachingHandler {
|
||||
pub backend: CacheBackend,
|
||||
pub controls: CacheControls,
|
||||
pub key: String,
|
||||
pub kwargs: CacheKwargs,
|
||||
pub max_age: Option<Duration>,
|
||||
}
|
||||
|
||||
impl LLMCachingHandler {
|
||||
pub async fn get(&self) -> Option<Value> {
|
||||
if !self.controls.reads() {
|
||||
return None;
|
||||
}
|
||||
let entry = self
|
||||
.backend
|
||||
.async_get_cache(&self.key, &self.kwargs)
|
||||
.await
|
||||
.ok()??;
|
||||
entry.fresh(now(), self.max_age).then_some(entry.response)
|
||||
}
|
||||
|
||||
pub async fn set(&self, response: Value) {
|
||||
if !self.controls.writes() {
|
||||
return;
|
||||
}
|
||||
let entry = CacheEntry {
|
||||
timestamp: now().as_secs_f64(),
|
||||
response,
|
||||
};
|
||||
let _ = self
|
||||
.backend
|
||||
.async_set_cache(&self.key, entry, self.kwargs.clone())
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn now() -> Duration {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
2
litellm-rust/crates/cache/src/lib.rs
vendored
2
litellm-rust/crates/cache/src/lib.rs
vendored
|
|
@ -1,6 +1,8 @@
|
|||
mod base_cache;
|
||||
mod caching;
|
||||
mod error;
|
||||
mod handler;
|
||||
pub use handler::LLMCachingHandler;
|
||||
|
||||
pub use base_cache::{
|
||||
BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheFuture, CacheKwargs,
|
||||
|
|
|
|||
|
|
@ -9,8 +9,12 @@ autotests = false
|
|||
[dependencies]
|
||||
litellm-host.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
litellm-token-counter.workspace = true
|
||||
|
||||
pyo3.workspace = true
|
||||
base64.workspace = true
|
||||
strum.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
|
|
|
|||
|
|
@ -6,9 +6,6 @@
|
|||
"start_time",
|
||||
"asynchronous"
|
||||
],
|
||||
"check_limits": [
|
||||
"kwargs"
|
||||
],
|
||||
"finalize": [
|
||||
"response",
|
||||
"logger",
|
||||
|
|
|
|||
|
|
@ -49,6 +49,9 @@ struct DeliveredStream {
|
|||
}
|
||||
|
||||
enum Pending {
|
||||
CacheRead,
|
||||
CacheWrite,
|
||||
StreamCacheWrite,
|
||||
DeploymentPreCall,
|
||||
DeploymentPostCall,
|
||||
DeploymentFailure,
|
||||
|
|
@ -56,6 +59,9 @@ enum Pending {
|
|||
}
|
||||
|
||||
pub struct LegacyLogging {
|
||||
cache: Option<crate::caching::Caching>,
|
||||
cache_hit: bool,
|
||||
accounted: bool,
|
||||
surface: LegacySurface,
|
||||
call: PublicCall,
|
||||
logger: Option<PythonLogger>,
|
||||
|
|
@ -88,6 +94,9 @@ impl LegacyLogging {
|
|||
asynchronous: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
cache: None,
|
||||
cache_hit: false,
|
||||
accounted: false,
|
||||
surface,
|
||||
call,
|
||||
logger: None,
|
||||
|
|
@ -120,9 +129,123 @@ impl LegacyLogging {
|
|||
fn prepare(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
|
||||
let prepared = prepare(py, self.call.kwargs().bind(py), self.logger()?)?.unbind();
|
||||
self.call.set_kwargs(prepared);
|
||||
if self.surface.call_type == "anthropic_messages" {
|
||||
self.cache = crate::caching::Caching::prepare(
|
||||
py,
|
||||
&self.call,
|
||||
self.surface.call_type,
|
||||
self.asynchronous,
|
||||
)?;
|
||||
if let Some(cache) = &self.cache
|
||||
&& cache.controls.reads()
|
||||
{
|
||||
match cache.read(py, self.asynchronous) {
|
||||
Ok(value) if self.asynchronous => {
|
||||
self.pending = Some(Pending::CacheRead);
|
||||
return Ok(LifecycleStep::Await(value));
|
||||
}
|
||||
result => return self.cache_read(py, result),
|
||||
}
|
||||
}
|
||||
}
|
||||
self.arguments(py)
|
||||
}
|
||||
|
||||
fn arguments(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
|
||||
if self.surface.call_type == "anthropic_messages" {
|
||||
if let Err(error) =
|
||||
crate::limits::adjust_max_tokens(py, &self.call, self.surface.call_type)
|
||||
&& is_cancellation(py, &error)
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py)))
|
||||
}
|
||||
|
||||
fn cache_read(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
result: PyResult<Py<PyAny>>,
|
||||
) -> PyResult<LifecycleStep> {
|
||||
let decoded = result.and_then(|value| {
|
||||
self.cache
|
||||
.as_ref()
|
||||
.ok_or_else(missing_state)?
|
||||
.decode(value.bind(py))
|
||||
});
|
||||
let response = match decoded {
|
||||
Ok(Some(response)) => response,
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
_ => return self.arguments(py),
|
||||
};
|
||||
self.cache_hit = true;
|
||||
self.logger()?
|
||||
.object(py)
|
||||
.getattr("model_call_details")?
|
||||
.set_item("cache_hit", true)?;
|
||||
let cache = self.cache.as_ref().ok_or_else(missing_state)?;
|
||||
self.logger()?
|
||||
.object(py)
|
||||
.getattr("litellm_params")?
|
||||
.set_item("preset_cache_key", &cache.key)?;
|
||||
if response
|
||||
.get("litellm_cached_anthropic_sse_events")
|
||||
.is_some()
|
||||
{
|
||||
self.call
|
||||
.kwargs()
|
||||
.bind(py)
|
||||
.set_item("_rust_messages_cached_response", to_py(py, &response)?)?;
|
||||
return Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py)));
|
||||
}
|
||||
self.end = Some(datetime(py, epoch_seconds())?);
|
||||
self.response = Some(crate::caching::cached_public(py, &response)?);
|
||||
self.finalize(py)?;
|
||||
Ok(LifecycleStep::Cached(
|
||||
self.response
|
||||
.as_ref()
|
||||
.ok_or_else(missing_state)?
|
||||
.clone_ref(py),
|
||||
))
|
||||
}
|
||||
|
||||
fn cache_write(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
|
||||
if let (Some(cache), Some(response)) = (&self.cache, &self.response)
|
||||
&& cache.controls.writes()
|
||||
&& !self.cache_hit
|
||||
{
|
||||
match cache.write(py, response.bind(py), self.asynchronous) {
|
||||
Ok(awaitable) if self.asynchronous => {
|
||||
self.pending = Some(Pending::CacheWrite);
|
||||
return Ok(LifecycleStep::Await(awaitable));
|
||||
}
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
self.finalize(py)
|
||||
}
|
||||
|
||||
fn dispatch_cache_hit(&self, py: Python<'_>) -> PyResult<()> {
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("result", &self.response)?;
|
||||
kwargs.set_item("start_time", &self.start)?;
|
||||
kwargs.set_item("end_time", &self.end)?;
|
||||
kwargs.set_item("cache_hit", true)?;
|
||||
let logger = self.logger()?.object(py);
|
||||
if self.asynchronous {
|
||||
let coroutine = logger.call_method("async_success_handler", (), Some(&kwargs))?;
|
||||
crate::python::Logging::Enqueue.call(py, (coroutine,))?;
|
||||
}
|
||||
logger.call_method(
|
||||
"handle_sync_success_callbacks_for_async_calls",
|
||||
(),
|
||||
Some(&kwargs),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finalize(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
|
||||
finalize(
|
||||
py,
|
||||
|
|
@ -138,6 +261,68 @@ impl LegacyLogging {
|
|||
.ok_or_else(missing_state)
|
||||
}
|
||||
|
||||
fn record_usage(&mut self, py: Python<'_>, usage: &Value) -> PyResult<()> {
|
||||
if self.accounted || self.surface.call_type != "anthropic_messages" {
|
||||
return Ok(());
|
||||
}
|
||||
self.accounted = true;
|
||||
let model = self
|
||||
.call
|
||||
.lookup(py, "model")?
|
||||
.map(|value| value.extract::<String>())
|
||||
.transpose()?;
|
||||
let info = model
|
||||
.map(|model| crate::limits::model_info(py, &model))
|
||||
.transpose()?
|
||||
.flatten();
|
||||
let cost = if self.cache_hit {
|
||||
Some(0.0)
|
||||
} else {
|
||||
info.as_ref()
|
||||
.and_then(|info| litellm_core_utils::cost_calculator::messages_cost(info, usage))
|
||||
};
|
||||
if let Some(cost) = cost {
|
||||
let logger = self.logger()?.object(py);
|
||||
logger.setattr("_native_response_cost", cost)?;
|
||||
logger
|
||||
.getattr("model_call_details")?
|
||||
.set_item("response_cost", cost)?;
|
||||
let sdk = py.import("litellm")?;
|
||||
if !self.cache_hit && sdk.getattr("max_budget")?.is_truthy()? {
|
||||
let current: f64 = sdk.getattr("_current_cost")?.extract()?;
|
||||
sdk.setattr("_current_cost", current + cost)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish_stream(&mut self, py: Python<'_>) -> PyResult<LifecycleStep> {
|
||||
let stream = self.stream.as_ref().ok_or_else(missing_state)?;
|
||||
let mut bytes = Vec::new();
|
||||
for chunk in stream.chunks.bind(py).iter() {
|
||||
bytes.extend_from_slice(chunk.cast::<pyo3::types::PyBytes>()?.as_bytes());
|
||||
}
|
||||
if let Some(parsed) = litellm_core_utils::messages_stream::MessagesStream::parse(&bytes) {
|
||||
self.record_usage(py, &parsed.usage)?;
|
||||
if !self.cache_hit
|
||||
&& let (Some(cache), Some(response)) = (&self.cache, parsed.cached_response)
|
||||
&& cache.controls.writes()
|
||||
{
|
||||
let response = to_py(py, &response)?;
|
||||
match cache.write(py, response.bind(py), self.asynchronous) {
|
||||
Ok(awaitable) if self.asynchronous => {
|
||||
self.pending = Some(Pending::StreamCacheWrite);
|
||||
return Ok(LifecycleStep::Await(awaitable));
|
||||
}
|
||||
Err(error) if is_cancellation(py, &error) => return Err(error),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.stream_success(py, self.stream.as_ref().ok_or_else(missing_state)?)?;
|
||||
Ok(LifecycleStep::Done)
|
||||
}
|
||||
|
||||
fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
|
||||
match self.try_dispatch_success(py) {
|
||||
Err(error) if error.is_instance_of::<PyException>(py) => {
|
||||
|
|
@ -149,6 +334,9 @@ impl LegacyLogging {
|
|||
}
|
||||
|
||||
fn try_dispatch_success(&self, py: Python<'_>) -> PyResult<()> {
|
||||
if self.cache_hit {
|
||||
return self.dispatch_cache_hit(py);
|
||||
}
|
||||
let logger = self.logger()?;
|
||||
let pending = || PendingSuccess {
|
||||
logger: logger.clone_ref(py),
|
||||
|
|
@ -361,7 +549,7 @@ impl PythonLifecycle for LegacyLogging {
|
|||
self.surface.call_type,
|
||||
)?));
|
||||
}
|
||||
self.finalize(py)
|
||||
self.cache_write(py)
|
||||
}
|
||||
|
||||
fn emit(&mut self, py: Python<'_>, event: LifecycleEvent<'_>) -> PyResult<LifecycleStep> {
|
||||
|
|
@ -385,10 +573,14 @@ impl PythonLifecycle for LegacyLogging {
|
|||
LifecycleEvent::Succeeded { timing, response } => {
|
||||
self.end = Some(datetime(py, timing.end_time)?);
|
||||
self.response = Some(response.clone_ref(py));
|
||||
match &self.stream {
|
||||
Some(stream) => self.stream_success(py, stream)?,
|
||||
None => self.dispatch_success(py)?,
|
||||
if self.stream.is_some() {
|
||||
return self.finish_stream(py);
|
||||
}
|
||||
if self.surface.call_type == "anthropic_messages" {
|
||||
let value: Value = from_py(response.bind(py))?;
|
||||
self.record_usage(py, value.get("usage").unwrap_or(&Value::Null))?;
|
||||
}
|
||||
self.dispatch_success(py)?;
|
||||
Ok(LifecycleStep::Done)
|
||||
}
|
||||
LifecycleEvent::Failed {
|
||||
|
|
@ -448,8 +640,28 @@ impl PythonLifecycle for LegacyLogging {
|
|||
}
|
||||
Pending::DeploymentPostCall => {
|
||||
self.response = Some(result?);
|
||||
self.cache_write(py)
|
||||
}
|
||||
Pending::CacheRead => self.cache_read(py, result),
|
||||
Pending::CacheWrite => {
|
||||
if let Err(error) = result
|
||||
&& is_cancellation(py, &error)
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
self.finalize(py)
|
||||
}
|
||||
Pending::StreamCacheWrite => {
|
||||
if let Err(error) = result
|
||||
&& is_cancellation(py, &error)
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
if let Some(stream) = &self.stream {
|
||||
self.stream_success(py, stream)?;
|
||||
}
|
||||
Ok(LifecycleStep::Done)
|
||||
}
|
||||
Pending::DeploymentFailure => self.dispatch_failure(py),
|
||||
Pending::AsyncFailure => match result {
|
||||
Err(failure) if is_cancellation(py, &failure) => Err(failure),
|
||||
|
|
@ -467,10 +679,14 @@ impl PythonLifecycle for LegacyLogging {
|
|||
self.body = None;
|
||||
self.context = None;
|
||||
self.stream = None;
|
||||
self.cache = None;
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
self.call.traverse(visit)?;
|
||||
if let Some(cache) = &self.cache {
|
||||
cache.traverse(visit)?;
|
||||
}
|
||||
if let Some(logger) = &self.logger {
|
||||
logger.traverse(visit)?;
|
||||
}
|
||||
|
|
|
|||
291
litellm-rust/crates/callbacks-legacy-python/src/caching.rs
Normal file
291
litellm-rust/crates/callbacks-legacy-python/src/caching.rs
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
use litellm_cache::{CacheControls, CacheKeyField, CacheKeyInput, cache_key};
|
||||
use litellm_host_python::{from_py, to_py};
|
||||
use pyo3::{
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
prelude::*,
|
||||
types::PyDict,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::PublicCall;
|
||||
|
||||
pub(crate) struct Caching {
|
||||
backend: Py<PyAny>,
|
||||
pub key: String,
|
||||
pub controls: CacheControls,
|
||||
kwargs: Py<PyDict>,
|
||||
max_age: Option<f64>,
|
||||
}
|
||||
|
||||
impl Caching {
|
||||
pub fn prepare(
|
||||
py: Python<'_>,
|
||||
call: &PublicCall,
|
||||
call_type: &str,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Option<Self>> {
|
||||
let sdk = py.import("litellm")?;
|
||||
let cache = sdk.getattr("cache")?;
|
||||
if cache.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let supported = cache.getattr("supported_call_types")?;
|
||||
if supported.is_none() || !supported.contains(call_type)? {
|
||||
return Ok(None);
|
||||
}
|
||||
let kwargs = call.kwargs().bind(py).copy()?;
|
||||
let names = [
|
||||
"max_tokens",
|
||||
"messages",
|
||||
"model",
|
||||
"metadata",
|
||||
"stop_sequences",
|
||||
"stream",
|
||||
"system",
|
||||
"temperature",
|
||||
"thinking",
|
||||
"tool_choice",
|
||||
"tools",
|
||||
"top_k",
|
||||
"top_p",
|
||||
"container",
|
||||
"api_key",
|
||||
"api_base",
|
||||
"client",
|
||||
"custom_llm_provider",
|
||||
];
|
||||
for (name, value) in names
|
||||
.iter()
|
||||
.filter(|name| !asynchronous || **name != "container")
|
||||
.zip(call.args().bind(py).iter())
|
||||
{
|
||||
kwargs.set_item(name, value)?;
|
||||
}
|
||||
let directives = kwargs
|
||||
.get_item("cache")?
|
||||
.unwrap_or_else(|| PyDict::new(py).into_any());
|
||||
let directive = |name: &str| directives.call_method1("get", (name,));
|
||||
let is_true =
|
||||
|value: Bound<'_, PyAny>| value.is(&true.into_pyobject(py).expect("bool conversion"));
|
||||
let controls = CacheControls {
|
||||
supported_call_type: true,
|
||||
configured: true,
|
||||
native_backend: false,
|
||||
default_on: cache.getattr("mode")?.eq("default_on")?,
|
||||
caching: kwargs
|
||||
.get_item("caching")?
|
||||
.filter(|v| !v.is_none())
|
||||
.map(is_true),
|
||||
no_cache: is_true(directive("no-cache")?),
|
||||
no_store: is_true(directive("no-store")?),
|
||||
use_cache: is_true(directive("use-cache")?),
|
||||
};
|
||||
if !controls.reads() && !controls.writes() {
|
||||
return Ok(None);
|
||||
}
|
||||
let metadata = kwargs
|
||||
.get_item("metadata")?
|
||||
.filter(|value| !value.is_none())
|
||||
.unwrap_or_else(|| PyDict::new(py).into_any());
|
||||
let params = kwargs
|
||||
.get_item("litellm_params")?
|
||||
.filter(|value| !value.is_none())
|
||||
.unwrap_or_else(|| PyDict::new(py).into_any());
|
||||
let preset = kwargs
|
||||
.get_item("cache_key")?
|
||||
.filter(|value| !value.is_none())
|
||||
.or_else(|| {
|
||||
params
|
||||
.call_method1("get", ("preset_cache_key",))
|
||||
.ok()
|
||||
.filter(|value| !value.is_none())
|
||||
});
|
||||
let key = if let Some(preset) = preset {
|
||||
preset.extract()?
|
||||
} else {
|
||||
let api_params = py
|
||||
.import("litellm.litellm_core_utils.model_param_helper")?
|
||||
.getattr("ModelParamHelper")?
|
||||
.call_method0("_get_all_llm_api_params")?;
|
||||
let internal = py
|
||||
.import("litellm.types.utils")?
|
||||
.getattr("all_litellm_params")?;
|
||||
let include_provider_parameters = sdk
|
||||
.getattr("enable_caching_on_provider_specific_optional_params")?
|
||||
.is_truthy()?;
|
||||
let model_group = metadata.call_method1("get", ("model_group",))?;
|
||||
let nested_metadata = params.call_method1("get", ("metadata", PyDict::new(py)))?;
|
||||
let model_group = if model_group.is_truthy()? {
|
||||
model_group
|
||||
} else if nested_metadata.is_none() {
|
||||
py.None().into_bound(py)
|
||||
} else {
|
||||
nested_metadata.call_method1("get", ("model_group",))?
|
||||
};
|
||||
let groups = metadata.call_method1("get", ("caching_groups",))?;
|
||||
let mut group = None;
|
||||
if groups.is_truthy()? {
|
||||
for item in groups.try_iter()? {
|
||||
let item = item?;
|
||||
if item.contains(&model_group)? {
|
||||
group = Some(item.str()?.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let fields = kwargs
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
let name: String = name.extract()?;
|
||||
let api_parameter = api_params.contains(&name)?;
|
||||
let internal_parameter = internal.contains(&name)?;
|
||||
let value = if value.is_none()
|
||||
|| (!api_parameter && (!include_provider_parameters || internal_parameter))
|
||||
{
|
||||
None
|
||||
} else if name == "model" && group.is_some() {
|
||||
group.clone()
|
||||
} else if name == "model" && model_group.is_truthy()? {
|
||||
Some(model_group.str()?.to_string())
|
||||
} else {
|
||||
Some(value.str()?.to_string())
|
||||
};
|
||||
Ok(CacheKeyField {
|
||||
name,
|
||||
value,
|
||||
api_parameter,
|
||||
internal_parameter,
|
||||
})
|
||||
})
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
let namespace = [
|
||||
directive("namespace")?,
|
||||
metadata.call_method1("get", ("redis_namespace",))?,
|
||||
cache.getattr("namespace")?,
|
||||
]
|
||||
.into_iter()
|
||||
.find(|value| value.is_truthy().unwrap_or(false))
|
||||
.map(|value| value.extract())
|
||||
.transpose()?;
|
||||
cache_key(&CacheKeyInput {
|
||||
fields,
|
||||
preset: None,
|
||||
namespace,
|
||||
include_provider_parameters,
|
||||
})
|
||||
};
|
||||
let backend_kwargs = PyDict::new(py);
|
||||
for name in [
|
||||
"messages",
|
||||
"input",
|
||||
"metadata",
|
||||
"litellm_params",
|
||||
"parent_otel_span",
|
||||
] {
|
||||
if let Some(value) = kwargs.get_item(name)? {
|
||||
backend_kwargs.set_item(name, value)?;
|
||||
}
|
||||
}
|
||||
let ttl = directive("ttl")?;
|
||||
let ttl = if !ttl.is_none() {
|
||||
Some(ttl)
|
||||
} else {
|
||||
let configured = cache.getattr("ttl")?;
|
||||
if configured.is_none() {
|
||||
kwargs.get_item("ttl")?
|
||||
} else {
|
||||
Some(configured)
|
||||
}
|
||||
};
|
||||
if let Some(ttl) = ttl {
|
||||
backend_kwargs.set_item("ttl", ttl)?;
|
||||
}
|
||||
let max_age = if asynchronous {
|
||||
directives.call_method1("get", ("s-max-age", directive("s-maxage")?))?
|
||||
} else {
|
||||
let first = directive("s-maxage")?;
|
||||
if first.is_truthy()? {
|
||||
first
|
||||
} else {
|
||||
directive("s-max-age")?
|
||||
}
|
||||
};
|
||||
Ok(Some(Self {
|
||||
backend: cache.getattr("cache")?.unbind(),
|
||||
key,
|
||||
controls,
|
||||
kwargs: backend_kwargs.unbind(),
|
||||
max_age: max_age.extract()?,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn read(&self, py: Python<'_>, asynchronous: bool) -> PyResult<Py<PyAny>> {
|
||||
self.backend
|
||||
.bind(py)
|
||||
.call_method(
|
||||
if asynchronous {
|
||||
"async_get_cache"
|
||||
} else {
|
||||
"get_cache"
|
||||
},
|
||||
(&self.key,),
|
||||
Some(self.kwargs.bind(py)),
|
||||
)
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
pub fn decode(&self, value: &Bound<'_, PyAny>) -> PyResult<Option<Value>> {
|
||||
if value.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let value: Value = from_py(value)?;
|
||||
let response = if let Some(timestamp) = value.get("timestamp").and_then(Value::as_f64) {
|
||||
if self
|
||||
.max_age
|
||||
.is_some_and(|max| litellm_host::event::epoch_seconds() - timestamp > max)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
value.get("response").cloned().unwrap_or(Value::Null)
|
||||
} else {
|
||||
value
|
||||
};
|
||||
let response = match response {
|
||||
Value::String(text) => serde_json::from_str(&text).unwrap_or(Value::Null),
|
||||
value => value,
|
||||
};
|
||||
Ok(response.is_object().then_some(response))
|
||||
}
|
||||
|
||||
pub fn write(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
response: &Bound<'_, PyAny>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let entry = PyDict::new(py);
|
||||
entry.set_item("timestamp", litellm_host::event::epoch_seconds())?;
|
||||
entry.set_item("response", response)?;
|
||||
self.backend
|
||||
.bind(py)
|
||||
.call_method(
|
||||
if asynchronous {
|
||||
"async_set_cache"
|
||||
} else {
|
||||
"set_cache"
|
||||
},
|
||||
(&self.key, entry),
|
||||
Some(self.kwargs.bind(py)),
|
||||
)
|
||||
.map(Bound::unbind)
|
||||
}
|
||||
|
||||
pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
visit.call(&self.backend)?;
|
||||
visit.call(&self.kwargs)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn cached_public(py: Python<'_>, value: &Value) -> PyResult<Py<PyAny>> {
|
||||
to_py(py, value)
|
||||
}
|
||||
|
|
@ -10,9 +10,11 @@
|
|||
//! without keeping a copy.
|
||||
|
||||
mod adapter;
|
||||
mod caching;
|
||||
mod call;
|
||||
mod callbacks;
|
||||
mod deferred;
|
||||
mod limits;
|
||||
mod logger;
|
||||
mod preparation;
|
||||
mod python;
|
||||
|
|
|
|||
107
litellm-rust/crates/callbacks-legacy-python/src/limits.rs
Normal file
107
litellm-rust/crates/callbacks-legacy-python/src/limits.rs
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
use std::sync::OnceLock;
|
||||
|
||||
use litellm_core_utils::token_counter::get_modified_max_tokens;
|
||||
use litellm_host_python::from_py;
|
||||
use litellm_token_counter::{CountableRequest, TokenCounter};
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::PublicCall;
|
||||
|
||||
pub(crate) fn adjust_max_tokens(
|
||||
py: Python<'_>,
|
||||
call: &PublicCall,
|
||||
call_type: &str,
|
||||
) -> PyResult<()> {
|
||||
if !matches!(
|
||||
call_type,
|
||||
"completion" | "acompletion" | "anthropic_messages"
|
||||
) {
|
||||
return Ok(());
|
||||
}
|
||||
let sdk = py.import("litellm")?;
|
||||
if !sdk.getattr("modify_params")?.is(&true.into_pyobject(py)?) {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(requested) = call
|
||||
.lookup(py, "max_tokens")?
|
||||
.filter(|value| !value.is_none())
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let requested: i64 = requested.extract()?;
|
||||
let Some(model) = call.lookup(py, "model")? else {
|
||||
return Ok(());
|
||||
};
|
||||
let model: String = model.extract()?;
|
||||
let info = model_info(py, &model)?;
|
||||
let Some(info) = info else {
|
||||
return Ok(());
|
||||
};
|
||||
let counter = counter(py, &model)?;
|
||||
let messages = if call.args().bind(py).len() > 1 {
|
||||
call.args().bind(py).get_item(1)?
|
||||
} else {
|
||||
call.lookup(py, "messages")?
|
||||
.unwrap_or_else(|| py.None().into_bound(py))
|
||||
};
|
||||
let request = PyDict::new(py);
|
||||
request.set_item("model", &model)?;
|
||||
request.set_item("messages", messages)?;
|
||||
let request: CountableRequest = from_py(&request)?;
|
||||
let count = py
|
||||
.detach(|| counter.count_request(&request))
|
||||
.map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?;
|
||||
let adjusted = get_modified_max_tokens(
|
||||
requested,
|
||||
info.get("max_input_tokens").and_then(Value::as_i64),
|
||||
info.get("max_output_tokens")
|
||||
.or_else(|| info.get("max_tokens"))
|
||||
.and_then(Value::as_i64),
|
||||
count.input_tokens,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
call.kwargs().bind(py).set_item("max_tokens", adjusted)
|
||||
}
|
||||
|
||||
pub(crate) fn model_info(py: Python<'_>, model: &str) -> PyResult<Option<Value>> {
|
||||
let sdk = py.import("litellm")?;
|
||||
let costs = sdk.getattr("model_cost")?;
|
||||
let aliases = sdk.getattr("model_alias_map")?;
|
||||
let resolved = aliases.call_method1("get", (model, model))?;
|
||||
let resolved: String = resolved.extract()?;
|
||||
let stripped = resolved.strip_prefix("anthropic/").unwrap_or(&resolved);
|
||||
for key in [&resolved, stripped, &format!("anthropic/{stripped}")] {
|
||||
let value = costs.call_method1("get", (key,))?;
|
||||
if !value.is_none() {
|
||||
return from_py(&value).map(Some);
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn counter(py: Python<'_>, model: &str) -> PyResult<&'static TokenCounter> {
|
||||
static CLAUDE: OnceLock<TokenCounter> = OnceLock::new();
|
||||
static CL100K: OnceLock<TokenCounter> = OnceLock::new();
|
||||
let sdk = py.import("litellm")?;
|
||||
let anthropic = sdk.getattr("anthropic_models")?.contains(model)?
|
||||
&& !model.contains("claude-3")
|
||||
&& !sdk.getattr("disable_hf_tokenizer_download")?.is_truthy()?;
|
||||
let cell = if anthropic { &CLAUDE } else { &CL100K };
|
||||
if let Some(counter) = cell.get() {
|
||||
return Ok(counter);
|
||||
}
|
||||
let loaded = if anthropic {
|
||||
let json: String = py
|
||||
.import("litellm.utils")?
|
||||
.getattr("claude_json_str")?
|
||||
.extract()?;
|
||||
py.detach(|| TokenCounter::from_json(&json))
|
||||
} else {
|
||||
py.detach(|| TokenCounter::from_tiktoken("cl100k_base"))
|
||||
}
|
||||
.map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?;
|
||||
let _ = cell.set(loaded);
|
||||
cell.get().ok_or_else(litellm_host_python::missing_state)
|
||||
}
|
||||
|
|
@ -29,10 +29,60 @@ pub fn prepare<'py>(
|
|||
.call(py, ())?
|
||||
.cast_into::<PyList>()?)
|
||||
})?;
|
||||
Wrapper::CheckLimits.call(py, (&arguments,))?;
|
||||
check_limits(py, &arguments)?;
|
||||
Ok(arguments)
|
||||
}
|
||||
|
||||
fn check_limits(py: Python<'_>, kwargs: &Bound<'_, PyDict>) -> PyResult<()> {
|
||||
let sdk = py.import("litellm")?;
|
||||
check_limits_with_settings(py, kwargs, sdk.as_any())
|
||||
}
|
||||
|
||||
fn check_limits_with_settings(
|
||||
py: Python<'_>,
|
||||
kwargs: &Bound<'_, PyDict>,
|
||||
sdk: &Bound<'_, PyAny>,
|
||||
) -> PyResult<()> {
|
||||
let max_budget = sdk.getattr("max_budget")?.extract::<Option<f64>>()?;
|
||||
if max_budget.is_some_and(|limit| limit != 0.0) {
|
||||
let current_cost = sdk.getattr("_current_cost")?.extract::<f64>()?;
|
||||
if let Err(error) = litellm_core_utils::budget::check_budget(current_cost, max_budget) {
|
||||
let arguments = PyDict::new(py);
|
||||
arguments.set_item("current_cost", error.current_cost)?;
|
||||
arguments.set_item("max_budget", error.max_budget)?;
|
||||
return Err(PyErr::from_value(
|
||||
sdk.getattr("BudgetExceededError")?
|
||||
.call((), Some(&arguments))?,
|
||||
));
|
||||
}
|
||||
}
|
||||
let limit = sdk.getattr("num_retries_per_request")?;
|
||||
if limit.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
let key = if kwargs.contains("litellm_metadata")? {
|
||||
"litellm_metadata"
|
||||
} else {
|
||||
"metadata"
|
||||
};
|
||||
let Some(metadata) = kwargs.get_item(key)? else {
|
||||
return Ok(());
|
||||
};
|
||||
if !metadata.is_instance(&py.import("collections.abc")?.getattr("Mapping")?)? {
|
||||
return Ok(());
|
||||
}
|
||||
let retry = metadata.call_method1("get", ("request_retry_count",))?;
|
||||
if retry.get_type().is(&py.get_type::<pyo3::types::PyInt>())
|
||||
&& retry.gt(0)?
|
||||
&& retry.ge(limit)?
|
||||
{
|
||||
return Err(pyo3::exceptions::PyRuntimeError::new_err(
|
||||
"Max retries per request hit!",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn inherit_credentials<'py>(
|
||||
py: Python<'py>,
|
||||
arguments: &Bound<'py, PyDict>,
|
||||
|
|
@ -73,6 +123,58 @@ fn inherit_credentials<'py>(
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn native_limits_preserve_budget_and_retry_boundaries() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = eval(
|
||||
py,
|
||||
c"
|
||||
import types
|
||||
class BudgetExceededError(Exception):
|
||||
def __init__(self, current_cost, max_budget):
|
||||
self.current_cost = current_cost
|
||||
self.max_budget = max_budget
|
||||
settings = types.SimpleNamespace(max_budget=2.0, _current_cost=2.0, num_retries_per_request=5,
|
||||
BudgetExceededError=BudgetExceededError)
|
||||
",
|
||||
);
|
||||
let settings = locals.get_item("settings").unwrap().unwrap();
|
||||
let arguments = PyDict::new(py);
|
||||
assert!(check_limits_with_settings(py, &arguments, &settings).is_ok());
|
||||
settings.setattr("_current_cost", 3.0).unwrap();
|
||||
let error = check_limits_with_settings(py, &arguments, &settings).unwrap_err();
|
||||
assert_eq!(
|
||||
error
|
||||
.value(py)
|
||||
.getattr("current_cost")
|
||||
.unwrap()
|
||||
.extract::<f64>()
|
||||
.unwrap(),
|
||||
3.0
|
||||
);
|
||||
settings.setattr("max_budget", 0.0).unwrap();
|
||||
for key in ["metadata", "litellm_metadata"] {
|
||||
for (count, refused) in [(0, false), (4, false), (5, true), (6, true)] {
|
||||
let metadata = PyDict::new(py);
|
||||
metadata.set_item("request_retry_count", count).unwrap();
|
||||
arguments.set_item(key, metadata).unwrap();
|
||||
assert_eq!(
|
||||
check_limits_with_settings(py, &arguments, &settings).is_err(),
|
||||
refused
|
||||
);
|
||||
}
|
||||
let metadata = PyDict::new(py);
|
||||
metadata.set_item("request_retry_count", true).unwrap();
|
||||
settings.setattr("num_retries_per_request", 0).unwrap();
|
||||
arguments.set_item(key, metadata).unwrap();
|
||||
assert!(check_limits_with_settings(py, &arguments, &settings).is_ok());
|
||||
arguments.del_item(key).unwrap();
|
||||
settings.setattr("num_retries_per_request", 5).unwrap();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(source, Some(&locals), Some(&locals)).unwrap();
|
||||
|
|
|
|||
|
|
@ -25,8 +25,6 @@ pub(crate) enum LegacyPython {
|
|||
pub(crate) enum Wrapper {
|
||||
#[strum(serialize = "setup")]
|
||||
Setup,
|
||||
#[strum(serialize = "check_limits")]
|
||||
CheckLimits,
|
||||
#[strum(serialize = "credential_list")]
|
||||
CredentialList,
|
||||
#[strum(serialize = "warn_unknown_credential")]
|
||||
|
|
|
|||
|
|
@ -90,8 +90,6 @@ assert prepared['document'] is replacement
|
|||
assert prepared['pages'] is replaced_kwargs['pages']
|
||||
assert prepared['litellm_logging_obj'] is logger
|
||||
assert 'litellm_logging_obj' not in replaced_kwargs
|
||||
[checked] = [value for name, value in logger.calls if name == 'check_limits']
|
||||
assert checked is prepared
|
||||
",
|
||||
);
|
||||
});
|
||||
|
|
@ -126,8 +124,6 @@ kwargs = {'logger': logger, 'vendor_extension': opaque}
|
|||
&locals,
|
||||
c"
|
||||
assert prepared['vendor_extension'] is opaque
|
||||
[checked] = [value for name, value in logger.calls if name == 'check_limits']
|
||||
assert checked['vendor_extension'] is opaque
|
||||
assert hooked == ([opaque] if asynchronous else []), hooked
|
||||
",
|
||||
);
|
||||
|
|
@ -247,7 +243,7 @@ assert all(value is failure for name, value in logger.calls if name.endswith('_h
|
|||
#[rstest]
|
||||
#[case::synchronous(false)]
|
||||
#[case::asynchronous(true)]
|
||||
fn a_limit_rejected_before_the_call_surfaces_as_the_callers_error(#[case] asynchronous: bool) {
|
||||
fn preparation_failure_before_the_call_preserves_exception_identity(#[case] asynchronous: bool) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = namespace(
|
||||
|
|
@ -258,13 +254,13 @@ class BudgetExceeded(Exception):
|
|||
|
||||
rejection = BudgetExceeded('over budget')
|
||||
|
||||
class LimitedLogger(StubLogger):
|
||||
def check_limits(self, arguments):
|
||||
class RejectedCredential(str):
|
||||
def __bool__(self):
|
||||
raise rejection
|
||||
|
||||
logger = LimitedLogger()
|
||||
logger = StubLogger()
|
||||
logger.hooks = {'pre': lambda kwargs: kwargs}
|
||||
kwargs = {'logger': logger}
|
||||
kwargs = {'logger': logger, 'litellm_credential_name': RejectedCredential('named')}
|
||||
",
|
||||
);
|
||||
let mut logging = legacy_call(py, &locals, asynchronous);
|
||||
|
|
|
|||
|
|
@ -27,6 +27,9 @@ for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_le
|
|||
sys.modules.setdefault(name, types.ModuleType(name))
|
||||
|
||||
legacy = sys.modules['litellm.rust_bridge.callbacks_legacy_python']
|
||||
sdk = sys.modules['litellm']
|
||||
sdk.max_budget = None
|
||||
sdk.num_retries_per_request = None
|
||||
CONTRACT = json.loads(python_contract)
|
||||
|
||||
|
||||
|
|
@ -50,7 +53,6 @@ FAKES = {
|
|||
logger=kwargs['logger_factory'](kwargs) if 'logger_factory' in kwargs else kwargs['logger'],
|
||||
kwargs=kwargs,
|
||||
),
|
||||
'check_limits': lambda arguments: arguments['logger'].check_limits(arguments),
|
||||
'finalize': lambda response, logger, kwargs, start, end: logger.record('finalize', response),
|
||||
'update_logging': lambda logger, kwargs, model, optional_params, litellm_params, provider: logger.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
|
|
|
|||
82
litellm-rust/crates/core-utils/src/budget.rs
Normal file
82
litellm-rust/crates/core-utils/src/budget.rs
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, thiserror::Error)]
|
||||
#[error("Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}")]
|
||||
pub struct BudgetExceeded {
|
||||
pub current_cost: f64,
|
||||
pub max_budget: f64,
|
||||
}
|
||||
|
||||
pub fn check_budget(current_cost: f64, max_budget: Option<f64>) -> Result<(), BudgetExceeded> {
|
||||
if let Some(max_budget) = max_budget.filter(|limit| *limit != 0.0)
|
||||
&& current_cost > max_budget
|
||||
{
|
||||
return Err(BudgetExceeded {
|
||||
current_cost,
|
||||
max_budget,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Budget {
|
||||
current_cost: Mutex<f64>,
|
||||
pub max_budget: Option<f64>,
|
||||
}
|
||||
|
||||
impl Budget {
|
||||
pub fn new(max_budget: Option<f64>, current_cost: f64) -> Self {
|
||||
Self {
|
||||
current_cost: Mutex::new(current_cost),
|
||||
max_budget,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn current_cost(&self) -> f64 {
|
||||
*self
|
||||
.current_cost
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
}
|
||||
|
||||
pub fn check(&self) -> Result<(), BudgetExceeded> {
|
||||
check_budget(self.current_cost(), self.max_budget)
|
||||
}
|
||||
|
||||
pub fn record(&self, cost: f64) {
|
||||
*self
|
||||
.current_cost
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner()) += cost;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn matches_python_truthiness_and_strict_boundary() {
|
||||
assert!(check_budget(100.0, None).is_ok());
|
||||
assert!(check_budget(100.0, Some(0.0)).is_ok());
|
||||
assert!(check_budget(1.0, Some(1.0)).is_ok());
|
||||
assert_eq!(check_budget(1.5, Some(1.0)).unwrap_err().current_cost, 1.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_completions_do_not_lose_spend() {
|
||||
let budget = Budget::new(Some(10.0), 0.0);
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..8 {
|
||||
scope.spawn(|| {
|
||||
for _ in 0..100 {
|
||||
budget.record(0.25);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
assert_eq!(budget.current_cost(), 200.0);
|
||||
assert!(budget.check().is_err());
|
||||
}
|
||||
}
|
||||
156
litellm-rust/crates/core-utils/src/cost_calculator.rs
Normal file
156
litellm-rust/crates/core-utils/src/cost_calculator.rs
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
use serde_json::Value;
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct TokenPrices {
|
||||
pub input: f64,
|
||||
pub output: f64,
|
||||
pub cache_read: f64,
|
||||
pub cache_creation: f64,
|
||||
pub cache_creation_1hr: f64,
|
||||
}
|
||||
|
||||
impl TokenPrices {
|
||||
pub fn from_model_info(info: &Value, usage: &Value) -> Option<Self> {
|
||||
let input = info.get("input_cost_per_token")?.as_f64()?;
|
||||
let output = info.get("output_cost_per_token")?.as_f64()?;
|
||||
let total_input = [
|
||||
"input_tokens",
|
||||
"cache_read_input_tokens",
|
||||
"cache_creation_input_tokens",
|
||||
]
|
||||
.iter()
|
||||
.map(|key| count(usage, key))
|
||||
.sum::<f64>();
|
||||
let threshold = info
|
||||
.as_object()?
|
||||
.keys()
|
||||
.filter_map(|key| {
|
||||
let threshold = key
|
||||
.strip_prefix("input_cost_per_token_above_")?
|
||||
.strip_suffix("_tokens")?;
|
||||
let number = threshold
|
||||
.strip_suffix('k')
|
||||
.unwrap_or(threshold)
|
||||
.parse::<f64>()
|
||||
.ok()?
|
||||
* if threshold.ends_with('k') {
|
||||
1000.0
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
(total_input > number).then_some((number, threshold))
|
||||
})
|
||||
.max_by(|a, b| a.0.total_cmp(&b.0))
|
||||
.map(|(_, suffix)| suffix);
|
||||
let tier = usage
|
||||
.get("service_tier")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|tier| matches!(*tier, "priority" | "flex"));
|
||||
let rate = |key: &str| -> Option<f64> {
|
||||
let base = tier
|
||||
.and_then(|tier| info.get(format!("{key}_{tier}")).and_then(Value::as_f64))
|
||||
.or_else(|| info.get(key).and_then(Value::as_f64));
|
||||
threshold
|
||||
.and_then(|threshold| {
|
||||
let key = format!("{key}_above_{threshold}_tokens");
|
||||
tier.and_then(|tier| info.get(format!("{key}_{tier}")).and_then(Value::as_f64))
|
||||
.or_else(|| info.get(key).and_then(Value::as_f64))
|
||||
})
|
||||
.or(base)
|
||||
};
|
||||
let input = rate("input_cost_per_token").unwrap_or(input);
|
||||
let creation = rate("cache_creation_input_token_cost").unwrap_or(input);
|
||||
Some(Self {
|
||||
input,
|
||||
output: rate("output_cost_per_token").unwrap_or(output),
|
||||
cache_read: rate("cache_read_input_token_cost").unwrap_or(input),
|
||||
cache_creation: creation,
|
||||
cache_creation_1hr: rate("cache_creation_input_token_cost_above_1hr")
|
||||
.unwrap_or(creation),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn messages_cost(&self, usage: &Value) -> f64 {
|
||||
let creation = match usage
|
||||
.get("cache_creation")
|
||||
.filter(|value| value.is_object())
|
||||
{
|
||||
Some(details) => {
|
||||
count(details, "ephemeral_5m_input_tokens") * self.cache_creation
|
||||
+ count(details, "ephemeral_1h_input_tokens") * self.cache_creation_1hr
|
||||
}
|
||||
None => count(usage, "cache_creation_input_tokens") * self.cache_creation,
|
||||
};
|
||||
count(usage, "input_tokens") * self.input
|
||||
+ count(usage, "output_tokens") * self.output
|
||||
+ count(usage, "cache_read_input_tokens") * self.cache_read
|
||||
+ creation
|
||||
}
|
||||
}
|
||||
|
||||
fn count(value: &Value, key: &str) -> f64 {
|
||||
value.get(key).and_then(Value::as_f64).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn messages_cost(info: &Value, usage: &Value) -> Option<f64> {
|
||||
let prices = TokenPrices::from_model_info(info, usage)?;
|
||||
let provider = info.get("provider_specific_entry").unwrap_or(&Value::Null);
|
||||
let speed = if usage.get("speed").and_then(Value::as_str) == Some("fast") {
|
||||
provider.get("fast").and_then(Value::as_f64).unwrap_or(1.0)
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
let geo = usage
|
||||
.get("inference_geo")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|geo| provider.get(geo))
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(1.0);
|
||||
let searches = usage
|
||||
.pointer("/server_tool_use/web_search_requests")
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or_default();
|
||||
let search_rate = info
|
||||
.pointer("/search_context_cost_per_query/search_context_size_medium")
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or_default();
|
||||
Some(prices.messages_cost(usage) * speed * geo + searches * search_rate)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn accounts_for_each_input_category_once_and_preserves_zero_prices() {
|
||||
let info = json!({"input_cost_per_token": 2.0, "output_cost_per_token": 3.0,
|
||||
"cache_read_input_token_cost": 0.0, "cache_creation_input_token_cost": 4.0,
|
||||
"cache_creation_input_token_cost_above_1hr": 5.0});
|
||||
let usage = json!({"input_tokens": 10, "output_tokens": 3,
|
||||
"cache_read_input_tokens": 100, "cache_creation_input_tokens": 7,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 2, "ephemeral_1h_input_tokens": 5}});
|
||||
assert_eq!(messages_cost(&info, &usage), Some(62.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_uses_total_input_and_is_strict() {
|
||||
let info = json!({"input_cost_per_token": 1.0, "output_cost_per_token": 2.0,
|
||||
"input_cost_per_token_above_100_tokens": 4.0});
|
||||
assert_eq!(
|
||||
messages_cost(&info, &json!({"input_tokens": 100})),
|
||||
Some(100.0)
|
||||
);
|
||||
assert_eq!(
|
||||
messages_cost(
|
||||
&info,
|
||||
&json!({"input_tokens": 100, "cache_read_input_tokens": 1})
|
||||
),
|
||||
Some(404.0)
|
||||
);
|
||||
assert_eq!(
|
||||
messages_cost(&json!({}), &json!({"input_tokens": 100})),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,10 +1,14 @@
|
|||
pub mod budget;
|
||||
pub mod call_arguments;
|
||||
pub mod core_helpers;
|
||||
pub mod cost_calculator;
|
||||
pub mod exception_mapping_utils;
|
||||
pub mod get_llm_provider_logic;
|
||||
pub mod messages_stream;
|
||||
pub mod params;
|
||||
pub mod prompt_templates;
|
||||
pub mod secret_redaction;
|
||||
pub mod serde_compat;
|
||||
pub mod settings;
|
||||
pub mod token_counter;
|
||||
pub mod url_utils;
|
||||
|
|
|
|||
40
litellm-rust/crates/core-utils/src/messages_stream.rs
Normal file
40
litellm-rust/crates/core-utils/src/messages_stream.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
use serde_json::{Map, Value};
|
||||
|
||||
pub struct MessagesStream {
|
||||
pub usage: Value,
|
||||
pub cached_response: Option<Value>,
|
||||
}
|
||||
|
||||
impl MessagesStream {
|
||||
pub fn parse(bytes: &[u8]) -> Option<Self> {
|
||||
let text = std::str::from_utf8(bytes).ok()?;
|
||||
let mut usage = Map::new();
|
||||
let mut stopped = false;
|
||||
let mut failed = false;
|
||||
let events: Vec<_> = text.split_inclusive("\n\n").collect();
|
||||
for event in &events {
|
||||
for line in event.lines().filter_map(|line| line.strip_prefix("data:")) {
|
||||
let Ok(value) = serde_json::from_str::<Value>(line.trim()) else {
|
||||
continue;
|
||||
};
|
||||
match value.get("type").and_then(Value::as_str) {
|
||||
Some("message_stop") => stopped = true,
|
||||
Some("error") => failed = true,
|
||||
_ => {}
|
||||
}
|
||||
if let Some(fields) = value
|
||||
.pointer("/message/usage")
|
||||
.or_else(|| value.get("usage"))
|
||||
.and_then(Value::as_object)
|
||||
{
|
||||
usage.extend(fields.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(Self {
|
||||
usage: Value::Object(usage),
|
||||
cached_response: (stopped && !failed)
|
||||
.then(|| serde_json::json!({"litellm_cached_anthropic_sse_events": events})),
|
||||
})
|
||||
}
|
||||
}
|
||||
55
litellm-rust/crates/core-utils/src/token_counter.rs
Normal file
55
litellm-rust/crates/core-utils/src/token_counter.rs
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
pub fn get_modified_max_tokens(
|
||||
user_max_tokens: i64,
|
||||
max_input_tokens: Option<i64>,
|
||||
max_output_tokens: Option<i64>,
|
||||
input_tokens: usize,
|
||||
buffer_perc: Option<f64>,
|
||||
buffer_num: Option<f64>,
|
||||
) -> i64 {
|
||||
let Some(output) = max_output_tokens else {
|
||||
return user_max_tokens;
|
||||
};
|
||||
let buffer =
|
||||
(buffer_perc.unwrap_or(0.1) * input_tokens as f64).max(buffer_num.unwrap_or(10.0)) as i64;
|
||||
let input = (input_tokens as i64).saturating_add(buffer);
|
||||
if max_input_tokens == Some(output) {
|
||||
if input <= output && user_max_tokens.saturating_add(input) > output {
|
||||
return output - input;
|
||||
}
|
||||
return user_max_tokens;
|
||||
}
|
||||
user_max_tokens.min(output)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn shared_context_includes_buffer_and_does_not_hide_overflow() {
|
||||
assert_eq!(
|
||||
get_modified_max_tokens(80, Some(100), Some(100), 40, None, None),
|
||||
50
|
||||
);
|
||||
assert_eq!(
|
||||
get_modified_max_tokens(80, Some(100), Some(100), 100, None, None),
|
||||
80
|
||||
);
|
||||
assert_eq!(
|
||||
get_modified_max_tokens(80, Some(100), Some(100), 90, None, None),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
get_modified_max_tokens(80, Some(1000), Some(100), 900, None, None),
|
||||
80
|
||||
);
|
||||
assert_eq!(
|
||||
get_modified_max_tokens(180, Some(1000), Some(100), 900, None, None),
|
||||
100
|
||||
);
|
||||
assert_eq!(
|
||||
get_modified_max_tokens(180, None, None, 900, None, None),
|
||||
180
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,8 @@ autotests = false
|
|||
[dependencies]
|
||||
litellm-types.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
litellm-token-counter.workspace = true
|
||||
litellm-host.workspace = true
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
|
|
@ -36,6 +38,7 @@ url.workspace = true
|
|||
veil.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-cache-memory.workspace = true
|
||||
litellm-auth-gcp.workspace = true
|
||||
litellm-llms = { workspace = true, features = ["test-support"] }
|
||||
rstest.workspace = true
|
||||
|
|
|
|||
67
litellm-rust/crates/core/src/client.rs
Normal file
67
litellm-rust/crates/core/src/client.rs
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use litellm_cache::LLMCachingHandler;
|
||||
use litellm_core_utils::{
|
||||
budget::Budget, cost_calculator::messages_cost, token_counter::get_modified_max_tokens,
|
||||
};
|
||||
use litellm_token_counter::{CountableRequest, TokenCounter};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct ClientOptions {
|
||||
pub cache: Option<LLMCachingHandler>,
|
||||
pub budget: Option<Arc<Budget>>,
|
||||
pub model_info: Option<Value>,
|
||||
pub token_counter: Option<Arc<TokenCounter>>,
|
||||
pub modify_params: bool,
|
||||
}
|
||||
|
||||
impl ClientOptions {
|
||||
pub fn adjust_max_tokens(&self, body: &mut Map<String, Value>) {
|
||||
if !self.modify_params {
|
||||
return;
|
||||
}
|
||||
let (Some(counter), Some(info), Some(requested)) = (
|
||||
&self.token_counter,
|
||||
&self.model_info,
|
||||
body.get("max_tokens").and_then(Value::as_i64),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let request =
|
||||
serde_json::json!({"model": body.get("model"), "messages": body.get("messages")});
|
||||
let Ok(request) = serde_json::from_value::<CountableRequest>(request) else {
|
||||
return;
|
||||
};
|
||||
let Ok(count) = counter.count_request(&request) else {
|
||||
return;
|
||||
};
|
||||
let output = info
|
||||
.get("max_output_tokens")
|
||||
.or_else(|| info.get("max_tokens"))
|
||||
.and_then(Value::as_i64);
|
||||
let adjusted = get_modified_max_tokens(
|
||||
requested,
|
||||
info.get("max_input_tokens").and_then(Value::as_i64),
|
||||
output,
|
||||
count.input_tokens,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
body.insert("max_tokens".into(), Value::from(adjusted));
|
||||
}
|
||||
|
||||
pub fn record_messages_usage(&self, usage: &Value, cache_hit: bool) -> Option<f64> {
|
||||
let cost = if cache_hit {
|
||||
Some(0.0)
|
||||
} else {
|
||||
self.model_info
|
||||
.as_ref()
|
||||
.and_then(|info| messages_cost(info, usage))
|
||||
};
|
||||
if !cache_hit && let (Some(budget), Some(cost)) = (&self.budget, cost) {
|
||||
budget.record(cost);
|
||||
}
|
||||
cost
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
pub mod audio_transcription;
|
||||
pub mod chat_completions;
|
||||
pub mod client;
|
||||
pub mod constants;
|
||||
pub mod error;
|
||||
pub mod messages;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
use litellm_llms::base_llm::chat::transformation::Error as LlmError;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
#[derive(Clone, Debug, PartialEq, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error(transparent)]
|
||||
BudgetExceeded(#[from] litellm_core_utils::budget::BudgetExceeded),
|
||||
#[error("invalid provider: {0}")]
|
||||
InvalidProvider(String),
|
||||
#[error("missing required field: {0}")]
|
||||
|
|
|
|||
147
litellm-rust/crates/core/src/messages/lifecycle_tests.rs
Normal file
147
litellm-rust/crates/core/src/messages/lifecycle_tests.rs
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use litellm_cache::{CacheControls, CacheKwargs, LLMCachingHandler};
|
||||
use litellm_cache_memory::InMemoryCache;
|
||||
use litellm_core_utils::budget::Budget;
|
||||
use litellm_token_counter::{TokenCounter, Tokenizer};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::{io::AsyncWriteExt, net::TcpListener};
|
||||
|
||||
use crate::{
|
||||
client::ClientOptions,
|
||||
messages::{Error, messages_with_options, types::MessagesRequest},
|
||||
};
|
||||
|
||||
fn options() -> ClientOptions {
|
||||
ClientOptions {
|
||||
cache: Some(LLMCachingHandler {
|
||||
backend: Arc::new(InMemoryCache::default()),
|
||||
controls: CacheControls {
|
||||
supported_call_type: true,
|
||||
configured: true,
|
||||
native_backend: true,
|
||||
default_on: true,
|
||||
..Default::default()
|
||||
},
|
||||
key: "test-key".into(),
|
||||
kwargs: CacheKwargs::default(),
|
||||
max_age: None,
|
||||
}),
|
||||
budget: Some(Arc::new(Budget::new(Some(100.0), 0.0))),
|
||||
model_info: Some(
|
||||
json!({"input_cost_per_token": 2.0, "output_cost_per_token": 3.0,
|
||||
"cache_read_input_token_cost": 0.5, "cache_creation_input_token_cost": 4.0}),
|
||||
),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn request(url: &str) -> MessagesRequest<'_> {
|
||||
MessagesRequest {
|
||||
model: "anthropic/test-model",
|
||||
body: json!({"model":"anthropic/test-model", "max_tokens": 100,
|
||||
"messages": [{"role": "user", "content": "hi"}]}),
|
||||
api_key: Some("test"),
|
||||
api_base: Some(url),
|
||||
custom_llm_provider: Some("anthropic"),
|
||||
extra_headers: None,
|
||||
timeout: Some(Duration::from_secs(2)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_server() -> (String, tokio::task::JoinHandle<Value>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let request = super::tests::read_http_request(&mut socket).await;
|
||||
let body = json!({"id":"msg_test", "type":"message", "role":"assistant", "model":"test-model",
|
||||
"content":[{"type":"text","text":"hello"}], "stop_reason":"end_turn", "stop_sequence":null,
|
||||
"usage":{"input_tokens":2,"output_tokens":3,"cache_read_input_tokens":4,"cache_creation_input_tokens":5}});
|
||||
socket
|
||||
.write_all(super::tests::write_response(&body.to_string()).as_bytes())
|
||||
.await
|
||||
.unwrap();
|
||||
serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap()
|
||||
});
|
||||
(url, server)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_messages_cache_hit_skips_network_and_does_not_double_charge() {
|
||||
let options = options();
|
||||
let (url, server) = start_server().await;
|
||||
let first = messages_with_options(request(&url), options.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
server.await.unwrap();
|
||||
let cost = 2.0 * 2.0 + 3.0 * 3.0 + 4.0 * 0.5 + 5.0 * 4.0;
|
||||
assert_eq!(options.budget.as_ref().unwrap().current_cost(), cost);
|
||||
let cached = messages_with_options(request(&url), options.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first, cached);
|
||||
assert_eq!(options.budget.as_ref().unwrap().current_cost(), cost);
|
||||
|
||||
let exceeded = ClientOptions {
|
||||
budget: Some(Arc::new(Budget::new(Some(cost - 1.0), cost))),
|
||||
..options
|
||||
};
|
||||
assert!(matches!(
|
||||
messages_with_options(request(&url), exceeded).await,
|
||||
Err(Error::BudgetExceeded(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_store_and_no_cache_control_different_operations() {
|
||||
let mut options = options();
|
||||
options.cache.as_mut().unwrap().controls.no_store = true;
|
||||
let (url, server) = start_server().await;
|
||||
messages_with_options(request(&url), options.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
server.await.unwrap();
|
||||
assert!(options.cache.as_ref().unwrap().get().await.is_none());
|
||||
|
||||
options.cache.as_mut().unwrap().controls.no_store = false;
|
||||
options
|
||||
.cache
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.set(json!({"id":"sentinel"}))
|
||||
.await;
|
||||
options.cache.as_mut().unwrap().controls.no_cache = true;
|
||||
assert!(options.cache.as_ref().unwrap().get().await.is_none());
|
||||
let (url, server) = start_server().await;
|
||||
let response = messages_with_options(request(&url), options.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
server.await.unwrap();
|
||||
options.cache.as_mut().unwrap().controls.no_cache = false;
|
||||
assert_eq!(
|
||||
options.cache.as_ref().unwrap().get().await.unwrap()["id"],
|
||||
response.id
|
||||
);
|
||||
}
|
||||
|
||||
struct Characters;
|
||||
impl Tokenizer for Characters {
|
||||
fn count_tokens(&self, text: &str) -> Result<usize, litellm_token_counter::Error> {
|
||||
Ok(text.chars().count())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opt_in_token_adjustment_reaches_the_wire_with_a_standard_counter_interface() {
|
||||
let (url, server) = start_server().await;
|
||||
let options = ClientOptions {
|
||||
token_counter: Some(Arc::new(TokenCounter::new(Characters))),
|
||||
modify_params: true,
|
||||
model_info: Some(json!({"max_input_tokens": 40, "max_output_tokens": 40})),
|
||||
..Default::default()
|
||||
};
|
||||
messages_with_options(request(&url), options).await.unwrap();
|
||||
let sent = server.await.unwrap();
|
||||
assert_eq!(sent["max_tokens"], 40 - (3 + 4 + 2 + 3) - 10);
|
||||
}
|
||||
|
|
@ -19,12 +19,21 @@ use serde_json::Value;
|
|||
use crate::messages::types::MessagesRequest;
|
||||
|
||||
pub async fn messages(request: MessagesRequest<'_>) -> Result<AnthropicMessagesResponse, Error> {
|
||||
messages_with_options(request, crate::client::ClientOptions::default()).await
|
||||
}
|
||||
|
||||
pub async fn messages_with_options(
|
||||
request: MessagesRequest<'_>,
|
||||
options: crate::client::ClientOptions,
|
||||
) -> Result<AnthropicMessagesResponse, Error> {
|
||||
let Value::Object(body) = request.body else {
|
||||
return Err(Error::InvalidRequest(
|
||||
"messages body must be an object".into(),
|
||||
));
|
||||
};
|
||||
let call = MessagesCall {
|
||||
cached_response: None,
|
||||
options,
|
||||
model: request.model.into(),
|
||||
body,
|
||||
api_key: request.api_key.map(Into::into),
|
||||
|
|
@ -41,5 +50,7 @@ pub async fn messages(request: MessagesRequest<'_>) -> Result<AnthropicMessagesR
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod lifecycle_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ pub enum MessagesOpResult {
|
|||
|
||||
/// The caller's request as the host projects it.
|
||||
pub struct MessagesCall {
|
||||
pub cached_response: Option<Value>,
|
||||
pub options: crate::client::ClientOptions,
|
||||
pub model: String,
|
||||
pub body: Map<String, Value>,
|
||||
pub api_key: Option<String>,
|
||||
|
|
@ -125,8 +127,48 @@ pub fn messages_machine() -> MessagesMachine {
|
|||
}
|
||||
|
||||
async fn execute(host: MessagesHost) -> Result<MessagesOutput, Error> {
|
||||
let MessagesOpResult::Request(call) = host.route(MessagesOp::ProjectRequest).await?;
|
||||
let MessagesOpResult::Request(mut call) = host.route(MessagesOp::ProjectRequest).await?;
|
||||
if let Some(budget) = &call.options.budget {
|
||||
budget.check()?;
|
||||
}
|
||||
let stream = call.streams();
|
||||
let cached = match call.cached_response.take() {
|
||||
Some(response) => Some(response),
|
||||
None => match &call.options.cache {
|
||||
Some(cache) => cache.get().await,
|
||||
None => None,
|
||||
},
|
||||
};
|
||||
if let Some(response) = cached {
|
||||
if !stream
|
||||
&& let Ok(message) =
|
||||
serde_json::from_value::<AnthropicMessagesResponse>(response.clone())
|
||||
{
|
||||
call.options
|
||||
.record_messages_usage(message.usage.as_ref().unwrap_or(&Value::Null), true);
|
||||
return Ok(MessagesOutput::Message(Box::new(message)));
|
||||
}
|
||||
if stream
|
||||
&& let Some(events) = response
|
||||
.get("litellm_cached_anthropic_sse_events")
|
||||
.and_then(Value::as_array)
|
||||
&& events.iter().all(Value::is_string)
|
||||
{
|
||||
if host.open(()).await? == Demand::More {
|
||||
for event in events {
|
||||
if host
|
||||
.deliver(Bytes::from(event.as_str().unwrap_or_default().to_owned()))
|
||||
.await?
|
||||
== Demand::Detached
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Ok(MessagesOutput::Streamed);
|
||||
}
|
||||
}
|
||||
call.options.adjust_max_tokens(&mut call.body);
|
||||
let request = prepare_provider_request(MessagesRequest {
|
||||
model: &call.model,
|
||||
body: Value::Object(call.body.clone()),
|
||||
|
|
@ -167,15 +209,22 @@ async fn execute(host: MessagesHost) -> Result<MessagesOutput, Error> {
|
|||
return Err(provider_error(response).await);
|
||||
}
|
||||
if stream {
|
||||
return relay(&host, response).await;
|
||||
return relay(&host, response, &call.options).await;
|
||||
}
|
||||
let text = response.text().await.map_err(network)?;
|
||||
host.emit(MachineEvent::ResponseReceived {
|
||||
raw: RawResponse { body: text.clone() },
|
||||
})
|
||||
.await?;
|
||||
decode_response(request.config, &request.model, &text)
|
||||
.map(|message| MessagesOutput::Message(Box::new(message)))
|
||||
let message = decode_response(request.config, &request.model, &text)?;
|
||||
call.options
|
||||
.record_messages_usage(message.usage.as_ref().unwrap_or(&Value::Null), false);
|
||||
if let Some(cache) = &call.options.cache
|
||||
&& let Ok(value) = serde_json::to_value(&message)
|
||||
{
|
||||
cache.set(value).await;
|
||||
}
|
||||
Ok(MessagesOutput::Message(Box::new(message)))
|
||||
}
|
||||
|
||||
/// Hands each upstream chunk to the caller as it arrives. A caller that stops reading
|
||||
|
|
@ -183,14 +232,26 @@ async fn execute(host: MessagesHost) -> Result<MessagesOutput, Error> {
|
|||
async fn relay(
|
||||
host: &MessagesHost,
|
||||
mut response: reqwest::Response,
|
||||
options: &crate::client::ClientOptions,
|
||||
) -> Result<MessagesOutput, Error> {
|
||||
if host.open(()).await? == Demand::Detached {
|
||||
return Ok(MessagesOutput::Streamed);
|
||||
}
|
||||
let mut collected = Vec::new();
|
||||
let mut detached = false;
|
||||
while let Some(chunk) = response.chunk().await.map_err(network)? {
|
||||
collected.extend_from_slice(&chunk);
|
||||
if host.deliver(chunk).await? == Demand::Detached {
|
||||
detached = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(stream) = litellm_core_utils::messages_stream::MessagesStream::parse(&collected) {
|
||||
options.record_messages_usage(&stream.usage, false);
|
||||
if !detached && let (Some(cache), Some(response)) = (&options.cache, stream.cached_response)
|
||||
{
|
||||
cache.set(response).await;
|
||||
}
|
||||
}
|
||||
Ok(MessagesOutput::Streamed)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ use super::{
|
|||
};
|
||||
use crate::messages::types::MessagesRequest;
|
||||
|
||||
async fn read_http_request(socket: &mut TcpStream) -> String {
|
||||
pub(super) async fn read_http_request(socket: &mut TcpStream) -> String {
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 1024];
|
||||
let header_end = loop {
|
||||
|
|
@ -48,7 +48,7 @@ async fn read_http_request(socket: &mut TcpStream) -> String {
|
|||
String::from_utf8(request).expect("request is utf8")
|
||||
}
|
||||
|
||||
fn write_response(body: &str) -> String {
|
||||
pub(super) fn write_response(body: &str) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
||||
body.len(),
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ pub fn missing_state() -> PyErr {
|
|||
pub enum LifecycleStep {
|
||||
Await(Py<PyAny>),
|
||||
Arguments(Py<PyDict>),
|
||||
Cached(Py<PyAny>),
|
||||
Wire(Box<WireRequest>),
|
||||
Response(Py<PyAny>),
|
||||
Done,
|
||||
|
|
|
|||
|
|
@ -207,6 +207,10 @@ where
|
|||
self.stage = Stage::Call;
|
||||
self.resume_machine(py, None)
|
||||
}
|
||||
(Expect::Arguments, LifecycleStep::Cached(response)) => {
|
||||
self.ended_at = Some(epoch_seconds());
|
||||
self.succeeded(py, response)
|
||||
}
|
||||
(Expect::Wire, LifecycleStep::Wire(wire)) => {
|
||||
self.resume_machine(py, Some(Ok(HostResult::BeforeSend(wire))))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -85,6 +85,11 @@ impl MessagesRouteHost {
|
|||
.transpose()?
|
||||
.flatten();
|
||||
Ok(MessagesCall {
|
||||
cached_response: arguments
|
||||
.get_item("_rust_messages_cached_response")?
|
||||
.map(|value| from_py(&value))
|
||||
.transpose()?,
|
||||
options: Default::default(),
|
||||
model,
|
||||
body,
|
||||
api_key: string("api_key")?,
|
||||
|
|
|
|||
107
litellm-rust/crates/python-bridge/tests/messages_lifecycle.py
Normal file
107
litellm-rust/crates/python-bridge/tests/messages_lifecycle.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import litellm
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.rust_bridge import _native
|
||||
from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest
|
||||
|
||||
|
||||
class Upstream(BaseHTTPRequestHandler):
|
||||
calls = []
|
||||
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers["content-length"])))
|
||||
self.calls.append(body)
|
||||
usage = {"input_tokens": 2, "output_tokens": 3, "cache_read_input_tokens": 4, "cache_creation_input_tokens": 5}
|
||||
message = {"id": "msg_native", "type": "message", "role": "assistant", "model": "test-native-sdk",
|
||||
"content": [{"type": "text", "text": "hello"}], "stop_reason": "end_turn", "stop_sequence": None,
|
||||
"usage": usage}
|
||||
if body.get("stream"):
|
||||
events = ({"type": "message_start", "message": {**message, "usage": {**usage, "output_tokens": 0}}},
|
||||
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 3}},
|
||||
{"type": "message_stop"})
|
||||
response = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode()
|
||||
else:
|
||||
response = json.dumps(message).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("content-type", "text/event-stream" if body.get("stream") else "application/json")
|
||||
self.send_header("content-length", str(len(response)))
|
||||
self.end_headers()
|
||||
self.wfile.write(response)
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
|
||||
class MessagesLifecycle(unittest.IsolatedAsyncioTestCase):
|
||||
def setUp(self):
|
||||
self.server = ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
self.original = (litellm.cache, litellm.max_budget, litellm._current_cost, litellm.modify_params)
|
||||
litellm.cache = Cache(type="local", supported_call_types=["anthropic_messages"])
|
||||
litellm.max_budget = 100.0
|
||||
litellm._current_cost = 0.0
|
||||
litellm.modify_params = False
|
||||
litellm.register_model({"test-native-sdk": {"litellm_provider": "anthropic", "mode": "chat",
|
||||
"input_cost_per_token": 2.0, "output_cost_per_token": 3.0,
|
||||
"cache_read_input_token_cost": 0.5, "cache_creation_input_token_cost": 4.0,
|
||||
"max_input_tokens": 200, "max_output_tokens": 50}})
|
||||
self.before = len(Upstream.calls)
|
||||
|
||||
async def asyncTearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join()
|
||||
litellm.cache, litellm.max_budget, litellm._current_cost, litellm.modify_params = self.original
|
||||
litellm.model_cost.pop("test-native-sdk", None)
|
||||
|
||||
def call(self, asynchronous=True, **extra):
|
||||
kwargs = {"max_tokens": 100, "messages": [{"role": "user", "content": "hello"}],
|
||||
"model": "test-native-sdk", "custom_llm_provider": "anthropic", "api_key": "test",
|
||||
"api_base": f"http://127.0.0.1:{self.server.server_port}", **extra}
|
||||
request = LiteLLMMessagesRequest(model=kwargs["model"], messages=kwargs["messages"], max_tokens=100,
|
||||
stream=kwargs.get("stream"), api_key="test", api_base=kwargs["api_base"],
|
||||
custom_llm_provider="anthropic", kwargs=kwargs)
|
||||
return (_native.amessages if asynchronous else _native.messages)(request, (), kwargs)
|
||||
|
||||
async def test_cache_hit_charges_once_and_budget_precedes_cache(self):
|
||||
first = await self.call()
|
||||
second = await self.call()
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(len(Upstream.calls) - self.before, 1)
|
||||
cost = 2 * 2.0 + 3 * 3.0 + 4 * 0.5 + 5 * 4.0
|
||||
self.assertEqual(litellm._current_cost, cost)
|
||||
litellm.max_budget = cost - 1
|
||||
with self.assertRaises(litellm.BudgetExceededError):
|
||||
await self.call()
|
||||
self.assertEqual(len(Upstream.calls) - self.before, 1)
|
||||
|
||||
async def test_stream_cache_replays_and_charges_once(self):
|
||||
first = b"".join([chunk async for chunk in await self.call(stream=True)])
|
||||
second = b"".join([chunk async for chunk in await self.call(stream=True)])
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(len(Upstream.calls) - self.before, 1)
|
||||
self.assertEqual(litellm._current_cost, 2 * 2.0 + 3 * 3.0 + 4 * 0.5 + 5 * 4.0)
|
||||
|
||||
async def test_opt_in_output_cap_is_sent_and_cached_under_original_request(self):
|
||||
litellm.modify_params = True
|
||||
await self.call()
|
||||
await self.call()
|
||||
self.assertEqual(len(Upstream.calls) - self.before, 1)
|
||||
self.assertEqual(Upstream.calls[-1]["max_tokens"], 50)
|
||||
|
||||
async def test_sync_messages_use_the_same_cache_and_budget(self):
|
||||
first = self.call(asynchronous=False)
|
||||
second = await self.call()
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(len(Upstream.calls) - self.before, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"native extension: {_native.__file__}")
|
||||
unittest.main()
|
||||
|
|
@ -504,6 +504,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
classifier_input: Mapping[str, JsonValue] | None = None
|
||||
baseline_cache_context: "BaselineCacheContext | None" = None
|
||||
baseline_observation: "CapturedBaselineObservation | None" = None
|
||||
_native_response_cost: float | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -1744,6 +1745,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if cache_hit is True:
|
||||
return 0.0
|
||||
|
||||
if self._native_response_cost is not None:
|
||||
return self._native_response_cost
|
||||
|
||||
if is_unbilled_non_inference_call(
|
||||
self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
|
||||
):
|
||||
|
|
|
|||
|
|
@ -64,21 +64,6 @@ def setup(
|
|||
return CallSetup(logger, prepared)
|
||||
|
||||
|
||||
def check_limits(kwargs: Mapping[str, object]) -> None:
|
||||
from litellm import (
|
||||
BudgetExceededError,
|
||||
_current_cost, # pyright: ignore[reportPrivateUsage] # shared SDK budget counter has no public accessor
|
||||
max_budget,
|
||||
num_retries_per_request,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit
|
||||
|
||||
if max_budget and _current_cost > max_budget:
|
||||
raise BudgetExceededError(current_cost=_current_cost, max_budget=max_budget)
|
||||
if max_retries_per_request_hit(kwargs, num_retries_per_request):
|
||||
raise RuntimeError("Max retries per request hit!")
|
||||
|
||||
|
||||
def finalize(
|
||||
response: object,
|
||||
logger: Logging,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from pydantic import TypeAdapter
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.rust_bridge import callbacks_legacy_python as legacy
|
||||
from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup
|
||||
from litellm.rust_bridge.callbacks_legacy_python import setup
|
||||
|
||||
_OCR_KWARGS: Final = MappingProxyType(
|
||||
{
|
||||
|
|
@ -21,33 +21,6 @@ _OCR_KWARGS: Final = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"])
|
||||
@pytest.mark.parametrize(
|
||||
"cap, request_retry_count, refused",
|
||||
[(5, 5, True), (5, 4, False), (0, 0, False), (0, 1, True)],
|
||||
ids=[
|
||||
"cap-above-four-reached",
|
||||
"cap-above-four-not-reached",
|
||||
"first-attempt-passes-cap-of-zero",
|
||||
"cap-of-zero-refuses-first-retry",
|
||||
],
|
||||
)
|
||||
def test_check_limits_reads_request_retry_count(
|
||||
monkeypatch: pytest.MonkeyPatch, metadata_key: str, cap: int, request_retry_count: int, refused: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "num_retries_per_request", cap)
|
||||
monkeypatch.setattr(litellm, "max_budget", None)
|
||||
kwargs: Final = {
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
metadata_key: {"request_retry_count": request_retry_count},
|
||||
}
|
||||
if refused:
|
||||
with pytest.raises(RuntimeError, match="Max retries per request hit!"):
|
||||
check_limits(kwargs)
|
||||
else:
|
||||
check_limits(kwargs)
|
||||
|
||||
|
||||
def _supplied_logger() -> Logging:
|
||||
return Logging(
|
||||
model="mistral/mistral-ocr-latest",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue