chore: merge main and avoid catalog insertion conflicts

This commit is contained in:
Cenab Batu Bora 2026-10-19 10:07:27 -06:00
commit 6635fb0b1d
89 changed files with 6414 additions and 2554 deletions

View file

@ -6,9 +6,11 @@ use litellm_llms::{
bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
};
use litellm_llms_types::formats::chat_completions::ChatMessage;
use serde_json::{Map, Value};
use super::Error;
use crate::types::{ChatCompletionsCall, ResolvedChatCompletionsRequest};
const HEADER_CONTEXT: &str = "chat completions";
@ -42,3 +44,58 @@ pub(super) fn string_headers(
) -> Result<Vec<(String, String)>, Error> {
shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from)
}
pub(super) fn resolve_request(
call: ChatCompletionsCall,
config: &'static dyn BaseConfig,
) -> Result<ResolvedChatCompletionsRequest, Error> {
let messages = serde_json::from_value::<Vec<ChatMessage>>(call.messages).map_err(|err| {
Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
"chat completions messages",
err,
))
})?;
if messages.is_empty() {
return Err(Error::InvalidRequest(
"chat completions requires at least one message".into(),
));
}
if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) {
return Err(Error::Unsupported(reason.0));
}
Ok(ResolvedChatCompletionsRequest {
messages,
optional_params: call.optional_params,
api_key: call.api_key,
api_base: call.api_base,
extra_headers: call.extra_headers,
timeout: call.timeout,
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::chat_completions_provider;
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
#[rstest]
#[case::anthropic(LlmProviders::Anthropic, ("anthropic-version", "2023-06-01"))]
#[case::bedrock(LlmProviders::Bedrock, ("Content-Type", "application/json"))]
fn resolves_supported_provider_configs(
#[case] provider: LlmProviders,
#[case] expected_header: (&str, &str),
) {
let config = chat_completions_provider(provider)
.expect("provider has a chat completions config")
.config();
assert!(config.default_headers().contains(&expected_header));
}
#[rstest]
#[case::openai(LlmProviders::Openai)]
fn rejects_providers_without_a_chat_config(#[case] provider: LlmProviders) {
assert!(chat_completions_provider(provider).is_none());
}
}

View file

@ -3,44 +3,66 @@ use std::time::Duration;
use litellm_auth::AuthServices;
use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body};
use litellm_http::{
Client,
outbound::OutboundRequest,
request::{truncate_error_body, with_default_headers},
};
use litellm_llms::base_llm::{
auth::{Authenticated, resolve_auth},
auth::{Authenticated, ValidatedEnvironment, resolve_auth},
chat::transformation::BaseConfig,
chat::transformation::ProviderChatResponseData,
};
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use litellm_secrets::source::Secrets;
use serde_json::Value;
use super::Error;
use crate::{constants::CHAT_COMPLETIONS_TIMEOUT_SECS, types::ProviderChatCompletionsRequest};
use crate::{
common_utils::string_headers, constants::CHAT_COMPLETIONS_TIMEOUT_SECS,
types::ResolvedChatCompletionsRequest,
};
#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately
pub(super) async fn execute(
http: &Client,
auth: &AuthServices,
request: ProviderChatCompletionsRequest,
config: &'static dyn BaseConfig,
provider: litellm_inference::provider::ResolvedProvider<'_>,
call: ResolvedChatCompletionsRequest,
secrets: Secrets,
cache: Option<litellm_cache_response::ScopedCache>,
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ChatCompletionsResponse, Error> {
let ProviderChatCompletionsRequest {
model,
custom_llm_provider,
config,
url,
body,
optional_params,
environment,
secrets,
timeout,
api_key,
} = request;
let validated = config.validate_environment(
string_headers(call.extra_headers)?,
call.api_key.as_deref(),
provider.model,
&call.optional_params,
&|key| secrets.get(key),
)?;
let url = config.get_complete_url(
call.api_base.as_deref(),
provider.model,
&call.optional_params,
&|key| secrets.get(key),
)?;
let body = config
.transform_request(provider.model, call.messages, call.optional_params.clone())?
.body;
let environment = ValidatedEnvironment {
headers: with_default_headers(validated.headers, config.default_headers()),
auth: validated.auth,
};
let provider_name = <&'static str>::from(provider.provider);
let context = RequestContext {
model: model.clone(),
custom_llm_provider,
optional_params: Value::Object(optional_params),
model: provider.model.to_owned(),
custom_llm_provider: provider_name.to_owned(),
optional_params: Value::Object(call.optional_params),
secret_fields: Vec::new(),
api_key,
api_key: call.api_key.map(litellm_auth::SecretValue::new),
};
let authenticated = resolve_auth(auth, environment, &|key| secrets.get(key)).await?;
let identity = litellm_host::interceptors::ProviderIdentity {
@ -76,7 +98,7 @@ pub(super) async fn execute(
},
wire.url,
&wire.body,
timeout,
call.timeout,
)?;
let response = litellm_inference::outbound::send(outbound, http)
@ -121,7 +143,7 @@ pub(super) async fn execute(
))
})?;
config
.transform_response(&model, ProviderChatResponseData { body })
.transform_response(provider.model, ProviderChatResponseData { body })
.map_err(Error::from)
.map_err(as_response_error)
},
@ -134,7 +156,7 @@ pub(super) async fn execute(
///
/// A config reports the same variants on either side of the call: a missing
/// field or an unsupported block can mean "this request cannot be translated"
/// during prepare and "this response cannot be normalized" here. Only the
/// before the request is sent and "this response cannot be normalized" here. Only the
/// second kind has already been billed, and a host that keeps a reference
/// implementation must not retry those, so collapse them to one variant that
/// can only mean the provider was already called.
@ -178,9 +200,10 @@ mod tests {
use super::*;
use crate::{
prepare::{prepare_provider_request, resolve_request},
types::ChatCompletionsRequest,
common_utils::{chat_completions_provider, resolve_request},
types::{ChatCompletionsCall, ResolvedChatCompletionsRequest},
};
use litellm_inference::provider::resolve_llm_provider;
const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
@ -217,22 +240,33 @@ mod tests {
}
}
fn prepared(api_base: &str) -> ProviderChatCompletionsRequest {
prepare_provider_request(
resolve_request(ChatCompletionsRequest {
model: "anthropic/claude-sonnet-4-5",
fn prepared(
api_base: &str,
) -> (
litellm_inference::provider::ResolvedProvider<'static>,
&'static dyn BaseConfig,
ResolvedChatCompletionsRequest,
) {
let provider =
resolve_llm_provider("anthropic/claude-sonnet-4-5", None, "chat completions").unwrap();
let config = chat_completions_provider(provider.provider)
.expect("provider has a chat completions config")
.config();
let request = resolve_request(
ChatCompletionsCall {
model: "anthropic/claude-sonnet-4-5".into(),
messages: json!([{"role": "user", "content": "hi"}]),
optional_params: json!({"max_tokens": 16}).as_object().unwrap().clone(),
api_key: Some("sk-test"),
api_base: Some(api_base),
api_key: Some("sk-test".into()),
api_base: Some(api_base.into()),
custom_llm_provider: None,
extra_headers: None,
timeout: None,
})
.unwrap(),
std::sync::Arc::new(|_: &str| None),
},
config,
)
.unwrap()
.unwrap();
(provider, config, request)
}
#[rstest]
@ -247,10 +281,14 @@ mod tests {
.await;
let interceptors = RecordingHooks::default();
let (provider, config, request) = prepared(&upstream.uri());
execute(
&Client::plain_for_test(),
&AuthServices::default(),
prepared(&upstream.uri()),
config,
provider,
request,
std::sync::Arc::new(|_: &str| None),
None,
None,
&interceptors,
@ -288,10 +326,14 @@ mod tests {
.await;
let interceptors = RecordingHooks::default();
let (provider, config, request) = prepared(&upstream.uri());
let error = execute(
&Client::plain_for_test(),
&AuthServices::default(),
prepared(&upstream.uri()),
config,
provider,
request,
std::sync::Arc::new(|_: &str| None),
None,
None,
&interceptors,

View file

@ -5,14 +5,13 @@ pub use litellm_inference::RouteError as Error;
mod common_utils;
pub mod constants;
pub(crate) mod handler;
mod prepare;
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use prepare::{prepare_provider_request, resolve_request};
use litellm_auth::AuthServices;
use litellm_inference::provider::resolve_llm_provider;
use litellm_secrets::source::SecretSource;
use std::sync::Arc;
use types::ChatCompletionsRequest;
use types::{ChatCompletionsCall, ChatCompletionsRequest};
#[derive(Clone)]
pub struct ChatCompletionsRoute {
@ -55,7 +54,7 @@ impl ChatCompletionsRoute {
} = options.into();
litellm_host::lifecycle::observe_unary(
observers.clone(),
self.run_call(
self.run(
request.into(),
cache_options,
interceptors,
@ -65,30 +64,52 @@ impl ChatCompletionsRoute {
.await
}
#[tracing::instrument(name = "litellm.route", skip_all, fields(
route = "chat_completions",
model = %call.model,
provider,
resolved_model,
stream = false,
outcome
))]
async fn run(
&self,
request: ChatCompletionsRequest<'_>,
call: ChatCompletionsCall,
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ChatCompletionsResponse, Error> {
let resolved = resolve_request(request)?;
let snapshot = self
.secrets
.resolve(&resolved.config.secret_names())
.await?;
let prepared = prepare_provider_request(resolved, snapshot)?;
litellm_inference::diagnostic::provider(&prepared.model, &prepared.custom_llm_provider);
let execute: futures_util::future::BoxFuture<'_, Result<ChatCompletionsResponse, Error>> =
Box::pin(handler::execute(
litellm_inference::diagnostic::unary(async {
let model = call.model.clone();
let custom_llm_provider = call.custom_llm_provider.clone();
let provider =
resolve_llm_provider(&model, custom_llm_provider.as_deref(), "chat completions")?;
let config = common_utils::chat_completions_provider(provider.provider)
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?
.config();
let call = common_utils::resolve_request(call, config)?;
litellm_inference::diagnostic::provider(
provider.model,
<&'static str>::from(provider.provider),
);
let secrets = self.secrets.resolve(&config.secret_names()).await?;
let execute: futures_util::future::BoxFuture<
'_,
Result<ChatCompletionsResponse, Error>,
> = Box::pin(handler::execute(
&self.http,
&self.auth,
prepared,
config,
provider,
call,
secrets,
self.cache.clone(),
cache_options,
interceptors,
observers,
));
execute.await
execute.await
})
.await
}
}

View file

@ -1,638 +0,0 @@
use litellm_auth::SecretValue;
use litellm_core_utils::settings::Lookup;
use litellm_http::request::with_default_headers;
use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
use litellm_llms_types::formats::chat_completions::ChatMessage;
use litellm_secrets::source::Secrets;
use serde_json::Value;
use super::{
Error,
common_utils::{chat_completions_provider, string_headers},
};
use crate::types::{
ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
};
use litellm_inference::provider::resolve_llm_provider;
pub(super) struct ResolvedProvider {
pub(super) model: String,
pub(super) custom_llm_provider: String,
pub(super) config: &'static dyn BaseConfig,
}
pub(super) fn resolve_provider_config<'a>(
model: &'a str,
custom_llm_provider: Option<&'a str>,
) -> Result<ResolvedProvider, Error> {
let provider_info = resolve_llm_provider(model, custom_llm_provider, "chat completions")?;
let config = chat_completions_provider(provider_info.provider)
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?
.config();
Ok(ResolvedProvider {
model: provider_info.model.to_string(),
custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
config,
})
}
pub(super) fn parse_messages(messages: Value) -> Result<Vec<ChatMessage>, Error> {
serde_json::from_value(messages).map_err(|err| {
Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
"chat completions messages",
err,
))
})
}
pub(super) fn resolve_request(
request: ChatCompletionsRequest<'_>,
) -> Result<ResolvedChatCompletionsRequest<'_>, Error> {
let ResolvedProvider {
model,
custom_llm_provider,
config,
} = resolve_provider_config(request.model, request.custom_llm_provider)?;
let messages = parse_messages(request.messages)?;
if messages.is_empty() {
return Err(Error::InvalidRequest(
"chat completions requires at least one message".into(),
));
}
if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) {
return Err(Error::Unsupported(reason.0));
}
Ok(ResolvedChatCompletionsRequest {
model,
custom_llm_provider,
config,
messages,
optional_params: request.optional_params,
api_key: request.api_key,
api_base: request.api_base,
extra_headers: request.extra_headers,
timeout: request.timeout,
})
}
fn validate_environment(
request: &ResolvedChatCompletionsRequest<'_>,
model: &str,
config: &dyn BaseConfig,
secrets: &dyn Lookup,
) -> Result<ValidatedEnvironment, Error> {
let env_lookup = |key: &str| secrets.get(key);
let forwarded = string_headers(request.extra_headers.clone())?;
let validated = config.validate_environment(
forwarded,
request.api_key,
model,
&request.optional_params,
&env_lookup,
)?;
Ok(ValidatedEnvironment {
headers: with_default_headers(validated.headers, config.default_headers()),
auth: validated.auth,
})
}
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
pub(super) fn prepare_provider_request(
request: ResolvedChatCompletionsRequest<'_>,
secrets: Secrets,
) -> Result<ProviderChatCompletionsRequest, Error> {
let environment =
validate_environment(&request, &request.model, request.config, secrets.as_ref())?;
let model = request.model;
let config = request.config;
let env_lookup = |key: &str| secrets.get(key);
let url = config.get_complete_url(
request.api_base,
&model,
&request.optional_params,
&env_lookup,
)?;
let transformed =
config.transform_request(&model, request.messages, request.optional_params.clone())?;
Ok(ProviderChatCompletionsRequest {
model,
custom_llm_provider: request.custom_llm_provider,
config,
url,
body: transformed.body,
optional_params: request.optional_params,
environment,
secrets,
timeout: request.timeout,
api_key: request.api_key.map(|key| SecretValue::new(key.to_string())),
})
}
#[cfg(test)]
mod tests {
use litellm_auth::CredentialPlacement;
use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth};
use serde_json::{Map, Value, json};
use super::{prepare_provider_request, resolve_request};
use crate::{
Error,
types::{ChatCompletionsRequest, ProviderChatCompletionsRequest},
};
fn prepare_chat_completions_call(
request: ChatCompletionsRequest<'_>,
) -> Result<ProviderChatCompletionsRequest, Error> {
prepare_provider_request(
resolve_request(request)?,
std::sync::Arc::new(|_: &str| None),
)
}
/// The headers as they go on the wire, credential applied.
fn wire_headers(prepared: &ProviderChatCompletionsRequest) -> Vec<(String, String)> {
tokio::runtime::Builder::new_current_thread()
.build()
.unwrap()
.block_on(resolve_auth(
&litellm_auth::AuthServices::default(),
prepared.environment.clone(),
&|_| None,
))
.unwrap()
.headers
}
fn request<'a>(
model: &'a str,
provider: Option<&'a str>,
messages: Value,
optional_params: Value,
) -> ChatCompletionsRequest<'a> {
ChatCompletionsRequest {
model,
messages,
optional_params: match optional_params {
Value::Object(map) => map,
other => panic!("params must be an object, got {other}"),
},
api_key: Some("sk-test"),
api_base: None,
custom_llm_provider: provider,
extra_headers: None,
timeout: None,
}
}
fn preparation_error(request: ChatCompletionsRequest<'_>) -> Error {
prepare_chat_completions_call(request)
.err()
.expect("request preparation should fail")
}
#[test]
fn resolves_the_provider_from_the_model_prefix() {
let prepared = prepare_chat_completions_call(request(
"anthropic/claude-sonnet-4-5",
None,
json!([{"role": "user", "content": "hi"}]),
json!({"max_tokens": 16}),
))
.expect("prepares");
assert_eq!(prepared.model, "claude-sonnet-4-5");
assert_eq!(prepared.url, "https://api.anthropic.com/v1/messages");
assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5"));
}
#[test]
fn strips_an_explicit_provider_prefix_from_the_model() {
let prepared = prepare_chat_completions_call(request(
"anthropic/claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
))
.expect("prepares");
assert_eq!(prepared.model, "claude-sonnet-4-5");
}
#[test]
fn adds_the_auth_and_default_headers() {
let prepared = prepare_chat_completions_call(request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
))
.expect("prepares");
assert!(
wire_headers(&prepared).contains(&("x-api-key".to_string(), "sk-test".to_string()))
);
assert!(
wire_headers(&prepared)
.contains(&("anthropic-version".to_string(), "2023-06-01".to_string()))
);
assert!(matches!(
prepared.environment.auth,
AuthScheme::Credential {
placement: CredentialPlacement::Header("x-api-key"),
..
}
));
}
#[test]
fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
// Python builds `{**headers, **anthropic_headers}`, so the deployment's key
// overwrites a forwarded one. Honouring the caller's would let whoever sends
// the request choose the Anthropic principal it bills to.
let mut call = request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
);
call.extra_headers = Some(Map::from_iter([(
"X-Api-Key".to_string(),
json!("sk-caller"),
)]));
let prepared = prepare_chat_completions_call(call).expect("prepares");
let headers = wire_headers(&prepared);
let keys: Vec<_> = headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
.collect();
assert_eq!(keys.len(), 1, "got {:?}", headers);
assert_eq!(keys[0].1, "sk-test");
}
#[test]
fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
// Anthropic's `validate_environment` pops `x-api-key` and sets `authorization`
// for an OAuth token, so re-adding the key here would put the credential into
// a header the host removed on purpose.
let mut call = request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
);
call.extra_headers = Some(Map::from_iter([
(
"Authorization".to_string(),
json!("Bearer sk-ant-oat01-token"),
),
("X-Api-Key".to_string(), json!("sk-caller")),
]));
let prepared = prepare_chat_completions_call(call).expect("prepares");
assert!(
!wire_headers(&prepared)
.iter()
.any(|(name, value)| name.eq_ignore_ascii_case("x-api-key") && value == "sk-test"),
"the resolved key must not be applied over an OAuth bearer, got {:?}",
wire_headers(&prepared)
);
assert!(
wire_headers(&prepared)
.iter()
.any(|(name, value)| name.eq_ignore_ascii_case("authorization")
&& value == "Bearer sk-ant-oat01-token")
);
}
#[test]
fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
// Only an OAuth bearer replaces the credential. Python sends the deployment's
// `x-api-key` alongside any other forwarded `authorization`, so deferring on
// the mere presence of that header would drop the deployment's auth.
let mut call = request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
);
call.extra_headers = Some(Map::from_iter([
("Authorization".to_string(), json!("Bearer unrelated")),
("X-Api-Key".to_string(), json!("sk-caller")),
]));
let prepared = prepare_chat_completions_call(call).expect("prepares");
let headers = wire_headers(&prepared);
let keys: Vec<_> = headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
.collect();
assert_eq!(keys.len(), 1, "got {:?}", headers);
assert_eq!(keys[0].1, "sk-test");
assert!(
wire_headers(&prepared)
.iter()
.any(|(name, value)| name.eq_ignore_ascii_case("authorization")
&& value == "Bearer unrelated"),
"the unrelated authorization must survive, got {:?}",
wire_headers(&prepared)
);
}
#[rstest::rstest]
fn rejects_empty_messages_before_resolving_credentials() {
let call = ChatCompletionsRequest {
api_key: None,
..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
};
assert!(matches!(preparation_error(call), Error::InvalidRequest(_)));
}
#[test]
fn rejects_an_unknown_provider() {
assert_eq!(
preparation_error(request(
"openai/gpt-4o",
None,
json!([{"role": "user", "content": "hi"}]),
json!({}),
)),
Error::InvalidProvider("openai".to_string())
);
}
#[test]
fn rejects_a_model_with_no_resolvable_provider() {
assert!(matches!(
preparation_error(request(
"claude-sonnet-4-5",
None,
json!([{"role": "user", "content": "hi"}]),
json!({}),
)),
Error::InvalidProvider(_)
));
}
#[test]
fn rejects_an_empty_or_malformed_message_list() {
assert_eq!(
preparation_error(request(
"anthropic/claude-sonnet-4-5",
None,
json!([]),
json!({}),
)),
Error::InvalidRequest(
"chat completions requires at least one message"
.to_string()
.into()
)
);
assert!(matches!(
preparation_error(request(
"anthropic/claude-sonnet-4-5",
None,
json!("not a list"),
json!({}),
)),
Error::InvalidRequest(_)
));
}
#[test]
fn rejects_non_string_extra_headers() {
let mut call = request(
"anthropic/claude-sonnet-4-5",
None,
json!([{"role": "user", "content": "hi"}]),
json!({}),
);
call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
assert_eq!(
preparation_error(call),
Error::Headers(litellm_http::request::HeaderError {
context: "chat completions",
name: "x-trace".to_string(),
actual: "number",
})
);
}
#[test]
fn prepares_a_bedrock_call_without_resolving_credentials() {
let mut call = request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({"maxTokens": 16}),
);
call.api_key = None;
let prepared = prepare_chat_completions_call(call).expect("prepares");
assert_eq!(
prepared.url,
"https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
);
assert!(matches!(
&prepared.environment.auth,
AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == "us-east-1"
));
// SigV4 signs the serialized body, so prepare must not have added an
// Authorization header; the signer does it over the bytes sent.
assert!(
!prepared
.environment
.headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("authorization"))
);
assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16}));
}
#[tokio::test]
async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
// Python signs only the AWS header set and reattaches the rest, so a header
// the caller forwarded rides along without joining the canonical request.
// Signing it makes Converse 403 on a deployment that works on Python.
let mut call = request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({
"maxTokens": 16,
"aws_access_key_id": "AKIDEXAMPLE",
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
}),
);
// A key would resolve to a bearer token and never reach the signer.
call.api_key = None;
call.extra_headers = Some(Map::from_iter([(
"x-request-id".to_string(),
json!("abc-123"),
)]));
let prepared = prepare_chat_completions_call(call).expect("prepares");
let authenticated = resolve_auth(
&litellm_auth::AuthServices::default(),
prepared.environment,
&|_| None,
)
.await
.expect("resolves");
let signed = crate::handler::outbound_request(
authenticated,
prepared.url,
&prepared.body,
prepared.timeout,
)
.expect("signs");
let authorization = signed
.header("authorization")
.expect("carries an authorization header")
.to_string();
assert!(
authorization.starts_with("AWS4-HMAC-SHA256"),
"expected a SigV4 signature, got {authorization}"
);
assert!(
!authorization.contains("x-request-id"),
"forwarded header reached SignedHeaders: {authorization}"
);
// It still goes on the wire, it is just not part of the signature.
assert!(
signed
.headers()
.iter()
.any(|(name, value)| name == "x-request-id" && value == "abc-123"),
"forwarded header was dropped instead of reattached"
);
}
#[rstest::rstest]
#[case::authorization("Authorization")]
#[case::amz_date("x-amz-date")]
#[case::security_token("x-amz-security-token")]
#[case::date("Date")]
#[tokio::test]
async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
let call = ChatCompletionsRequest {
api_key: None,
extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
..request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({
"maxTokens": 16,
"aws_access_key_id": "AKIDEXAMPLE",
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
}),
)
};
let prepared = prepare_chat_completions_call(call).expect("prepares");
let authenticated = resolve_auth(
&litellm_auth::AuthServices::default(),
prepared.environment,
&|_| None,
)
.await
.expect("resolves");
let error = crate::handler::outbound_request(
authenticated,
prepared.url,
&prepared.body,
prepared.timeout,
)
.expect_err("conflicting signing headers must fail");
assert!(
matches!(error, Error::Unsupported(_)),
"{forwarded} returned {error:?}"
);
}
#[test]
fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
// `get_request_headers` assigns `headers["Authorization"]` unconditionally
// once a bearer token resolves, so the deployment's identity wins on
// Python. Keeping the caller's would authorize and bill the call as a
// different principal, and only when the deployment carries `rust: true`.
let mut call = request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({"maxTokens": 16}),
);
call.extra_headers = Some(Map::from_iter([(
"Authorization".to_string(),
json!("Bearer caller-supplied"),
)]));
let prepared = prepare_chat_completions_call(call).expect("prepares");
let headers = wire_headers(&prepared);
let authorizations: Vec<_> = headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("authorization"))
.map(|(_, value)| value.as_str())
.collect();
assert_eq!(
authorizations,
vec!["Bearer sk-test"],
"the deployment token must be the only authorization on the wire"
);
}
#[test]
fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
// The opposite precedence, and deliberate: Anthropic's own transform
// honours a forwarded OAuth bearer, so the Bedrock fix above must not be
// generalized into a rule that the configured key always wins.
//
// An OAuth bearer is the whole of that exception. This forwarded a plain
// `x-api-key` until round 17, which read as the same claim and was not:
// Python overwrites a forwarded `x-api-key` with the deployment's.
let mut call = request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
);
call.extra_headers = Some(Map::from_iter([(
"authorization".to_string(),
json!("Bearer sk-ant-oat01-forwarded"),
)]));
let prepared = prepare_chat_completions_call(call).expect("prepares");
let headers = wire_headers(&prepared);
let keys: Vec<_> = headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
.map(|(_, value)| value.as_str())
.collect();
assert!(keys.is_empty(), "got {:?}", headers);
assert!(
wire_headers(&prepared)
.iter()
.any(|(name, value)| name.eq_ignore_ascii_case("authorization")
&& value == "Bearer sk-ant-oat01-forwarded")
);
}
#[test]
fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
// The configured bearer identity has its own account and quota boundary,
// so a request carrying one must not be signed as whatever principal the
// host's AWS credentials resolve to.
let prepared = prepare_chat_completions_call(request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({"maxTokens": 16}),
))
.expect("prepares");
assert!(matches!(
&prepared.environment.auth,
AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret }
if secret.expose() == "sk-test"
));
assert!(
wire_headers(&prepared)
.iter()
.any(|(name, value)| name.eq_ignore_ascii_case("authorization")
&& value == "Bearer sk-test"),
"prepare did not carry the bearer token"
);
}
}

View file

@ -1,4 +1,3 @@
use litellm_host::observation::ObservationSender;
use std::convert::Infallible;
use litellm_host::{
@ -7,10 +6,7 @@ use litellm_host::{
};
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use super::{
ChatCompletionsRoute, Error,
types::{ChatCompletionsCall, ChatCompletionsRequest},
};
use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
pub struct ChatCompletions;
@ -37,44 +33,12 @@ impl ChatCompletionsRoute {
call,
observers,
move |call, _, interceptors, observers| async move {
self.run_call(call, cache_options, &interceptors, observers.as_ref())
self.run(call, cache_options, &interceptors, observers.as_ref())
.await
.map(CallOutput::Complete)
},
)
}
#[tracing::instrument(name = "litellm.route", skip_all, fields(
route = "chat_completions",
model = %call.model,
provider,
resolved_model,
stream = false,
outcome
))]
pub(super) async fn run_call(
&self,
call: ChatCompletionsCall,
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ChatCompletionsResponse, Error> {
litellm_inference::diagnostic::unary(async {
let request = ChatCompletionsRequest {
model: &call.model,
messages: call.messages,
optional_params: call.optional_params,
api_key: call.api_key.as_deref(),
api_base: call.api_base.as_deref(),
custom_llm_provider: call.custom_llm_provider.as_deref(),
extra_headers: call.extra_headers,
timeout: call.timeout,
};
self.run(request, cache_options, interceptors, observers)
.await
})
.await
}
}
impl litellm_inference::caching::Cachable for ChatCompletions {

View file

@ -1,8 +1,5 @@
use litellm_secrets::source::Secrets;
use std::time::Duration;
use litellm_auth::SecretValue;
use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
use litellm_llms_types::formats::chat_completions::ChatMessage;
use serde_json::{Map, Value};
@ -49,31 +46,11 @@ impl From<ChatCompletionsRequest<'_>> for ChatCompletionsCall {
}
}
pub struct ResolvedChatCompletionsRequest<'a> {
pub model: String,
pub custom_llm_provider: String,
pub config: &'static dyn BaseConfig,
pub struct ResolvedChatCompletionsRequest {
pub messages: Vec<ChatMessage>,
pub optional_params: Map<String, Value>,
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub api_key: Option<String>,
pub api_base: Option<String>,
pub extra_headers: Option<Map<String, Value>>,
pub timeout: Option<Duration>,
}
pub struct ProviderChatCompletionsRequest {
pub model: String,
pub custom_llm_provider: String,
pub config: &'static dyn BaseConfig,
pub url: String,
pub body: Value,
/// The route's parameters before the provider transformation, reported to the host
/// beside the wire request.
pub optional_params: Map<String, Value>,
/// The forwarded and default headers plus how the call authenticates; the credential
/// itself is applied when the request is sent.
pub environment: ValidatedEnvironment,
pub secrets: Secrets,
pub timeout: Option<Duration>,
pub api_key: Option<SecretValue>,
}

View file

@ -0,0 +1,439 @@
use std::time::Duration;
use litellm_inference_chat::{Error, types::ChatCompletionsRequest};
use rstest::rstest;
use serde_json::{Map, Value, json};
use wiremock::ResponseTemplate;
mod support;
use support::*;
const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
fn request<'a>(
model: &'a str,
provider: Option<&'a str>,
messages: Value,
optional_params: Value,
) -> ChatCompletionsRequest<'a> {
let Value::Object(optional_params) = optional_params else {
panic!("params must be an object");
};
ChatCompletionsRequest {
model,
messages,
optional_params,
api_key: Some("sk-test"),
api_base: None,
custom_llm_provider: provider,
extra_headers: None,
timeout: Some(Duration::from_secs(10)),
}
}
async fn complete(
request: ChatCompletionsRequest<'_>,
) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletionsResponse, Error> {
chat_completions_route().execute(request, &(), None).await
}
fn object(value: Value) -> Map<String, Value> {
let Value::Object(map) = value else {
panic!("expected an object");
};
map
}
fn anthropic_response() -> ResponseTemplate {
ResponseTemplate::new(200).set_body_raw(ANTHROPIC_MESSAGE, "application/json")
}
fn bedrock_response() -> ResponseTemplate {
json_response(json!({
"output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}
}))
}
#[rstest]
#[tokio::test]
async fn resolves_the_provider_from_the_model_prefix() {
let upstream = upstream([anthropic_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
..request(
"anthropic/claude-sonnet-4-5",
None,
json!([{"role": "user", "content": "hi"}]),
json!({"max_tokens": 16}),
)
})
.await
.expect("call succeeds");
let sent = only_request(&upstream).await;
assert_eq!(sent.url.path(), "/v1/messages");
assert_eq!(sent.json()["model"], "claude-sonnet-4-5");
}
#[rstest]
#[tokio::test]
async fn strips_an_explicit_provider_prefix_from_the_model() {
let upstream = upstream([anthropic_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
..request(
"anthropic/claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
)
})
.await
.expect("call succeeds");
assert_eq!(
only_request(&upstream).await.json()["model"],
"claude-sonnet-4-5"
);
}
#[rstest]
#[tokio::test]
async fn adds_the_auth_and_default_headers() {
let upstream = upstream([anthropic_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
..request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
)
})
.await
.expect("call succeeds");
let sent = only_request(&upstream).await;
assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
assert_eq!(sent.header("anthropic-version"), Some("2023-06-01"));
}
#[rstest]
#[tokio::test]
async fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
let upstream = upstream([anthropic_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
extra_headers: Some(object(json!({"X-Api-Key": "sk-caller"}))),
..request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
)
})
.await
.expect("call succeeds");
assert_eq!(
only_request(&upstream).await.header_values("x-api-key"),
["sk-test"]
);
}
#[rstest]
#[tokio::test]
async fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
let upstream = upstream([anthropic_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
extra_headers: Some(object(json!({
"Authorization": "Bearer sk-ant-oat01-token",
"X-Api-Key": "sk-caller"
}))),
..request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
)
})
.await
.expect("call succeeds");
let sent = only_request(&upstream).await;
assert_eq!(sent.header_values("x-api-key"), ["sk-caller"]);
assert_eq!(
sent.header_values("authorization"),
["Bearer sk-ant-oat01-token"]
);
}
#[rstest]
#[tokio::test]
async fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
let upstream = upstream([anthropic_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
extra_headers: Some(object(json!({
"Authorization": "Bearer unrelated",
"X-Api-Key": "sk-caller"
}))),
..request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
)
})
.await
.expect("call succeeds");
let sent = only_request(&upstream).await;
assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
assert_eq!(sent.header_values("authorization"), ["Bearer unrelated"]);
}
#[rstest]
#[tokio::test]
async fn rejects_empty_messages_before_resolving_credentials() {
let error = complete(ChatCompletionsRequest {
api_key: None,
..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
})
.await
.expect_err("empty messages are rejected");
assert!(matches!(error, Error::InvalidRequest(_)));
}
#[rstest]
#[tokio::test]
async fn rejects_an_unknown_provider() {
assert_eq!(
complete(request(
"openai/gpt-4o",
None,
json!([{"role": "user", "content": "hi"}]),
json!({}),
))
.await
.unwrap_err(),
Error::InvalidProvider("openai".into())
);
}
#[rstest]
#[tokio::test]
async fn rejects_a_model_with_no_resolvable_provider() {
assert!(matches!(
complete(request(
"claude-sonnet-4-5",
None,
json!([{"role": "user", "content": "hi"}]),
json!({}),
))
.await,
Err(Error::InvalidProvider(_))
));
}
#[rstest]
#[case::empty(json!([]), Some("chat completions requires at least one message"))]
#[case::malformed(json!("not a list"), None)]
#[tokio::test]
async fn rejects_an_empty_or_malformed_message_list(
#[case] messages: Value,
#[case] expected: Option<&str>,
) {
let error = complete(request(
"anthropic/claude-sonnet-4-5",
None,
messages,
json!({}),
))
.await
.unwrap_err();
if let Some(expected) = expected {
assert_eq!(error, Error::InvalidRequest(expected.to_string().into()));
} else {
assert!(matches!(error, Error::InvalidRequest(_)));
}
}
#[rstest]
#[tokio::test]
async fn rejects_non_string_extra_headers() {
assert_eq!(
complete(ChatCompletionsRequest {
extra_headers: Some(Map::from_iter([("x-trace".to_string(), json!(7))])),
..request(
"anthropic/claude-sonnet-4-5",
None,
json!([{"role": "user", "content": "hi"}]),
json!({}),
)
})
.await
.unwrap_err(),
Error::Headers(litellm_http::request::HeaderError {
context: "chat completions",
name: "x-trace".to_string(),
actual: "number",
})
);
}
#[rstest]
#[tokio::test]
async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
let upstream = upstream([bedrock_response()]).await;
let base = upstream.uri();
let mut call = request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({
"maxTokens": 16,
"aws_access_key_id": "AKIDEXAMPLE",
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
}),
);
call.api_key = None;
call.api_base = Some(&base);
call.extra_headers = Some(Map::from_iter([(
"x-request-id".to_string(),
json!("abc-123"),
)]));
complete(call).await.expect("call succeeds");
let sent = only_request(&upstream).await;
let authorization = sent.header("authorization").expect("signed request");
assert!(authorization.starts_with("AWS4-HMAC-SHA256"));
assert!(!authorization.contains("x-request-id"));
assert_eq!(sent.header("x-request-id"), Some("abc-123"));
assert_eq!(sent.url.path(), "/model/anthropic.claude-v2/converse");
let body: Value = serde_json::from_slice(&sent.body).unwrap();
assert_eq!(body["inferenceConfig"], json!({"maxTokens": 16}));
assert_eq!(
body["messages"],
json!([{"role": "user", "content": [{"text": "hi"}]}])
);
}
#[rstest]
#[case::authorization("Authorization")]
#[case::amz_date("x-amz-date")]
#[case::security_token("x-amz-security-token")]
#[case::date("Date")]
#[tokio::test]
async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
let upstream = wiremock::MockServer::start().await;
let base = upstream.uri();
let call = ChatCompletionsRequest {
api_key: None,
api_base: Some(&base),
extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
..request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({
"maxTokens": 16,
"aws_access_key_id": "AKIDEXAMPLE",
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
}),
)
};
let error = complete(call)
.await
.expect_err("conflicting signing headers fail");
assert!(matches!(error, Error::Unsupported(_)), "{error:?}");
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[tokio::test]
async fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
let upstream = upstream([bedrock_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
extra_headers: Some(Map::from_iter([(
"Authorization".to_string(),
json!("Bearer caller-supplied"),
)])),
..request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({"maxTokens": 16}),
)
})
.await
.expect("call succeeds");
assert_eq!(
only_request(&upstream).await.header_values("authorization"),
["Bearer sk-test"]
);
}
#[rstest]
#[tokio::test]
async fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
let upstream = upstream([anthropic_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
extra_headers: Some(Map::from_iter([(
"authorization".to_string(),
json!("Bearer sk-ant-oat01-forwarded"),
)])),
..request(
"claude-sonnet-4-5",
Some("anthropic"),
json!([{"role": "user", "content": "hi"}]),
json!({}),
)
})
.await
.expect("call succeeds");
let sent = only_request(&upstream).await;
assert!(sent.header_values("x-api-key").is_empty());
assert_eq!(
sent.header_values("authorization"),
["Bearer sk-ant-oat01-forwarded"]
);
}
#[rstest]
#[tokio::test]
async fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
let upstream = upstream([bedrock_response()]).await;
let base = upstream.uri();
complete(ChatCompletionsRequest {
api_base: Some(&base),
..request(
"bedrock/us-east-1/anthropic.claude-v2",
None,
json!([{"role": "user", "content": "hi"}]),
json!({"maxTokens": 16}),
)
})
.await
.expect("call succeeds");
assert_eq!(
only_request(&upstream).await.header_values("authorization"),
["Bearer sk-test"]
);
}

View file

@ -7,8 +7,8 @@
- delegate authentication policy, beta selection, payload rewriting, and response interpretation to those adapters
- keep provider policy out of request preparation and transport handlers
- calling a concrete provider helper for every provider is still a policy dependency
- Route types such as `MessagesCall`, prepared requests, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
- Route types such as `MessagesCall`, provider calls, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
- The route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks
- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `src/lib.rs` explicitly sequences cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `handler::execute` explicitly sequences the provider call build in Python's handler order, cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
- Preserve the order of validation, normalization, caller-requested parameter removal, and provider transformation when that order affects observable behavior
- Test provider dispatch, auth precedence, header handling, transformations, and responses through behavior, not source structure

View file

@ -82,17 +82,17 @@ mod tests {
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
#[rstest]
#[case::tencent("tencent")]
#[case::anthropic("anthropic")]
#[case::azure_ai("azure_ai")]
#[case::bedrock("bedrock")]
#[case::deepseek("deepseek")]
#[case::edenai("edenai")]
#[case::minimax("minimax")]
#[case::vertex_ai("vertex_ai")]
fn provider_keeps_its_python_name(#[case] name: &str) {
let provider =
messages_provider(name.parse::<LlmProviders>().unwrap(), "claude-sonnet-4-5").unwrap();
#[case::tencent(LlmProviders::Tencent, "tencent")]
#[case::anthropic(LlmProviders::Anthropic, "anthropic")]
#[case::azure_ai(LlmProviders::AzureAi, "azure_ai")]
#[case::bedrock(LlmProviders::Bedrock, "bedrock")]
#[case::deepseek(LlmProviders::Deepseek, "deepseek")]
#[case::edenai(LlmProviders::Edenai, "edenai")]
#[case::minimax(LlmProviders::Minimax, "minimax")]
#[case::vertex_ai(LlmProviders::VertexAi, "vertex_ai")]
#[case::openrouter(LlmProviders::Openrouter, "openrouter")]
fn provider_keeps_its_python_name(#[case] provider: LlmProviders, #[case] name: &str) {
let provider = messages_provider(provider, "claude-sonnet-4-5").unwrap();
assert_eq!(provider.as_str(), name);
}

View file

@ -1,148 +1,271 @@
use std::time::Duration;
use bytes::Bytes;
use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
use litellm_host::interceptors::{Interceptors, ProviderIdentity, RequestContext, WireRequest};
use litellm_http::transport::Error as TransportError;
use futures_util::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream};
use litellm_auth::{AuthServices, SecretValue};
use litellm_core_utils::{
dot_notation_indexing::delete_nested_value,
get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
};
use litellm_host::interceptors::{
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
};
use litellm_http::{request::with_default_headers, transport::Error as TransportError};
use litellm_llms::base_llm::{
auth::{Authenticated, resolve_auth},
auth::{Authenticated, ValidatedEnvironment, resolve_auth},
messages::{
context::MessagesTransformContext,
streaming::{self, ByteStream, StreamDecoder},
transformation::BaseMessagesConfig,
},
};
use litellm_llms_types::formats::messages::MessagesResponse;
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
use litellm_secrets::source::Secrets;
use litellm_tracing::ByteChunk;
use serde_json::Value;
use super::{
Error, MessagesCallResponse, MessagesRoute, common_utils::truncate_error_body,
prepare::ProviderMessagesRequest,
Error, MessagesCall, MessagesCallResponse, MessagesRoute,
common_utils::{MessagesProvider, string_headers, truncate_error_body},
constants::MESSAGES_TIMEOUT_SECS,
route::Messages,
types::invalid_request,
};
use crate::constants::MESSAGES_TIMEOUT_SECS;
use litellm_inference::{context::CallContext, outbound::outbound_request};
use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request};
pub(super) struct ProviderCall {
pub identity: ProviderIdentity,
pub wire: WireRequest,
pub(super) provider: super::common_utils::MessagesProvider,
struct ProviderCall {
identity: ProviderIdentity,
wire: WireRequest,
provider: MessagesProvider,
signer: Option<litellm_auth_aws::SigV4Signer>,
timeout: Option<Duration>,
stream: bool,
}
impl ProviderCall {
pub fn cacheable(&self) -> bool {
fn cacheable(&self) -> bool {
self.signer.is_none()
}
}
impl MessagesRoute {
pub(super) async fn prepare_outbound(
&self,
request: ProviderMessagesRequest,
context: &CallContext<'_, impl Interceptors<Error>>,
) -> Result<ProviderCall, Error> {
let ProviderMessagesRequest {
provider,
url,
body,
environment,
timeout,
api_key,
} = request;
let request_context = RequestContext {
model: body.model.clone(),
custom_llm_provider: provider.as_str().to_string(),
optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
secret_fields: Vec::new(),
api_key,
};
let authenticated =
resolve_auth(&self.auth, environment, &|key| std::env::var(key).ok()).await?;
let identity = ProviderIdentity {
model: request_context.model.clone(),
provider: request_context.custom_llm_provider.clone(),
};
let wire = context
.interceptors
.before_provider_request(
WireRequest {
url,
headers: authenticated.headers,
body: provider
.config()
.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
},
request_context,
)
.await?;
let stream = match wire.body.get("stream") {
None | Some(Value::Null) => false,
Some(Value::Bool(stream)) => *stream,
Some(value) => {
return Err(Error::InvalidRequest(
litellm_llms::ErrorDetail::InvalidValue {
field: "stream",
expected: "a boolean",
actual: value.clone(),
},
));
}
};
Ok(ProviderCall {
identity,
wire,
provider,
signer: authenticated.signer,
timeout,
stream,
pub(super) async fn execute(
route: &MessagesRoute,
provider: MessagesProvider,
model: String,
call: MessagesCall,
secrets: Secrets,
context: &CallContext<'_, impl Interceptors<Error>>,
) -> Result<MessagesCallResponse, Error> {
let request = provider_call(
&route.auth,
provider,
model,
call,
secrets.as_ref(),
context,
)
.boxed()
.await?;
let cache = CallCache::<Messages>::from_wire(
route.cache.as_ref().filter(|_| request.cacheable()),
context.cache,
&request.identity,
&request.wire,
);
let identity = request.identity.clone();
let (output, source) = match cache.lookup().await {
Some(hit) => hit,
None => (
call_provider(&route.http, request, context).await?,
ResultSource::Provider,
),
};
let reported = match &output {
MessagesCallResponse::Complete(response) => provider.config().reported_cost(response),
MessagesCallResponse::Stream { .. } => None,
};
context
.result_ready(ExecutionFacts {
provider: identity,
cost: Cost::settle(&source, reported),
source: source.clone(),
})
}
.await?;
Ok(cache.finish(output, &source).await)
}
pub(super) async fn call_provider(
&self,
request: ProviderCall,
context: &CallContext<'_, impl Interceptors<Error>>,
) -> Result<MessagesCallResponse, Error> {
let ProviderCall {
identity,
wire,
provider,
signer,
timeout,
stream,
} = request;
let provider_name = provider.as_str();
log_request_body(provider_name, stream, &wire.body);
let response = send(
&self.http,
Authenticated {
headers: wire.headers,
signer,
async fn provider_call(
auth: &AuthServices,
provider: MessagesProvider,
model: String,
call: MessagesCall,
secrets: &(dyn Lookup + Send + Sync),
context: &CallContext<'_, impl Interceptors<Error>>,
) -> Result<ProviderCall, Error> {
let MessagesCall {
body,
api_key,
api_base,
litellm_params,
extra_headers,
provider_specific_header,
timeout,
shaping,
..
} = call;
let connection_params = litellm_router_types::LitellmParams {
api_base: api_base.clone().or(litellm_params.api_base.clone()),
..litellm_params
};
let config = provider.config();
let env_lookup = |key: &str| secrets.get(key);
let sanitized = config.shape_request(
MessagesRequest { model, ..body },
shaping.settings.reasoning_auto_summary,
)?;
let trimmed =
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
let scoped =
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
let forwarded = string_headers(Some(
extra_headers.into_iter().flatten().chain(scoped).collect(),
))?;
let validated = config.validate_environment(
forwarded,
api_key.as_deref(),
&trimmed.model,
&trimmed.params,
&connection_params,
&env_lookup,
)?;
let body = config.transform_anthropic_messages_request(
trimmed,
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
with_default_headers(validated.headers, config.default_headers()),
&body,
),
auth: validated.auth,
};
let url = config.get_complete_url(
api_base.as_deref(),
&body.model,
&connection_params,
body.params.stream == Some(true),
&env_lookup,
)?;
let request_context = RequestContext {
model: body.model.clone(),
custom_llm_provider: provider.as_str().to_string(),
optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
secret_fields: Vec::new(),
api_key: api_key.map(SecretValue::new),
};
let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
let identity = ProviderIdentity {
model: request_context.model.clone(),
provider: request_context.custom_llm_provider.clone(),
};
let wire = context
.interceptors
.before_provider_request(
WireRequest {
url,
headers: authenticated.headers,
body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
},
&wire.url,
&wire.body,
timeout,
request_context,
)
.await?;
if !response.status().is_success() {
return Err(provider_error(response).await);
}
let config = provider.config();
if stream {
return Ok(streaming_response(
response,
config.stream_decoder(),
provider_name,
let stream = match wire.body.get("stream") {
None | Some(Value::Null) => false,
Some(Value::Bool(stream)) => *stream,
Some(value) => {
return Err(Error::InvalidRequest(
litellm_llms::ErrorDetail::InvalidValue {
field: "stream",
expected: "a boolean",
actual: value.clone(),
},
));
}
let text = response.text().await.map_err(network)?;
log_response_body(&text);
context.response_received(&text).await?;
decode_response(config, &identity.model, &text)
.map(|message| MessagesCallResponse::Complete(Box::new(message)))
};
Ok(ProviderCall {
identity,
wire,
provider,
signer: authenticated.signer,
timeout: timeout.or_else(|| shaping.settings.request_timeout()),
stream,
})
}
fn without_additional_drop_params(
request: MessagesRequest,
paths: &[String],
) -> Result<MessagesRequest, Error> {
if paths.is_empty() {
return Ok(request);
}
let params = serde_json::to_value(request.params).map_err(invalid_request)?;
let trimmed = paths
.iter()
.fold(params, |params, path| delete_nested_value(params, path));
Ok(MessagesRequest {
params: serde_json::from_value(trimmed).map_err(invalid_request)?,
..request
})
}
async fn call_provider(
http: &litellm_http::Client,
request: ProviderCall,
context: &CallContext<'_, impl Interceptors<Error>>,
) -> Result<MessagesCallResponse, Error> {
let ProviderCall {
identity,
wire,
provider,
signer,
timeout,
stream,
} = request;
let provider_name = provider.as_str();
log_request_body(provider_name, stream, &wire.body);
let response = send(
http,
Authenticated {
headers: wire.headers,
signer,
},
&wire.url,
&wire.body,
timeout,
)
.await?;
if !response.status().is_success() {
return Err(provider_error(response).await);
}
let config = provider.config();
if stream {
return Ok(streaming_response(
response,
config.stream_decoder(),
provider_name,
));
}
let text = response.text().await.map_err(network)?;
log_response_body(&text);
context.response_received(&text).await?;
decode_response(config, &identity.model, &text)
.map(|message| MessagesCallResponse::Complete(Box::new(message)))
}
fn serialize_failure(err: serde_json::Error) -> Error {
@ -271,10 +394,482 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) {
#[cfg(test)]
mod tests {
use rstest::rstest;
use litellm_llms_types::headers::ProviderSpecificHeaders;
use rstest::{fixture, rstest};
use serde_json::{Map, Value, json};
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
use super::*;
use crate::{MessagesSettings, MessagesShaping, common_utils::messages_provider};
use litellm_inference::provider::resolve_llm_provider;
#[fixture]
fn shaping() -> MessagesShaping {
MessagesShaping::default()
}
fn body(value: Value) -> MessagesRequest {
serde_json::from_value(value).unwrap()
}
async fn provider_call_for(call: MessagesCall) -> Result<ProviderCall, Error> {
provider_call_with_secrets(call, &|_: &str| None).await
}
async fn provider_call_with_secrets(
call: MessagesCall,
secrets: &(dyn Lookup + Send + Sync),
) -> Result<ProviderCall, Error> {
let provider = resolve_llm_provider(
&call.body.model,
call.custom_llm_provider.as_deref(),
"messages",
)?;
let config = messages_provider(provider.provider, provider.model)
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?;
let model = provider.model.to_owned();
let interceptors = ();
let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default());
provider_call(
&AuthServices::default(),
config,
model,
call,
secrets,
&context,
)
.await
}
async fn wire_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
provider_call_for(MessagesCall {
body: body(fields),
api_key: Some("sk-test".into()),
api_base: Some("https://anthropic.test".into()),
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping,
})
.await
.map(|call| call.wire.body)
}
#[rstest]
#[case::api_key(
&[("ANTHROPIC_API_KEY", "sk-secret")],
&[("x-api-key", "sk-secret")],
"https://api.anthropic.com/v1/messages"
)]
#[case::auth_token(
&[("ANTHROPIC_AUTH_TOKEN", "token")],
&[("authorization", "Bearer token")],
"https://api.anthropic.com/v1/messages"
)]
#[case::api_base(
&[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
&[("x-api-key", "sk-secret")],
"https://gateway.test/v1/messages"
)]
#[case::sdk_base_url(
&[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
&[("x-api-key", "sk-secret")],
"https://sdk.test/v1/messages"
)]
#[tokio::test]
async fn credentials_and_base_come_from_the_resolved_secrets(
shaping: MessagesShaping,
#[case] secrets: &[(&str, &str)],
#[case] expected_auth: &[(&str, &str)],
#[case] expected_url: &str,
) {
let lookup = |name: &str| {
secrets
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
};
let call = provider_call_with_secrets(
MessagesCall {
body: body(
json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
),
api_key: None,
api_base: None,
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping,
},
&lookup,
)
.await
.unwrap();
let auth: Vec<(&str, &str)> = call
.wire
.headers
.iter()
.filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
.map(|(name, value)| (name.as_str(), value.as_str()))
.collect();
assert_eq!(
(auth.as_slice(), call.wire.url.as_str()),
(expected_auth, expected_url)
);
}
#[rstest]
#[tokio::test]
async fn missing_credentials_are_reported_before_the_request_is_transformed(
shaping: MessagesShaping,
) {
let result = provider_call_for(MessagesCall {
body: body(
json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}),
),
api_key: None,
api_base: None,
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping,
})
.await;
assert!(
matches!(
result,
Err(Error::Auth(litellm_auth::Error::MissingApiKey { .. }))
),
"{:?}",
result.err()
);
}
#[rstest]
#[case::nothing_configured(None, None, None)]
#[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
#[case::call_timeout_wins(
Some(Duration::from_secs(30)),
Some(1800.0),
Some(Duration::from_secs(30))
)]
#[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
#[case::unusable_setting_ignored(None, Some(-1.0), None)]
#[case::zero_setting_ignored(None, Some(0.0), None)]
#[tokio::test]
async fn timeout_falls_back_to_the_configured_request_timeout(
shaping: MessagesShaping,
#[case] call_timeout: Option<Duration>,
#[case] request_timeout: Option<f64>,
#[case] expected: Option<Duration>,
) {
let call = provider_call_for(MessagesCall {
body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
api_key: Some("sk-test".into()),
api_base: Some("https://anthropic.test".into()),
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: call_timeout,
shaping: MessagesShaping {
settings: MessagesSettings { request_timeout, ..shaping.settings },
..shaping
},
})
.await
.unwrap();
assert_eq!(call.timeout, expected);
}
#[rstest]
#[case::top_level_and_nested_paths(
json!({
"max_tokens": 1024,
"thinking": {"type": "enabled", "budget_tokens": 2048},
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
"metadata": {"user_id": "u1"},
"tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
}),
&["thinking", "context_management", "tools[*].input_examples"],
json!({
"max_tokens": 1024,
"metadata": {"user_id": "u1"},
"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
}),
)]
#[case::no_paths(
json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
&[],
json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
)]
#[case::model_and_messages_are_never_dropped(
json!({"max_tokens": 16}),
&["model", "messages", "messages[0].content"],
json!({"max_tokens": 16}),
)]
#[tokio::test]
async fn wire_body_drops_configured_paths(
shaping: MessagesShaping,
#[case] fields: Value,
#[case] additional_drop_params: &[&str],
#[case] expected_fields: Value,
) {
let with_messages = |fields: Value| -> Value {
let Value::Object(fields) = fields else {
unreachable!()
};
Value::Object(
[
("model".to_string(), json!("claude-test")),
(
"messages".to_string(),
json!([{"role": "user", "content": "hi"}]),
),
]
.into_iter()
.chain(fields)
.collect(),
)
};
let shaping = MessagesShaping {
settings: MessagesSettings {
additional_drop_params: additional_drop_params
.iter()
.map(ToString::to_string)
.collect(),
..shaping.settings
},
..shaping
};
assert_eq!(
wire_body(with_messages(fields), shaping).await,
Ok(with_messages(expected_fields))
);
}
#[rstest]
#[case::model_prefix_picks_the_provider(
"azure_ai/claude-test",
None,
&[("x-priority", "extra"), ("x-scoped", "azure_ai")]
)]
#[case::explicit_provider(
"claude-test",
Some("anthropic"),
&[("x-priority", "scoped"), ("x-scoped", "anthropic")]
)]
#[case::provider_prefix_on_an_anthropic_model(
"anthropic/claude-test",
None,
&[("x-priority", "scoped"), ("x-scoped", "anthropic")]
)]
#[tokio::test]
async fn provider_specific_headers_follow_the_resolved_provider(
shaping: MessagesShaping,
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
#[case] expected: &[(&str, &str)],
) {
let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
{"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
{"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
]))
.unwrap();
let call = provider_call_for(MessagesCall {
body: body(
json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
),
api_key: Some("sk-test".into()),
api_base: Some("https://resource.services.ai.azure.com".into()),
custom_llm_provider: custom_llm_provider.map(Into::into),
litellm_params: Default::default(),
extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
provider_specific_header: Some(configured),
timeout: None,
shaping,
})
.await
.unwrap();
let caller_headers: Vec<(&str, &str)> = call
.wire
.headers
.iter()
.filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
.map(|(name, value)| (name.as_str(), value.as_str()))
.collect();
assert_eq!(caller_headers, expected);
}
#[rstest]
#[tokio::test]
async fn wire_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
assert_eq!(
wire_body(
json!({
"model": "anthropic/claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16
}),
shaping,
)
.await,
Ok(json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16
}))
);
}
#[rstest]
#[tokio::test]
async fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
let shaping = MessagesShaping {
settings: MessagesSettings {
reasoning_auto_summary: true,
additional_drop_params: vec!["thinking.display".to_string()],
..shaping.settings
},
..shaping
};
assert_eq!(
wire_body(
json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 4096,
"thinking": {"type": "enabled", "budget_tokens": 2048}
}),
shaping,
)
.await,
Ok(json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 4096,
"thinking": {"type": "enabled", "budget_tokens": 2048}
}))
);
}
#[rstest]
#[tokio::test]
async fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(
shaping: MessagesShaping,
) {
let shaping = MessagesShaping {
settings: MessagesSettings {
additional_drop_params: vec!["metadata.user_id".to_string()],
..shaping.settings
},
..shaping
};
assert!(matches!(
wire_body(
json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16,
"metadata": {"user_id": 123}
}),
shaping,
)
.await,
Err(Error::InvalidRequest(_))
));
}
#[rstest]
#[tokio::test]
async fn wire_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
assert_eq!(
wire_body(
json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16,
"metadata": {"user_id": 123}
}),
shaping,
)
.await,
Err(Error::InvalidRequest(
litellm_llms::ErrorDetail::InvalidValue {
field: "metadata.user_id",
expected: "a string",
actual: json!(123),
}
))
);
}
#[rstest]
#[case::mantle(
"bedrock_mantle/claude-test",
"https://bedrock-mantle.us-west-2.api.aws",
"/anthropic/v1/messages",
"bedrock"
)]
#[case::platform(
"bedrock/claude_platform/claude-test",
"https://aws-external-anthropic.us-west-2.api.aws",
"/v1/messages",
"aws-external-anthropic"
)]
#[tokio::test]
async fn explicit_endpoint_region_is_used_by_the_signer(
shaping: MessagesShaping,
#[case] model: &str,
#[case] base: &str,
#[case] path: &str,
#[case] service: &str,
) {
let call = provider_call_with_secrets(MessagesCall {
body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
api_key: None,
api_base: Some(base.into()),
custom_llm_provider: None,
litellm_params: crate::litellm_params(Map::from_iter([
("model".into(), json!(model)),
("aws_access_key_id".into(), json!("AKIDEXAMPLE")),
("aws_secret_access_key".into(), json!("secret")),
]))
.unwrap(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping,
}, &|name: &str| match name {
"AWS_REGION_NAME" => Some("us-east-1".into()),
"ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
_ => None,
})
.await
.unwrap();
assert_eq!(call.wire.url, format!("{base}{path}"));
let signed = outbound_request(
Authenticated {
headers: call.wire.headers,
signer: call.signer,
},
call.wire.url,
&call.wire.body,
None,
)
.unwrap();
let authorization = signed.header("authorization").unwrap_or_default();
assert!(
authorization.contains(&format!("/us-west-2/{service}/aws4_request")),
"{authorization}"
);
}
#[rstest]
#[case::event(

View file

@ -1,18 +1,19 @@
mod common_utils;
mod constants;
mod handler;
mod prepare;
pub mod route;
mod types;
use futures_util::FutureExt;
use litellm_auth::AuthServices;
use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource};
use litellm_inference::{caching::CallCache, context::CallContext};
use litellm_host::interceptors::Interceptors;
use litellm_inference::context::CallContext;
use litellm_inference::provider::resolve_llm_provider;
use litellm_secrets::source::SecretSource;
use std::sync::Arc;
use crate::common_utils::messages_provider;
pub use litellm_inference::RouteError as Error;
pub use types::{
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
@ -74,39 +75,23 @@ impl MessagesRoute {
context: CallContext<'_, impl Interceptors<Error>>,
) -> Result<MessagesCallResponse, Error> {
litellm_inference::diagnostic::call(async {
let prepared = prepare::prepare(call, self.secrets.as_ref()).await?;
let provider = resolve_llm_provider(
&call.body.model,
call.custom_llm_provider.as_deref(),
"messages",
)?;
let config = messages_provider(provider.provider, provider.model).ok_or_else(|| {
Error::InvalidProvider(<&str>::from(provider.provider).to_string())
})?;
litellm_inference::diagnostic::provider(
&prepared.body.model,
prepared.provider.as_str(),
provider.model,
<&'static str>::from(provider.provider),
);
let request = self.prepare_outbound(prepared, &context).boxed().await?;
let cache = CallCache::<route::Messages>::from_wire(
self.cache.as_ref().filter(|_| request.cacheable()),
context.cache,
&request.identity,
&request.wire,
);
let identity = request.identity.clone();
let config = request.provider.config();
let (output, source) = match cache.lookup().await {
Some(hit) => hit,
None => (
self.call_provider(request, &context).await?,
ResultSource::Provider,
),
};
let reported = match &output {
MessagesCallResponse::Complete(response) => config.reported_cost(response),
MessagesCallResponse::Stream { .. } => None,
};
context
.result_ready(ExecutionFacts {
provider: identity,
cost: Cost::settle(&source, reported),
source: source.clone(),
})
.await?;
Ok(cache.finish(output, &source).await)
let model = provider.model.to_owned();
let secrets = self.secrets.resolve(config.config().secret_names()).await?;
handler::execute(self, config, model, call, secrets, &context)
.boxed()
.await
})
.await
}

View file

@ -1,554 +0,0 @@
use std::time::Duration;
use litellm_auth::SecretValue;
use litellm_core_utils::{
dot_notation_indexing::delete_nested_value,
get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
};
use litellm_http::request::with_default_headers;
use litellm_llms::base_llm::{
auth::ValidatedEnvironment, messages::context::MessagesTransformContext,
};
use litellm_llms_types::formats::messages::MessagesRequest;
use litellm_secrets::source::SecretSource;
use super::{
Error, MessagesCall,
common_utils::{MessagesProvider, messages_provider, string_headers},
types::invalid_request,
};
use litellm_inference::provider::resolve_llm_provider;
struct ResolvedProvider {
model: String,
provider: MessagesProvider,
}
pub(super) struct ProviderMessagesRequest {
pub(super) provider: MessagesProvider,
pub(super) url: String,
pub(super) body: MessagesRequest,
pub(super) environment: ValidatedEnvironment,
pub(super) timeout: Option<Duration>,
/// The caller's own credential, reported to the host beside the wire request.
pub(super) api_key: Option<SecretValue>,
}
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
pub(super) async fn prepare(
call: MessagesCall,
secrets: &dyn SecretSource,
) -> Result<ProviderMessagesRequest, Error> {
let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
let secrets = secrets
.resolve(resolved.provider.config().secret_names())
.await?;
prepare_provider_request(call, resolved, secrets.as_ref())
}
fn resolve_provider(
model: &str,
custom_llm_provider: Option<&str>,
) -> Result<ResolvedProvider, Error> {
let resolved = resolve_llm_provider(model, custom_llm_provider, "messages")?;
let provider = messages_provider(resolved.provider, resolved.model)
.ok_or_else(|| Error::InvalidProvider(<&str>::from(resolved.provider).to_string()))?;
Ok(ResolvedProvider {
model: resolved.model.to_string(),
provider,
})
}
fn prepare_provider_request(
call: MessagesCall,
resolved: ResolvedProvider,
secrets: &dyn Lookup,
) -> Result<ProviderMessagesRequest, Error> {
let ResolvedProvider { model, provider } = resolved;
let MessagesCall {
body,
api_key,
api_base,
litellm_params,
extra_headers,
provider_specific_header,
timeout,
shaping,
..
} = call;
let connection_params = litellm_router_types::LitellmParams {
api_base: api_base.clone().or(litellm_params.api_base.clone()),
..litellm_params
};
let config = provider.config();
let env_lookup = |key: &str| secrets.get(key);
let sanitized = config.shape_request(
MessagesRequest { model, ..body },
shaping.settings.reasoning_auto_summary,
)?;
let trimmed =
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
let transformed = config.transform_anthropic_messages_request(
trimmed,
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
)?;
let scoped =
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
let forwarded = string_headers(Some(
extra_headers.into_iter().flatten().chain(scoped).collect(),
))?;
let forwarded = config.request_headers(
with_default_headers(forwarded, config.default_headers()),
&transformed,
);
let environment = config.validate_environment(
forwarded,
api_key.as_deref(),
&transformed.model,
&connection_params,
&env_lookup,
)?;
let url = config.get_complete_url(
api_base.as_deref(),
&transformed.model,
&connection_params,
transformed.params.stream == Some(true),
&env_lookup,
)?;
Ok(ProviderMessagesRequest {
provider,
url,
body: transformed,
environment,
timeout: timeout.or_else(|| shaping.settings.request_timeout()),
api_key: api_key.map(SecretValue::new),
})
}
fn without_additional_drop_params(
request: MessagesRequest,
paths: &[String],
) -> Result<MessagesRequest, Error> {
if paths.is_empty() {
return Ok(request);
}
let params = serde_json::to_value(request.params).map_err(invalid_request)?;
let trimmed = paths
.iter()
.fold(params, |params, path| delete_nested_value(params, path));
Ok(MessagesRequest {
params: serde_json::from_value(trimmed).map_err(invalid_request)?,
..request
})
}
#[cfg(test)]
mod tests {
use litellm_llms::base_llm::auth::resolve_auth;
use litellm_llms_types::headers::ProviderSpecificHeaders;
use rstest::{fixture, rstest};
use serde_json::{Map, Value, json};
use super::*;
use crate::{MessagesSettings, MessagesShaping};
#[fixture]
fn shaping() -> MessagesShaping {
MessagesShaping::default()
}
fn body(value: Value) -> MessagesRequest {
serde_json::from_value(value).unwrap()
}
fn prepare(call: MessagesCall) -> Result<ProviderMessagesRequest, Error> {
prepare_with_secrets(call, &|_: &str| None)
}
fn prepare_with_secrets(
call: MessagesCall,
secrets: &dyn Lookup,
) -> Result<ProviderMessagesRequest, Error> {
let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
prepare_provider_request(call, resolved, secrets)
}
/// The headers as they go on the wire, credential applied.
fn wire_headers(prepared: &ProviderMessagesRequest) -> Vec<(String, String)> {
tokio::runtime::Builder::new_current_thread()
.build()
.unwrap()
.block_on(resolve_auth(
&litellm_auth::AuthServices::default(),
prepared.environment.clone(),
&|_| None,
))
.unwrap()
.headers
}
#[rstest]
#[case::api_key(
&[("ANTHROPIC_API_KEY", "sk-secret")],
&[("x-api-key", "sk-secret")],
"https://api.anthropic.com/v1/messages"
)]
#[case::auth_token(
&[("ANTHROPIC_AUTH_TOKEN", "token")],
&[("authorization", "Bearer token")],
"https://api.anthropic.com/v1/messages"
)]
#[case::api_base(
&[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
&[("x-api-key", "sk-secret")],
"https://gateway.test/v1/messages"
)]
#[case::sdk_base_url(
&[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
&[("x-api-key", "sk-secret")],
"https://sdk.test/v1/messages"
)]
fn credentials_and_base_come_from_the_resolved_secrets(
shaping: MessagesShaping,
#[case] secrets: &[(&str, &str)],
#[case] expected_auth: &[(&str, &str)],
#[case] expected_url: &str,
) {
let lookup = |name: &str| {
secrets
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
};
let prepared = prepare_with_secrets(
MessagesCall {
body: body(
json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
),
api_key: None,
api_base: None,
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping,
},
&lookup,
)
.unwrap();
let headers = wire_headers(&prepared);
let auth: Vec<(&str, &str)> = headers
.iter()
.filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
.map(|(name, value)| (name.as_str(), value.as_str()))
.collect();
assert_eq!(
(auth.as_slice(), prepared.url.as_str()),
(expected_auth, expected_url)
);
}
#[rstest]
#[case::nothing_configured(None, None, None)]
#[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
#[case::call_timeout_wins(
Some(Duration::from_secs(30)),
Some(1800.0),
Some(Duration::from_secs(30))
)]
#[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
#[case::unusable_setting_ignored(None, Some(-1.0), None)]
#[case::zero_setting_ignored(None, Some(0.0), None)]
fn timeout_falls_back_to_the_configured_request_timeout(
shaping: MessagesShaping,
#[case] call_timeout: Option<Duration>,
#[case] request_timeout: Option<f64>,
#[case] expected: Option<Duration>,
) {
let prepared = prepare(MessagesCall {
body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
api_key: Some("sk-test".into()),
api_base: Some("https://anthropic.test".into()),
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: call_timeout,
shaping: MessagesShaping {
settings: MessagesSettings { request_timeout, ..shaping.settings },
..shaping
},
})
.unwrap();
assert_eq!(prepared.timeout, expected);
}
fn prepared_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
prepare(MessagesCall {
body: body(fields),
api_key: Some("sk-test".into()),
api_base: Some("https://anthropic.test".into()),
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping,
})
.map(|prepared| serde_json::to_value(prepared.body).unwrap())
}
#[rstest]
#[case::top_level_and_nested_paths(
json!({
"max_tokens": 1024,
"thinking": {"type": "enabled", "budget_tokens": 2048},
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
"metadata": {"user_id": "u1"},
"tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
}),
&["thinking", "context_management", "tools[*].input_examples"],
json!({
"max_tokens": 1024,
"metadata": {"user_id": "u1"},
"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
}),
)]
#[case::no_paths(
json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
&[],
json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
)]
#[case::model_and_messages_are_never_dropped(
json!({"max_tokens": 16}),
&["model", "messages", "messages[0].content"],
json!({"max_tokens": 16}),
)]
fn prepared_body_drops_configured_paths(
shaping: MessagesShaping,
#[case] fields: Value,
#[case] additional_drop_params: &[&str],
#[case] expected_fields: Value,
) {
let with_messages = |fields: Value| -> Value {
let Value::Object(fields) = fields else {
unreachable!()
};
Value::Object(
[
("model".to_string(), json!("claude-test")),
(
"messages".to_string(),
json!([{"role": "user", "content": "hi"}]),
),
]
.into_iter()
.chain(fields)
.collect(),
)
};
let shaping = MessagesShaping {
settings: MessagesSettings {
additional_drop_params: additional_drop_params
.iter()
.map(ToString::to_string)
.collect(),
..shaping.settings
},
..shaping
};
assert_eq!(
prepared_body(with_messages(fields), shaping),
Ok(with_messages(expected_fields))
);
}
#[rstest]
#[case::model_prefix_picks_the_provider(
"azure_ai/claude-test",
None,
&[("x-priority", "extra"), ("x-scoped", "azure_ai")]
)]
#[case::explicit_provider(
"claude-test",
Some("anthropic"),
&[("x-priority", "scoped"), ("x-scoped", "anthropic")]
)]
#[case::provider_prefix_on_an_anthropic_model(
"anthropic/claude-test",
None,
&[("x-priority", "scoped"), ("x-scoped", "anthropic")]
)]
fn provider_specific_headers_follow_the_resolved_provider(
shaping: MessagesShaping,
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
#[case] expected: &[(&str, &str)],
) {
let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
{"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
{"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
]))
.unwrap();
let prepared = prepare(MessagesCall {
body: body(
json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
),
api_key: Some("sk-test".into()),
api_base: Some("https://resource.services.ai.azure.com".into()),
custom_llm_provider: custom_llm_provider.map(Into::into),
litellm_params: Default::default(),
extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
provider_specific_header: Some(configured),
timeout: None,
shaping,
})
.unwrap();
let caller_headers: Vec<(&str, &str)> = prepared
.environment
.headers
.iter()
.filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
.map(|(name, value)| (name.as_str(), value.as_str()))
.collect();
assert_eq!(caller_headers, expected);
}
#[rstest]
fn prepared_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
assert_eq!(
prepared_body(
json!({
"model": "anthropic/claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16
}),
shaping,
),
Ok(json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16
}))
);
}
#[rstest]
fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
let shaping = MessagesShaping {
settings: MessagesSettings {
reasoning_auto_summary: true,
additional_drop_params: vec!["thinking.display".to_string()],
..shaping.settings
},
..shaping
};
assert_eq!(
prepared_body(
json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 4096,
"thinking": {"type": "enabled", "budget_tokens": 2048}
}),
shaping,
),
Ok(json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 4096,
"thinking": {"type": "enabled", "budget_tokens": 2048}
}))
);
}
#[rstest]
fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) {
let shaping = MessagesShaping {
settings: MessagesSettings {
additional_drop_params: vec!["metadata.user_id".to_string()],
..shaping.settings
},
..shaping
};
assert!(matches!(
prepared_body(
json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16,
"metadata": {"user_id": 123}
}),
shaping,
),
Err(Error::InvalidRequest(_))
));
}
#[rstest]
fn prepared_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
assert_eq!(
prepared_body(
json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16,
"metadata": {"user_id": 123}
}),
shaping,
),
Err(Error::InvalidRequest(
litellm_llms::ErrorDetail::InvalidValue {
field: "metadata.user_id",
expected: "a string",
actual: json!(123),
}
))
);
}
#[rstest]
#[case::mantle(
"bedrock_mantle/claude-test",
"https://bedrock-mantle.us-west-2.api.aws",
"/anthropic/v1/messages",
"bedrock"
)]
#[case::platform(
"bedrock/claude_platform/claude-test",
"https://aws-external-anthropic.us-west-2.api.aws",
"/v1/messages",
"aws-external-anthropic"
)]
fn explicit_endpoint_region_is_used_by_the_signer(
shaping: MessagesShaping,
#[case] model: &str,
#[case] base: &str,
#[case] path: &str,
#[case] service: &str,
) {
let prepared = prepare_with_secrets(MessagesCall {
body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
api_key: None,
api_base: Some(base.into()),
custom_llm_provider: None,
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping,
}, &|name: &str| match name {
"AWS_REGION_NAME" => Some("us-east-1".into()),
"ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
_ => None,
}).unwrap();
assert_eq!(prepared.url, format!("{base}{path}"));
assert!(
matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service)
);
}
}

View file

@ -44,8 +44,8 @@ pub fn consumed_optional_param_names(
model: &str,
custom_llm_provider: Option<&str>,
) -> Result<Vec<&'static str>, Error> {
let (model, config) = resolve_provider_config(model, custom_llm_provider)?;
let provider_fields = config.get_supported_ocr_params(&model);
let (provider, config) = resolve_provider_config(model, custom_llm_provider)?;
let provider_fields = config.get_supported_ocr_params(provider.model);
let auth_fields: &[&str] = match config {
OcrConfigKind::AwsTextract | OcrConfigKind::AwsTextractAnalyze => AWS_AUTH_OPTION_FIELDS,
OcrConfigKind::AzureAi

View file

@ -1,81 +0,0 @@
use litellm_host::observation::ObservationSender;
use std::sync::Arc;
use litellm_host::interceptors::Interceptors;
use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
use super::{
handler::perform_ocr_request,
types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest},
};
#[derive(Clone)]
pub struct OcrRoute {
client: OcrClient,
}
impl OcrRoute {
pub fn new(client: OcrClient) -> Self {
Self { client }
}
pub async fn execute(
&self,
request: LiteLLMOcrRequest,
interceptors: &impl Interceptors<Error>,
observers: Option<ObservationSender>,
) -> Result<LiteLLMOcrResponse, Error> {
litellm_host::lifecycle::observe_unary(
observers.clone(),
self.run(request, interceptors, observers.as_ref()),
)
.await
}
#[tracing::instrument(name = "litellm.route", skip_all, fields(
route = "ocr",
model = %request.model,
resolved_model = %request.model,
provider = <&str>::from(request.config.provider()),
stream = false,
outcome
))]
pub(super) async fn run(
&self,
request: LiteLLMOcrRequest,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<LiteLLMOcrResponse, Error> {
litellm_inference::diagnostic::unary(async {
let caller_document = matches!(&request.document, OcrDocumentInput::Document(_));
let prepared = prepare_request_document(request).await?;
let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
Box::pin(perform_ocr_request(
&self.client,
prepared,
interceptors,
caller_document,
observers,
));
execute.await
})
.await
}
}
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
async fn prepare_request_document(
request: LiteLLMOcrRequest<OcrDocumentInput>,
) -> Result<ResolvedOcrRequest, Error> {
if let OcrDocumentInput::Document(_) = &request.document {
return request.map_document(super::document::prepare_document);
}
let logger = litellm_tracing::Logger::current();
let span = tracing::Span::current();
tokio::task::spawn_blocking(move || {
logger.scope(|| span.in_scope(|| request.map_document(super::document::prepare_document)))
})
.await
.map_err(|error| Error::DocumentTask(Arc::new(error)))?
}

View file

@ -1,10 +1,10 @@
use std::{collections::BTreeMap as Map, io::Read, path::Path};
use std::{collections::BTreeMap as Map, io::Read, path::Path, sync::Arc};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_llms::base_llm::ocr::{error::Error, transformation::OCR_INLINE_MAX_BYTES};
use litellm_llms_types::formats::ocr::OcrDocument;
use crate::types::OcrDocumentInput;
use crate::types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest};
pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
match input {
@ -24,6 +24,21 @@ pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
}
}
pub(crate) async fn resolve_document(
request: LiteLLMOcrRequest<OcrDocumentInput>,
) -> Result<ResolvedOcrRequest, Error> {
if let OcrDocumentInput::Document(_) = &request.document {
return request.map_document(prepare_document);
}
let logger = litellm_tracing::Logger::current();
let span = tracing::Span::current();
tokio::task::spawn_blocking(move || {
logger.scope(|| span.in_scope(|| request.map_document(prepare_document)))
})
.await
.map_err(|error| Error::DocumentTask(Arc::new(error)))?
}
pub fn read_path_document(path: &Path, mime_type: Option<&str>) -> Result<OcrDocument, Error> {
let mut bytes = Vec::new();
std::fs::File::open(path)

View file

@ -4,29 +4,45 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender};
use litellm_llms::base_llm::ocr::{
error::Error,
handler::{CallHooks, OcrClient},
transformation::PreparedOcrRequest,
transformation::{OcrConnection, PreparedOcrRequest},
};
use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
use litellm_secrets::source::Secrets;
use serde_json::Value;
use super::{arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind};
use crate::types::ResolvedOcrRequest;
use super::{arguments::is_secret_param, provider_config::OcrConfigKind};
use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
pub(crate) async fn perform_ocr_request(
pub(crate) async fn execute(
client: &OcrClient,
request: ResolvedOcrRequest,
config: OcrConfigKind,
call: ResolvedOcrRequest,
secrets: Secrets,
host: &impl Interceptors<Error>,
caller_document: bool,
observers: Option<&ObservationSender>,
) -> Result<LiteLLMOcrResponse, Error> {
request.response_format()?;
let config = request.config;
let secrets = client
.secret_source()
.resolve(&config.secret_names())
.await
.map_err(|error| Error::Secret(std::sync::Arc::new(error)))?;
let request = prepare_request(request, caller_document, client, secrets);
call.response_format()?;
let LiteLLMOcrRequest {
model,
document,
credentials,
transport,
optional_params,
input_sources,
azure_ad_token_provider,
..
} = call;
let resolved = config.resolve_credentials(credentials, &secrets);
let request = PreparedOcrRequest {
model,
document,
connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
caller_document,
optional_params,
input_sources,
azure_ad_token_provider,
};
let interceptors = OcrCallHooks::new(host, &request, config, observers);
config.ocr(client, &request, &interceptors).await
}
@ -86,3 +102,257 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
Box::pin(self.interceptors.after_provider_response(raw))
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
use litellm_llms_types::formats::ocr::OcrResponseFormat;
use rstest::rstest;
use serde_json::{Value, json};
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
use super::*;
use crate::{
OcrRoute,
types::LiteLLMOcrRequest,
wire::{OcrWireRequest, decode_request},
};
fn client() -> OcrClient {
OcrClient::for_test(
litellm_http::Client::plain_for_test(),
litellm_http::Client::no_redirect_for_test(),
)
.with_secrets(litellm_inference_testing::no_secrets())
}
fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
decode_request(OcrWireRequest {
model: model.into(),
document,
api_key: Some(litellm_auth::SecretValue::new("test-key")),
api_base: Some(base.into()),
custom_llm_provider: None,
extra_headers: None,
optional_params: options.as_object().unwrap().clone(),
input_sources: Default::default(),
timeout_seconds: Some(2.0),
})
.unwrap()
}
async fn run_request(request: LiteLLMOcrRequest) -> Result<LiteLLMOcrResponse, Error> {
OcrRoute::new(client()).execute(request, &(), None).await
}
async fn upstream(request_count: u64, response: Value) -> MockServer {
let server = MockServer::start().await;
Mock::given(any())
.respond_with(ResponseTemplate::new(200).set_body_json(response))
.up_to_n_times(request_count)
.mount(&server)
.await;
server
}
fn image(url: &str) -> Value {
json!({"type": "image_url", "image_url": url})
}
fn body(request: &wiremock::Request) -> Value {
serde_json::from_slice(&request.body).unwrap()
}
fn assert_mistral_request(request: &wiremock::Request) {
assert_eq!(
request
.headers
.get("authorization")
.unwrap()
.to_str()
.unwrap(),
"Bearer test-key"
);
assert_eq!(
request
.headers
.get("content-type")
.unwrap()
.to_str()
.unwrap(),
"application/json"
);
assert_eq!(
body(request),
json!({
"model": "mistral-ocr-maas",
"document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
"pages": [0, 2],
"include_image_base64": true,
"unknown": "preserved"
})
);
}
#[rstest]
#[tokio::test]
async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
let upstream = upstream(
1,
json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
)
.await;
let request = request(
"cohere/parse",
&upstream.uri(),
image("https://example.com/original.png"),
json!({
"output_format": "markdown", "timeout": 30,
"extra_body": {
"output_format": {"future": true},
"document": {"type": "image_url", "image_url": "https://example.com/a.png",
"provider_options": {"nested": [false, 0, null]}}
}
}),
);
run_request(request).await.unwrap();
let [request] = upstream
.received_requests()
.await
.unwrap()
.try_into()
.unwrap();
assert_eq!(
body(&request),
json!({
"model": "parse", "output_format": {"future": true},
"document": {"type": "image_url", "image_url": "https://example.com/a.png",
"provider_options": {"nested": [false, 0, null]}}
})
);
}
#[rstest]
#[tokio::test]
async fn explicit_null_options_use_defaults_before_http() {
let upstream = upstream(
1,
json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
)
.await;
let request = request(
"cohere/parse",
&upstream.uri(),
image("https://example.com/a.png"),
json!({"output_format": null, "req_format": null}),
);
assert_eq!(
request.response_format().unwrap(),
OcrResponseFormat::Litellm
);
run_request(request).await.unwrap();
let [request] = upstream
.received_requests()
.await
.unwrap()
.try_into()
.unwrap();
let body = body(&request);
assert_eq!(body["output_format"], "markdown");
assert!(body.get("req_format").is_none());
}
#[rstest]
#[tokio::test]
async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
let upstream = upstream(
2,
json!({
"pages": [{"index": 0, "markdown": "hello"}],
"extra": "preserved"
}),
)
.await;
let options = json!({
"pages": [0, 2],
"include_image_base64": true,
"vertex_project": "project-1",
"vertex_location": "us-central1",
"unknown": "preserved"
});
let document =
json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
let direct_call = request(
"mistral/mistral-ocr-maas",
&upstream.uri(),
document.clone(),
options.clone(),
);
let vertex_call = request(
"vertex_ai/mistral-ocr-maas",
&upstream.uri(),
document,
options,
);
assert_eq!(direct_call.transport.timeout, Some(Duration::from_secs(2)));
assert_eq!(vertex_call.transport.timeout, Some(Duration::from_secs(2)));
let direct_response = run_request(direct_call).await.unwrap().into_json();
let vertex_response = run_request(vertex_call).await.unwrap().into_json();
let [direct_http, vertex_http] = upstream
.received_requests()
.await
.unwrap()
.try_into()
.unwrap();
assert_eq!(direct_http.url.path(), "/v1/ocr");
assert_eq!(
vertex_http.url.path(),
"/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
);
assert_mistral_request(&direct_http);
assert_mistral_request(&vertex_http);
assert_eq!(direct_response, vertex_response);
assert_eq!(direct_response["model"], "mistral-ocr-maas");
assert_eq!(direct_response["object"], "ocr");
assert_eq!(direct_response["extra"], "preserved");
}
#[derive(serde::Deserialize)]
struct KnownParams {
pages: Option<Vec<i64>>,
}
#[rstest]
fn parsed_provider_params_separates_known_and_extra_params() {
let arguments: CallArguments = serde_json::from_value(json!({
"pages": [0, 2],
"future_ocr_option": true,
"extra_body": {"provider_option": "value"}
}))
.unwrap();
let known: KnownParams = parse_options(&arguments).unwrap();
assert_eq!(known.pages, Some(vec![0, 2]));
assert_eq!(arguments["future_ocr_option"], true);
assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
assert_eq!(
arguments
.iter()
.filter(|(name, _)| name.as_str() != "pages")
.count(),
2
);
assert_eq!(
compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
json!({
"pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
})
);
}
}

View file

@ -1,10 +1,82 @@
pub mod arguments;
mod client;
pub use client::OcrRoute;
pub mod document;
pub(crate) mod handler;
pub(crate) mod prepare;
pub mod provider_config;
pub mod route;
pub mod types;
pub mod wire;
use std::sync::Arc;
use litellm_host::interceptors::Interceptors;
use litellm_host::observation::ObservationSender;
use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
use crate::{
handler::execute,
types::{LiteLLMOcrRequest, OcrDocumentInput},
};
#[derive(Clone)]
pub struct OcrRoute {
client: OcrClient,
}
impl OcrRoute {
pub fn new(client: OcrClient) -> Self {
Self { client }
}
pub async fn execute(
&self,
request: LiteLLMOcrRequest,
interceptors: &impl Interceptors<Error>,
observers: Option<ObservationSender>,
) -> Result<LiteLLMOcrResponse, Error> {
litellm_host::lifecycle::observe_unary(
observers.clone(),
self.run(request, interceptors, observers.as_ref()),
)
.await
}
#[tracing::instrument(name = "litellm.route", skip_all, fields(
route = "ocr",
model = %call.model,
resolved_model = %call.model,
provider = <&str>::from(call.config.provider()),
stream = false,
outcome
))]
async fn run(
&self,
call: LiteLLMOcrRequest,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<LiteLLMOcrResponse, Error> {
litellm_inference::diagnostic::unary(async {
let caller_document = matches!(&call.document, OcrDocumentInput::Document(_));
let call = document::resolve_document(call).await?;
let config = call.config;
let secrets = self
.client
.secret_source()
.resolve(&config.secret_names())
.await
.map_err(|error| Error::Secret(Arc::new(error)))?;
let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
Box::pin(execute(
&self.client,
config,
call,
secrets,
interceptors,
caller_document,
observers,
));
execute.await
})
.await
}
}

View file

@ -1,309 +0,0 @@
use litellm_auth::{InputSource, SecretValue, Sourced};
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
use litellm_llms::base_llm::ocr::{
handler::OcrClient,
transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
};
use litellm_secrets::source::Secrets;
use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
pub(crate) fn prepare_request(
request: ResolvedOcrRequest,
caller_document: bool,
client: &OcrClient,
secrets: Secrets,
) -> PreparedOcrRequest {
let credentials = request.credentials.clone();
let (preferred_api_key_env, api_base_env) = match request.config.provider() {
LlmProviders::Mistral => (
Some("MISTRAL_AZURE_API_KEY"),
Some("MISTRAL_AZURE_API_BASE"),
),
LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
_ => (None, None),
};
let secret = |name: &str| secrets.truthy(name);
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
credentials.api_key.clone().or_else(|| {
preferred_api_key_env
.into_iter()
.chain(request.config.get_api_key_env_var())
.find_map(secret)
.map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
})
});
let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
credentials.api_base.clone().or_else(|| {
api_base_env
.and_then(secret)
.map(|value| Sourced::new(value, InputSource::Environment))
})
});
let resolved = request
.config
.resolve_connection_params(OcrCredentialInputs {
dynamic_api_key,
dynamic_api_base,
..credentials
});
let LiteLLMOcrRequest {
model,
document,
transport,
optional_params,
input_sources,
azure_ad_token_provider,
..
} = request;
PreparedOcrRequest {
model,
document,
connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
caller_document,
optional_params,
input_sources,
azure_ad_token_provider,
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use futures_util::future::BoxFuture;
use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
use litellm_host::interceptors::WireRequest;
use litellm_llms::{
base_llm::ocr::{
error::Error,
handler::{CallHooks, OcrClient},
transformation::BaseOcrConfig,
},
cohere::ocr::transformation::CohereParseConfig,
mistral::ocr::transformation::MistralOcrConfig,
vertex_ai::ocr::transformation::VertexAiOcrConfig,
};
use litellm_llms_types::formats::ocr::OcrResponseFormat;
use serde_json::{Value, json};
use super::*;
use crate::{
document::prepare_document,
types::LiteLLMOcrRequest,
wire::{OcrWireRequest, decode_request},
};
/// Stands in for a host with no interceptors registered.
struct NoHooks;
impl CallHooks<Error> for NoHooks {
fn before_provider_request(
&self,
wire: WireRequest,
) -> BoxFuture<'_, Result<WireRequest, Error>> {
Box::pin(async move { Ok(wire) })
}
fn response_received<'a>(&'a self, _body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
Box::pin(async { Ok(()) })
}
}
fn client() -> OcrClient {
OcrClient::for_test(
litellm_http::Client::plain_for_test(),
litellm_http::Client::no_redirect_for_test(),
)
}
fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
decode_request(OcrWireRequest {
model: model.into(),
document,
api_key: Some(litellm_auth::SecretValue::new("test-key")),
api_base: Some(base.into()),
custom_llm_provider: None,
extra_headers: None,
optional_params: options.as_object().unwrap().clone(),
input_sources: Default::default(),
timeout_seconds: Some(2.0),
})
.unwrap()
}
fn prepared(request: LiteLLMOcrRequest) -> PreparedOcrRequest {
prepare_request(
request.map_document(prepare_document).unwrap(),
true,
&client(),
std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
)
}
fn image(url: &str) -> Value {
json!({"type": "image_url", "image_url": url})
}
#[rstest::rstest]
#[tokio::test]
async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
let request = request(
"cohere/parse",
"https://example.com",
image("https://example.com/original.png"),
json!({
"output_format": "markdown", "timeout": 30,
"extra_body": {
"output_format": {"future": true},
"document": {"type": "image_url", "image_url": "https://example.com/a.png",
"provider_options": {"nested": [false, 0, null]}}
}
}),
);
let http = CohereParseConfig
.prepare_request(&prepared(request), &client(), &NoHooks)
.await
.unwrap();
let body: Value = serde_json::from_slice(http.body()).unwrap();
assert_eq!(
body,
json!({
"model": "parse", "output_format": {"future": true},
"document": {"type": "image_url", "image_url": "https://example.com/a.png",
"provider_options": {"nested": [false, 0, null]}}
})
);
}
#[rstest::rstest]
#[tokio::test]
async fn explicit_null_options_use_defaults_before_http() {
let request = request(
"cohere/parse",
"https://example.com",
image("https://example.com/a.png"),
json!({"output_format": null, "req_format": null}),
);
assert_eq!(
request.response_format().unwrap(),
OcrResponseFormat::Litellm
);
let http = CohereParseConfig
.prepare_request(&prepared(request), &client(), &NoHooks)
.await
.unwrap();
let body: Value = serde_json::from_slice(http.body()).unwrap();
assert_eq!(body["output_format"], "markdown");
assert!(body.get("req_format").is_none());
}
#[rstest::rstest]
#[tokio::test]
async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
let options = json!({
"pages": [0, 2],
"include_image_base64": true,
"vertex_project": "project-1",
"vertex_location": "us-central1",
"unknown": "preserved"
});
let document =
json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
let direct = prepared(request(
"mistral/mistral-ocr-maas",
"https://mistral.test",
document.clone(),
options.clone(),
));
let vertex = prepared(request(
"vertex_ai/mistral-ocr-maas",
"https://vertex.test",
document,
options,
));
let direct_http = MistralOcrConfig
.prepare_request(&direct, &client(), &NoHooks)
.await
.unwrap();
let vertex_http = VertexAiOcrConfig
.prepare_request(&vertex, &client(), &NoHooks)
.await
.unwrap();
assert_eq!(direct_http.url(), "https://mistral.test/v1/ocr");
assert_eq!(
vertex_http.url(),
"https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
);
for http in [&direct_http, &vertex_http] {
assert_eq!(http.header("authorization").unwrap(), "Bearer test-key");
assert_eq!(http.header("content-type").unwrap(), "application/json");
assert_eq!(http.timeout(), Some(Duration::from_secs(2)));
let body: Value = serde_json::from_slice(http.body()).unwrap();
assert_eq!(
body,
json!({
"model": "mistral-ocr-maas",
"document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
"pages": [0, 2],
"include_image_base64": true,
"unknown": "preserved"
})
);
}
let payload = serde_json::to_vec(
&json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}),
)
.unwrap();
let direct_response = MistralOcrConfig
.transform_ocr_response(&direct.model, &payload, OcrResponseFormat::Litellm)
.unwrap()
.into_json();
let vertex_response = VertexAiOcrConfig
.transform_ocr_response(&vertex.model, &payload, OcrResponseFormat::Litellm)
.unwrap()
.into_json();
assert_eq!(direct_response, vertex_response);
assert_eq!(direct_response["model"], "mistral-ocr-maas");
assert_eq!(direct_response["object"], "ocr");
assert_eq!(direct_response["extra"], "preserved");
}
#[derive(serde::Deserialize)]
struct KnownParams {
pages: Option<Vec<i64>>,
}
#[rstest::rstest]
fn parsed_provider_params_separates_known_and_extra_params() {
let arguments: CallArguments = serde_json::from_value(json!({
"pages": [0, 2],
"future_ocr_option": true,
"extra_body": {"provider_option": "value"}
}))
.unwrap();
let known: KnownParams = parse_options(&arguments).unwrap();
assert_eq!(known.pages, Some(vec![0, 2]));
assert_eq!(arguments["future_ocr_option"], true);
assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
assert_eq!(
arguments
.iter()
.filter(|(name, _)| name.as_str() != "pages")
.count(),
2
);
assert_eq!(
compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
json!({
"pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
})
);
}
}

View file

@ -1,5 +1,6 @@
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
use litellm_auth::{InputSource, SecretValue, Sourced};
use litellm_core_utils::get_llm_provider_logic::{LlmProviders, get_custom_llm_provider};
use litellm_inference::provider::{ResolvedProvider, resolve_llm_provider};
use litellm_llms::{
aws_textract::ocr::{
analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation,
@ -25,6 +26,7 @@ use litellm_llms::{
},
};
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat};
use litellm_secrets::source::Secrets;
macro_rules! with_config {
($kind:expr, $config:ident => $body:expr) => {
@ -115,7 +117,18 @@ impl OcrConfigKind {
}
pub(crate) fn secret_names(self) -> Vec<&'static str> {
with_config!(self, config => config.secret_names())
let names = with_config!(self, config => config.secret_names());
let fallback_names: &[&str] = match self.provider() {
LlmProviders::Mistral => &["MISTRAL_AZURE_API_KEY", "MISTRAL_AZURE_API_BASE"],
LlmProviders::AzureAi => &["AZURE_AI_API_BASE"],
_ => &[],
};
let additional_names = fallback_names
.iter()
.copied()
.filter(|name| !names.contains(name))
.collect::<Vec<_>>();
names.into_iter().chain(additional_names).collect()
}
pub(crate) fn get_health_check_document(self) -> OcrDocument {
@ -129,6 +142,43 @@ impl OcrConfigKind {
with_config!(self, config => config.resolve_connection_params(inputs))
}
pub(crate) fn resolve_credentials(
self,
credentials: OcrCredentialInputs,
secrets: &Secrets,
) -> ResolvedOcrCredentials {
let (preferred_api_key_env, api_base_env) = match self.provider() {
LlmProviders::Mistral => (
Some("MISTRAL_AZURE_API_KEY"),
Some("MISTRAL_AZURE_API_BASE"),
),
LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
_ => (None, None),
};
let secret = |name: &str| secrets.truthy(name);
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
credentials.api_key.clone().or_else(|| {
preferred_api_key_env
.into_iter()
.chain(self.get_api_key_env_var())
.find_map(secret)
.map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
})
});
let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
credentials.api_base.clone().or_else(|| {
api_base_env
.and_then(secret)
.map(|value| Sourced::new(value, InputSource::Environment))
})
});
self.resolve_connection_params(OcrCredentialInputs {
dynamic_api_key,
dynamic_api_base,
..credentials
})
}
pub(crate) async fn ocr(
self,
client: &OcrClient,
@ -164,7 +214,7 @@ pub fn passthrough_response(
endpoint: &str,
body: &[u8],
) -> Result<Option<LiteLLMOcrResponse>, Error> {
let (model, config) = resolve_provider_config(model, Some("azure_ai"))?;
let (provider, config) = resolve_provider_config(model, Some("azure_ai"))?;
let segments: Vec<&str> = endpoint
.split('/')
.filter(|segment| !segment.is_empty())
@ -173,7 +223,7 @@ pub fn passthrough_response(
OcrConfigKind::AzureAi => segments == AZURE_AI_OCR_PATH,
OcrConfigKind::AzureCohere => segments == AZURE_COHERE_PARSE_PATH,
OcrConfigKind::AzureDocumentIntelligence => {
segments == AzureDocumentIntelligenceOcrConfig::analyze_path(&model)?
segments == AzureDocumentIntelligenceOcrConfig::analyze_path(provider.model)?
}
other => {
let provider: &'static str = other.provider().into();
@ -183,24 +233,28 @@ pub fn passthrough_response(
if !is_ocr_endpoint {
return Ok(None);
}
with_config!(config, config => config.transform_ocr_response(&model, body, OcrResponseFormat::Litellm))
with_config!(config, config => config.transform_ocr_response(provider.model, body, OcrResponseFormat::Litellm))
.map(Some)
}
pub(crate) fn resolve_provider_config(
model: &str,
custom_llm_provider: Option<&str>,
) -> Result<(String, OcrConfigKind), Error> {
let provider =
get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider {
model,
custom_llm_provider: LlmProviders::Mistral.into(),
});
let llm_provider = provider
.custom_llm_provider
.parse::<LlmProviders>()
.map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?;
let config = match llm_provider {
pub(crate) fn resolve_provider_config<'a>(
model: &'a str,
custom_llm_provider: Option<&'a str>,
) -> Result<(ResolvedProvider<'a>, OcrConfigKind), Error> {
let provider = match resolve_llm_provider(model, custom_llm_provider, "ocr") {
Ok(provider) => provider,
Err(_) if get_custom_llm_provider(model, custom_llm_provider).is_none() => {
ResolvedProvider {
model,
provider: LlmProviders::Mistral,
}
}
Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
return Err(Error::InvalidProvider(provider));
}
Err(error) => return Err(Error::InvalidProvider(error.to_string())),
};
let config = match provider.provider {
LlmProviders::AwsTextract => match TextractOperation::from_model(provider.model)? {
TextractOperation::DetectDocumentText => OcrConfigKind::AwsTextract,
TextractOperation::AnalyzeDocument => OcrConfigKind::AwsTextractAnalyze,
@ -227,11 +281,11 @@ pub(crate) fn resolve_provider_config(
LlmProviders::VertexAi => OcrConfigKind::VertexAi,
_ => {
return Err(Error::InvalidProvider(
provider.custom_llm_provider.to_string(),
<&'static str>::from(provider.provider).to_string(),
));
}
};
Ok((provider.model.to_string(), config))
Ok((provider, config))
}
fn is_document_intelligence_model(model: &str) -> bool {
@ -243,7 +297,7 @@ fn is_document_intelligence_model(model: &str) -> bool {
mod tests {
use std::collections::HashSet;
use litellm_auth::{InputSource, Sourced};
use litellm_auth::Sourced;
use litellm_llms::{
base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document,
};
@ -484,8 +538,8 @@ mod tests {
#[case] expected_config: OcrConfigKind,
) {
let expected_model = qualified_model.split_once('/').unwrap().1;
let (model, config) = resolve_provider_config(qualified_model, None).unwrap();
assert_eq!(model, expected_model);
let (provider, config) = resolve_provider_config(qualified_model, None).unwrap();
assert_eq!(provider.model, expected_model);
assert_eq!(config, expected_config);
}
@ -528,7 +582,7 @@ mod tests {
expected_config
);
assert_eq!(
resolve_provider_config(model, None).unwrap().0,
resolve_provider_config(model, None).unwrap().0.model,
model.split_once('/').unwrap().1
);
}

View file

@ -75,6 +75,8 @@ impl OcrConnectionInputs {
pub struct LiteLLMOcrRequest<D = OcrDocumentInput> {
pub model: String,
pub(crate) requested_model: String,
pub(crate) custom_llm_provider: Option<String>,
pub document: D,
pub credentials: OcrCredentialInputs,
pub transport: OcrTransportConfig,
@ -91,7 +93,8 @@ impl LiteLLMOcrRequest {
custom_llm_provider: Option<&str>,
optional_params: CallArguments,
) -> Result<Self, Error> {
let (model, config) = resolve_provider_config(&model, custom_llm_provider)?;
let (provider, config) = resolve_provider_config(&model, custom_llm_provider)?;
let resolved_model = provider.model.to_owned();
let default_transport = OcrTransportConfig::default();
let max_response_bytes = optional_params
.get("max_response_bytes")
@ -116,7 +119,9 @@ impl LiteLLMOcrRequest {
.collect();
Ok(Self {
model,
model: resolved_model,
requested_model: model,
custom_llm_provider: custom_llm_provider.map(str::to_owned),
document: document.into(),
credentials: OcrCredentialInputs::default(),
transport,
@ -135,6 +140,8 @@ impl<D> LiteLLMOcrRequest<D> {
) -> Result<LiteLLMOcrRequest<T>, E> {
Ok(LiteLLMOcrRequest {
model: self.model,
requested_model: self.requested_model,
custom_llm_provider: self.custom_llm_provider,
document: map(self.document)?,
credentials: self.credentials,
transport: self.transport,
@ -148,6 +155,8 @@ impl<D> LiteLLMOcrRequest<D> {
pub fn with_document<T>(self, document: T) -> LiteLLMOcrRequest<T> {
LiteLLMOcrRequest {
model: self.model,
requested_model: self.requested_model,
custom_llm_provider: self.custom_llm_provider,
document,
credentials: self.credentials,
transport: self.transport,

View file

@ -66,6 +66,19 @@ async fn interception_runs_once_and_observers_receive_ordered_success_events() {
assert_eq!(received(&upstream).await.len(), 1);
}
#[rstest::rstest]
#[case::mutated_model("model-b")]
#[tokio::test]
async fn provider_request_uses_the_request_model_after_it_is_mutated(#[case] model: &str) {
let upstream = upstream([pages_response()]).await;
let mut request = ocr_request("mistral/model-a", &upstream.uri(), json!({}));
request.model = model.into();
perform(request).await.unwrap();
assert_eq!(only_request(&upstream).await.json()["model"], json!(model));
}
#[rstest::rstest]
#[tokio::test]
async fn a_blocking_before_send_prevents_the_call_and_emits_one_failure() {

View file

@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender;
use std::time::Duration;
use futures_util::StreamExt;
use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest};
use litellm_llms::base_llm::auth::{Authenticated, resolve_auth};
use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth};
use litellm_secrets::source::Secrets;
use super::{
Error,
types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead},
Error, ResponsesRoute,
types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead},
};
pub(super) async fn execute(
http: &litellm_http::Client,
auth: &litellm_auth::AuthServices,
request: ProviderResponsesRequest,
cache: Option<litellm_cache_response::ScopedCache>,
route: &ResponsesRoute,
ResponsesProvider { identity, config }: ResponsesProvider,
call: ResponsesCall,
secrets: Secrets,
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ResponsesOutput, Error> {
let authenticated = resolve_auth(auth, request.environment, &|_| None).await?;
let identity = litellm_host::interceptors::ProviderIdentity {
model: request.context.model.clone(),
provider: request.context.custom_llm_provider.clone(),
let lookup = |name: &str| secrets.get(name);
let environment = config.validate_environment(
litellm_http::request::string_headers("responses", call.extra_headers)?,
call.api_key.as_deref(),
&lookup,
)?;
let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
let context = RequestContext {
model: identity.model.clone(),
custom_llm_provider: identity.provider.clone(),
optional_params: serde_json::Value::Object(call.optional_params.clone()),
secret_fields: Vec::new(),
api_key: match &environment.auth {
AuthScheme::Credential { secret, .. } => Some(secret.clone()),
_ => None,
},
};
let body = config.transform_responses_api_request(
&identity.model,
call.input,
call.optional_params,
)?;
let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?;
let wire = interceptors
.before_provider_request(
WireRequest {
url: request.url,
url,
headers: authenticated.headers,
body: request.body,
body,
},
request.context,
context,
)
.await?;
let cache = cache.filter(|_| authenticated.signer.is_none());
let cache = route
.cache
.clone()
.filter(|_| authenticated.signer.is_none());
let cache_request = litellm_inference::caching::CacheRequest::from_wire(
identity,
cache.as_ref().map(|_| &wire),
@ -59,9 +81,9 @@ pub(super) async fn execute(
},
wire.url,
&wire.body,
Some(request.timeout.unwrap_or(Duration::from_secs(600))),
Some(call.timeout.unwrap_or(Duration::from_secs(600))),
)?;
let response = litellm_inference::outbound::send(outbound, http)
let response = litellm_inference::outbound::send(outbound, &route.http)
.await
.map_err(network)?;
let status = response.status().as_u16();
@ -103,8 +125,7 @@ pub(super) async fn execute(
.map_err(Error::post_call)?;
let value = serde_json::from_str(&body)
.map_err(|error| Error::InvalidResponse(error.to_string().into()))?;
request
.config
config
.transform_response_api_response(value)
.map(ResponsesOutput::Complete)
.map_err(Error::from)

View file

@ -7,14 +7,30 @@ pub mod types;
pub mod websocket;
mod handler;
mod prepare;
use std::sync::Arc;
use litellm_auth::AuthServices;
use litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
};
use litellm_host::interceptors::Interceptors;
use litellm_host::interceptors::ProviderIdentity;
use litellm_llms::{
base_llm::responses::transformation::BaseResponsesApiConfig,
openai::responses::transformation::OpenAiResponsesApiConfig,
openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
};
use litellm_secrets::source::SecretSource;
use types::{ResponsesCall, ResponsesOutput};
use types::{ResponsesCall, ResponsesOutput, ResponsesProvider};
fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> {
match provider {
LlmProviders::Openai => Some(&OpenAiResponsesApiConfig),
LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG),
_ => None,
}
}
#[derive(Clone)]
pub struct ResponsesRoute {
@ -78,34 +94,68 @@ impl ResponsesRoute {
observers: Option<&ObservationSender>,
) -> Result<ResponsesOutput, Error> {
litellm_inference::diagnostic::call(async {
self.run_provider(call, cache_options, interceptors, observers)
.await
let explicit = call
.custom_llm_provider
.as_deref()
.filter(|provider| !provider.is_empty());
let (provider_name, resolved_model) =
match get_custom_llm_provider(&call.model, explicit) {
Some(CustomLlmProvider {
model,
custom_llm_provider,
}) => (custom_llm_provider, model),
None => ("openai", call.model.as_str()),
};
if resolved_model.is_empty() {
return Err(Error::InvalidProvider(call.model));
}
let provider = provider_name
.parse::<LlmProviders>()
.map_err(|_| Error::Unsupported("native HTTP responses provider"))?;
let config = responses_provider(provider)
.ok_or(Error::Unsupported("native HTTP responses provider"))?;
let wire_model = match provider {
LlmProviders::Openai if resolved_model.contains('/') => {
return Err(Error::InvalidProvider(call.model));
}
LlmProviders::Openai => resolved_model.into(),
LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => {
format!("openrouter/{resolved_model}")
}
LlmProviders::Openrouter => resolved_model.into(),
_ => return Err(Error::Unsupported("native HTTP responses provider")),
};
let provider = ResponsesProvider {
identity: ProviderIdentity {
model: wire_model,
provider: provider_name.into(),
},
config,
};
litellm_inference::diagnostic::provider(
&provider.identity.model,
&provider.identity.provider,
);
let secrets = self
.secrets
.resolve(
provider
.config
.secret_names(call.api_key.as_deref(), call.api_base.as_deref()),
)
.await?;
let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
Box::pin(handler::execute(
self,
provider,
call,
secrets,
cache_options,
interceptors,
observers,
));
execute.await
})
.await
}
async fn run_provider(
&self,
call: ResponsesCall,
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ResponsesOutput, Error> {
let request = prepare::prepare(call, self.secrets.as_ref()).await?;
litellm_inference::diagnostic::provider(
&request.context.model,
&request.context.custom_llm_provider,
);
let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
Box::pin(handler::execute(
&self.http,
&self.auth,
request,
self.cache.clone(),
cache_options,
interceptors,
observers,
));
execute.await
}
}

View file

@ -1,162 +0,0 @@
use litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
};
use litellm_host::interceptors::{ProviderIdentity, RequestContext};
use litellm_llms::{
base_llm::responses::transformation::BaseResponsesApiConfig,
openai::responses::transformation::OpenAiResponsesApiConfig,
openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
};
use litellm_secrets::source::SecretSource;
use serde_json::Value;
use super::{
Error,
types::{ProviderResponsesRequest, ResponsesCall},
};
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
pub(super) async fn prepare(
call: ResponsesCall,
secrets: &dyn SecretSource,
) -> Result<ProviderResponsesRequest, Error> {
let ResponsesProvider { identity, config } =
resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
let provider = identity.provider.as_str();
let model = identity.model.as_str();
let snapshot = secrets
.resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
.await?;
let lookup = |name: &str| snapshot.get(name);
let environment = config.validate_environment(
litellm_http::request::string_headers("responses", call.extra_headers)?,
call.api_key.as_deref(),
&lookup,
)?;
let context = RequestContext {
model: model.into(),
custom_llm_provider: provider.into(),
optional_params: Value::Object(call.optional_params.clone()),
secret_fields: Vec::new(),
api_key: match &environment.auth {
litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => {
Some(secret.clone())
}
_ => None,
},
};
let body = config.transform_responses_api_request(model, call.input, call.optional_params)?;
Ok(ProviderResponsesRequest {
url: config.get_complete_url(call.api_base.as_deref(), &lookup),
config,
environment,
body,
context,
timeout: call.timeout,
})
}
pub(super) struct ResponsesProvider {
pub identity: ProviderIdentity,
pub config: &'static dyn BaseResponsesApiConfig,
}
/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id.
pub(super) fn resolve_provider(
model: &str,
custom_llm_provider: Option<&str>,
) -> Result<ResponsesProvider, Error> {
let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
Some(CustomLlmProvider {
model,
custom_llm_provider,
}) => (custom_llm_provider, model),
None => ("openai", model),
};
if resolved.is_empty() {
return Err(Error::InvalidProvider(model.into()));
}
let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
match provider.parse::<LlmProviders>() {
Ok(LlmProviders::Openai) if resolved.contains('/') => {
return Err(Error::InvalidProvider(model.into()));
}
Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
&OPENROUTER_RESPONSES_CONFIG,
format!("openrouter/{resolved}"),
),
Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
_ => return Err(Error::Unsupported("native HTTP responses provider")),
};
Ok(ResponsesProvider {
identity: ProviderIdentity {
model: wire_model,
provider: provider.into(),
},
config,
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
/// Pinned against Python's `get_llm_provider` on 2026-10-11.
#[rstest]
#[case::bare("gpt-5", None, "openai", "gpt-5")]
#[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")]
#[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")]
#[case::openrouter_prefix(
"openrouter/anthropic/claude",
None,
"openrouter",
"anthropic/claude"
)]
#[case::openrouter_explicit(
"openrouter/anthropic/claude",
Some("openrouter"),
"openrouter",
"anthropic/claude"
)]
#[case::openrouter_explicit_unprefixed(
"anthropic/claude",
Some("openrouter"),
"openrouter",
"anthropic/claude"
)]
#[case::openrouter_native_id(
"openrouter/auto",
Some("openrouter"),
"openrouter",
"openrouter/auto"
)]
#[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")]
fn providers_and_wire_models_match_python(
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
#[case] provider: &str,
#[case] wire_model: &str,
) {
let resolved = resolve_provider(model, custom_llm_provider).unwrap();
assert_eq!(resolved.identity.provider, provider);
assert_eq!(resolved.identity.model, wire_model);
}
#[rstest]
#[case::other_prefix("other/model", None)]
#[case::other_explicit("model", Some("other"))]
#[case::openai_with_vendor_segment("openai/other/model", None)]
#[case::empty_model("openai/", None)]
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
assert!(matches!(
resolve_provider(model, custom_llm_provider),
Err(Error::InvalidProvider(_) | Error::Unsupported(_))
));
}
}

View file

@ -2,9 +2,8 @@ use std::time::Duration;
use bytes::Bytes;
use litellm_host::call::CallOutput;
use litellm_llms::base_llm::{
auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig,
};
use litellm_host::interceptors::ProviderIdentity;
use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig;
use litellm_llms_types::formats::responses::ResponsesApiResponse;
use serde_json::{Map, Value};
@ -21,17 +20,13 @@ pub struct ResponsesCall {
pub timeout: Option<Duration>,
}
pub(crate) struct ResponsesProvider {
pub identity: ProviderIdentity,
pub config: &'static dyn BaseResponsesApiConfig,
}
pub struct ResponsesStreamHead {
pub headers: Vec<(String, String)>,
}
pub type ResponsesOutput = CallOutput<ResponsesApiResponse, ResponsesStreamHead, Bytes, Error>;
pub(super) struct ProviderResponsesRequest {
pub config: &'static dyn BaseResponsesApiConfig,
pub environment: ValidatedEnvironment,
pub url: String,
pub body: Value,
pub context: litellm_host::interceptors::RequestContext,
pub timeout: Option<Duration>,
}

View file

@ -7,6 +7,7 @@ use std::sync::Arc;
use futures_util::TryStreamExt;
use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
use litellm_inference_responses::{
Error,
route::Responses,
types::{ResponsesCall, ResponsesOutput},
};
@ -32,6 +33,110 @@ fn call() -> ResponsesCall {
}
}
#[rstest]
#[case::bare_openai("gpt-5", None, "gpt-5")]
#[case::openai_prefix("openai/gpt-5", None, "gpt-5")]
#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")]
#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")]
#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")]
#[case::explicit_openrouter_prefix(
"openrouter/anthropic/claude",
Some("openrouter"),
"anthropic/claude"
)]
#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")]
#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")]
#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")]
#[tokio::test]
async fn provider_resolution_preserves_the_wire_model(
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
#[case] expected_wire_model: &str,
) {
let response = json!({
"id": "response-1",
"model": expected_wire_model,
"output": [{"type": "message", "content": []}],
"usage": {"total_tokens": 7}
});
let upstream = upstream([json_response(response)]).await;
responses_route(no_secrets())
.execute(
ResponsesCall {
model: model.into(),
api_key: Some("test-key".into()),
api_base: Some(upstream.uri()),
custom_llm_provider: custom_llm_provider.map(str::to_owned),
..call()
},
&(),
None,
)
.await
.unwrap();
assert_eq!(
only_request(&upstream).await.json()["model"],
expected_wire_model
);
}
#[rstest]
#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))]
#[case::unsupported_responses_provider("test-model", Some("anthropic"))]
#[tokio::test]
async fn unsupported_providers_fail_before_transport(
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
) {
let upstream = upstream([]).await;
let result = responses_route(no_secrets())
.execute(
ResponsesCall {
model: model.into(),
api_base: Some(upstream.uri()),
custom_llm_provider: custom_llm_provider.map(str::to_owned),
..call()
},
&(),
None,
)
.await;
assert!(matches!(
result,
Err(Error::Unsupported("native HTTP responses provider"))
));
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[case::openai_with_vendor_segment("openai/other/model", None)]
#[case::empty_model("openai/", None)]
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
#[tokio::test]
async fn invalid_provider_models_fail_before_transport(
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
) {
let upstream = upstream([]).await;
let result = responses_route(no_secrets())
.execute(
ResponsesCall {
model: model.into(),
api_base: Some(upstream.uri()),
custom_llm_provider: custom_llm_provider.map(str::to_owned),
..call()
},
&(),
None,
)
.await;
assert!(matches!(result, Err(Error::InvalidProvider(_))));
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[case::direct(false)]
#[case::hosted(true)]

View file

@ -1,28 +1,57 @@
use std::time::Duration;
use litellm_http::{Client, request::truncate_error_body};
use litellm_llms::base_llm::auth::resolve_auth;
use litellm_http::{
Client,
request::{string_headers, truncate_error_body, with_default_headers},
};
use litellm_llms::base_llm::{
audio_transcription::transformation::BaseAudioTranscriptionConfig,
auth::{ValidatedEnvironment, resolve_auth},
};
use litellm_secrets::source::Secrets;
use serde_json::Value;
use super::Error;
use crate::{
constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::ProviderAudioTranscriptionRequest,
};
use crate::{constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::AudioTranscriptionRequest};
use litellm_inference::provider::ResolvedProvider;
pub async fn execute_audio_transcription_provider_call(
pub(super) async fn execute(
http: &Client,
auth: &litellm_auth::AuthServices,
request: ProviderAudioTranscriptionRequest,
config: &'static dyn BaseAudioTranscriptionConfig,
provider: ResolvedProvider<'_>,
call: AudioTranscriptionRequest<'_>,
secrets: Secrets,
) -> Result<Value, Error> {
let env_lookup = |key: &str| request.secrets.get(key);
let authenticated = resolve_auth(auth, request.environment.clone(), &env_lookup).await?;
let env_lookup = |key: &str| secrets.get(key);
let forwarded = string_headers("audio transcription", call.extra_headers)?;
let validated = config.validate_environment(
forwarded,
provider.model,
&call.optional_params,
&env_lookup,
)?;
let environment = ValidatedEnvironment {
headers: with_default_headers(validated.headers, config.default_headers()),
auth: validated.auth,
};
let url = config.get_complete_url(
call.api_base,
provider.model,
&call.optional_params,
&env_lookup,
)?;
let filtered_params = config.map_transcription_params(&call.optional_params);
let body = config
.transform_audio_transcription_request(provider.model, call.audio, filtered_params)?
.body;
let authenticated = resolve_auth(auth, environment, &env_lookup).await?;
let outbound = litellm_inference::outbound::outbound_request(
authenticated,
request.url.clone(),
&request.body,
url,
&body,
Some(
request
.timeout
call.timeout
.unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)),
),
)?;
@ -47,8 +76,7 @@ pub async fn execute_audio_transcription_provider_call(
error,
))
})?;
Ok(request
.config
.transform_audio_transcription_response(&request.model, response_json)?
Ok(config
.transform_audio_transcription_response(provider.model, response_json)?
.into_json())
}

View file

@ -2,16 +2,29 @@ pub mod types;
pub use litellm_inference::RouteError as Error;
mod constants;
mod handler;
mod prepare;
pub use handler::execute_audio_transcription_provider_call;
use litellm_auth::AuthServices;
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
use litellm_inference::provider::resolve_llm_provider;
use litellm_llms::{
base_llm::audio_transcription::transformation::BaseAudioTranscriptionConfig,
bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
};
use litellm_secrets::source::SecretSource;
pub use prepare::prepare_audio_transcription_provider_call;
use serde_json::Value;
use std::sync::Arc;
use crate::types::AudioTranscriptionRequest;
fn audio_transcription_provider(
provider: LlmProviders,
) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
match provider {
LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
_ => None,
}
}
#[derive(Clone)]
pub struct AudioTranscriptionRoute {
http: litellm_http::Client,
@ -32,6 +45,10 @@ impl AudioTranscriptionRoute {
}
}
pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
self.run(request).await
}
#[tracing::instrument(name = "litellm.route", skip_all, fields(
route = "audio_transcription",
model = request.model,
@ -40,13 +57,23 @@ impl AudioTranscriptionRoute {
stream = false,
outcome
))]
pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
async fn run(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
litellm_inference::diagnostic::unary(async {
let request =
prepare_audio_transcription_provider_call(request, self.secrets.as_ref()).await?;
litellm_inference::diagnostic::provider(&request.model, &request.custom_llm_provider);
let provider = resolve_llm_provider(
request.model,
request.custom_llm_provider,
"audio transcription",
)?;
let config = audio_transcription_provider(provider.provider).ok_or_else(|| {
Error::InvalidProvider(<&str>::from(provider.provider).to_string())
})?;
litellm_inference::diagnostic::provider(
provider.model,
<&'static str>::from(provider.provider),
);
let secrets = self.secrets.resolve(&config.secret_names()).await?;
let execute: futures_util::future::BoxFuture<'_, Result<Value, Error>> = Box::pin(
execute_audio_transcription_provider_call(&self.http, &self.auth, request),
handler::execute(&self.http, &self.auth, config, provider, request, secrets),
);
execute.await
})

View file

@ -1,65 +0,0 @@
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
use litellm_http::request::string_headers;
use litellm_http::request::with_default_headers;
use litellm_llms::{
base_llm::{
audio_transcription::transformation::BaseAudioTranscriptionConfig,
auth::ValidatedEnvironment,
},
bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
};
use litellm_secrets::source::SecretSource;
use super::Error;
use crate::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
use litellm_inference::provider::resolve_llm_provider;
fn provider_config(provider: LlmProviders) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
match provider {
LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
_ => None,
}
}
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
pub async fn prepare_audio_transcription_provider_call(
request: AudioTranscriptionRequest<'_>,
secrets: &dyn SecretSource,
) -> Result<ProviderAudioTranscriptionRequest, Error> {
let provider_info = resolve_llm_provider(
request.model,
request.custom_llm_provider,
"audio transcription",
)?;
let model = provider_info.model.to_string();
let config = provider_config(provider_info.provider)
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?;
let snapshot = secrets.resolve(&config.secret_names()).await?;
let env_lookup = |key: &str| snapshot.get(key);
let forwarded = string_headers("audio transcription", request.extra_headers)?;
let validated =
config.validate_environment(forwarded, &model, &request.optional_params, &env_lookup)?;
let environment = ValidatedEnvironment {
headers: with_default_headers(validated.headers, config.default_headers()),
auth: validated.auth,
};
let url = config.get_complete_url(
request.api_base,
&model,
&request.optional_params,
&env_lookup,
)?;
let filtered_params = config.map_transcription_params(&request.optional_params);
let transformed =
config.transform_audio_transcription_request(&model, request.audio, filtered_params)?;
Ok(ProviderAudioTranscriptionRequest {
model,
custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
config,
url,
body: transformed.body,
environment,
secrets: snapshot,
timeout: request.timeout,
})
}

View file

@ -1,9 +1,5 @@
use litellm_secrets::source::Secrets;
use std::time::Duration;
use litellm_llms::base_llm::{
audio_transcription::transformation::BaseAudioTranscriptionConfig, auth::ValidatedEnvironment,
};
use serde_json::{Map, Value};
pub struct AudioTranscriptionRequest<'a> {
@ -16,37 +12,3 @@ pub struct AudioTranscriptionRequest<'a> {
pub optional_params: Map<String, Value>,
pub timeout: Option<Duration>,
}
#[derive(Clone)]
pub struct ProviderAudioTranscriptionRequest {
pub model: String,
pub custom_llm_provider: String,
pub config: &'static dyn BaseAudioTranscriptionConfig,
pub url: String,
pub body: Value,
pub environment: ValidatedEnvironment,
pub secrets: Secrets,
pub timeout: Option<Duration>,
}
impl ProviderAudioTranscriptionRequest {
pub fn model(&self) -> &str {
&self.model
}
pub fn custom_llm_provider(&self) -> &str {
&self.custom_llm_provider
}
pub fn url(&self) -> &str {
&self.url
}
pub fn body(&self) -> &Value {
&self.body
}
pub fn with_body(self, body: Value) -> Self {
Self { body, ..self }
}
}

View file

@ -74,6 +74,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
headers: Headers,
api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
_litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -887,6 +888,7 @@ mod tests {
headers(forwarded),
api_key,
"claude",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
&env(vars),
)
@ -1206,6 +1208,7 @@ mod tests {
Vec::new(),
None,
"claude",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
&record,
);

View file

@ -1,7 +1,7 @@
use litellm_auth::SecretValue;
use litellm_http::request::{has_bearer_auth, has_header};
use litellm_llms_types::{
formats::messages::MessagesRequest,
formats::messages::{MessagesOptionalParams, MessagesRequest},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
@ -83,6 +83,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
headers: Headers,
api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
_litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -237,6 +238,7 @@ mod tests {
.collect(),
api_key,
"claude",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
&|_| None,
)
@ -622,6 +624,7 @@ mod tests {
Vec::new(),
None,
"claude",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
&record,
);

View file

@ -1,4 +1,6 @@
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
use litellm_llms_types::formats::messages::{
MessagesOptionalParams, MessagesRequest, MessagesResponse,
};
use litellm_router_types::LitellmParams;
use serde_json::Value;
@ -55,6 +57,7 @@ pub trait BaseMessagesConfig: Sync {
headers: Headers,
api_key: Option<&str>,
model: &str,
optional_params: &MessagesOptionalParams,
litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error>;
@ -110,6 +113,7 @@ mod tests {
headers: Headers,
_api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
_litellm_params: &LitellmParams,
_env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {

View file

@ -5,7 +5,7 @@ use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{header_value, with_header, without_headers};
use litellm_llms_types::{
formats::messages::MessagesRequest,
formats::messages::{MessagesOptionalParams, MessagesRequest},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
@ -60,6 +60,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
headers: Headers,
api_key: Option<&str>,
_model: &str,
optional_params: &MessagesOptionalParams,
params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -67,11 +68,8 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
.filter(|value| !value.trim().is_empty())
.or_else(|| {
WORKSPACE_FIELDS.iter().find_map(|field| {
params
.extra
.get(*field)?
.as_str()
.filter(|value| !value.trim().is_empty())
workspace_value(optional_params.extra.get(*field))
.or_else(|| workspace_value(params.extra.get(*field)))
})
});
let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
@ -182,6 +180,10 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
}
}
fn workspace_value(value: Option<&Value>) -> Option<&str> {
value?.as_str().filter(|value| !value.trim().is_empty())
}
fn platform_region(
base: Option<&str>,
params: &LitellmParams,
@ -260,6 +262,7 @@ mod tests {
vec![],
Some("key"),
"claude-test",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
&|_| None
)
@ -280,6 +283,7 @@ mod tests {
],
Some("explicit"),
"claude-test",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
&|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()),
)
@ -345,7 +349,14 @@ mod tests {
.get_complete_url(None, "claude-test", &params, false, &env)
.unwrap();
let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
.validate_environment(vec![], None, "claude-test", &params, &env)
.validate_environment(
vec![],
None,
"claude-test",
&MessagesOptionalParams::default(),
&params,
&env,
)
.unwrap();
assert_eq!(
url,

View file

@ -16,7 +16,7 @@ use litellm_auth_aws::{
resolve_bedrock_region,
};
use litellm_llms_types::formats::messages::{
MessagesRequest,
MessagesOptionalParams, MessagesRequest,
streaming::{MessagesStreamEvent, MessagesStreamUsage},
};
use litellm_llms_types::providers::bedrock::{
@ -146,6 +146,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
headers: Headers,
api_key: Option<&str>,
model: &str,
_optional_params: &MessagesOptionalParams,
litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -618,6 +619,7 @@ mod tests {
Vec::new(),
None,
"anthropic.claude-3",
&MessagesOptionalParams::default(),
&litellm_params,
&|_| None,
)
@ -658,6 +660,7 @@ mod tests {
vec![("authorization".into(), "Bearer forwarded".into())],
api_key,
"anthropic.claude-3",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
&env,
)

View file

@ -6,7 +6,7 @@ use litellm_auth_aws::bedrock_model_id_and_region;
use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{with_header, without_headers};
use litellm_llms_types::{
formats::messages::MessagesRequest,
formats::messages::{MessagesOptionalParams, MessagesRequest},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
};
use litellm_router_types::LitellmParams;
@ -59,6 +59,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
headers: Headers,
api_key: Option<&str>,
model: &str,
_optional_params: &MessagesOptionalParams,
params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -317,7 +318,14 @@ mod tests {
"https://private.test/anthropic/v1/messages"
);
let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
.validate_environment(vec![], None, model, &params, &|_| None)
.validate_environment(
vec![],
None,
model,
&MessagesOptionalParams::default(),
&params,
&|_| None,
)
.unwrap();
assert!(
matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region)
@ -347,7 +355,14 @@ mod tests {
.get_complete_url(None, "claude-test", &params, false, &|_| None)
.unwrap();
let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
.validate_environment(vec![], None, "claude-test", &params, &|_| None)
.validate_environment(
vec![],
None,
"claude-test",
&MessagesOptionalParams::default(),
&params,
&|_| None,
)
.unwrap();
assert_eq!(
url,
@ -401,6 +416,7 @@ mod tests {
],
key,
"claude-test",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
&|name| match name {
"BEDROCK_MANTLE_API_KEY" => Some("mantle".into()),

View file

@ -104,6 +104,7 @@ impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig {
headers: Headers,
api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
_litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -298,6 +299,7 @@ mod tests {
.collect(),
api_key,
"deepseek-v4-pro",
&MessagesOptionalParams::default(),
&LitellmParams::default(),
env,
)

View file

@ -36,6 +36,7 @@ impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig {
headers: Headers,
api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {

View file

@ -42,6 +42,7 @@ impl BaseMessagesConfig for MinimaxMessagesConfig {
headers: Headers,
api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -183,7 +184,14 @@ mod tests {
.flatten()
};
let environment = MINIMAX_MESSAGES_CONFIG
.validate_environment(vec![], key, "test-model", &Default::default(), &env)
.validate_environment(
vec![],
key,
"test-model",
&MessagesOptionalParams::default(),
&Default::default(),
&env,
)
.unwrap();
assert!(matches!(
environment.auth,

View file

@ -34,6 +34,7 @@ impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
headers: Headers,
api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {

View file

@ -43,6 +43,7 @@ impl BaseMessagesConfig for TencentAnthropicMessagesConfig {
headers: Headers,
api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -229,7 +230,14 @@ mod tests {
.flatten()
};
let environment = TENCENT_MESSAGES_CONFIG
.validate_environment(vec![], key, "test-model", &Default::default(), &env)
.validate_environment(
vec![],
key,
"test-model",
&MessagesOptionalParams::default(),
&Default::default(),
&env,
)
.unwrap();
assert!(matches!(
environment.auth,

View file

@ -114,6 +114,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
headers: Headers,
_api_key: Option<&str>,
_model: &str,
_optional_params: &MessagesOptionalParams,
litellm_params: &LitellmParams,
_env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
@ -439,6 +440,7 @@ mod tests {
.collect(),
Some("ignored-api-key"),
"claude-sonnet-4-5",
&MessagesOptionalParams::default(),
&params(Some("proj"), None),
&no_env,
)

View file

@ -3113,14 +3113,19 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
return response_ids[-1] if response_ids else None
@staticmethod
def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject:
def create_logging_object(
results: Sequence[Mapping[str, object]], litellm_model_name: str | None = None
) -> LiteLLMRealtimeStreamLoggingObject:
tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results)
return LiteLLMRealtimeStreamLoggingObject(
logging_object: Final = LiteLLMRealtimeStreamLoggingObject(
id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results),
usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results),
results=cast(OpenAIRealtimeStreamList, list(results)), # cast-ok: raw WS frames, unvalidated field
service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None,
)
if litellm_model_name is not None:
logging_object.hidden_params = {"litellm_model_name": litellm_model_name}
return logging_object
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"

View file

@ -16,6 +16,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
BaseGoogleGenAIGenerateContentConfig,
)
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes
@ -194,12 +195,14 @@ class GenerateContentHelper:
if litellm_logging_obj is None:
raise ValueError("litellm_logging_obj is required, but got None")
vertex_location: Final = VertexBase.explicit_vertex_ai_location(dict(litellm_params))
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model,
optional_params=dict(generate_content_config_dict),
litellm_params={
"litellm_call_id": litellm_call_id,
**({"vertex_location": vertex_location} if vertex_location else {}),
},
custom_llm_provider=custom_llm_provider,
)

View file

@ -142,14 +142,12 @@ def get_base_model_from_litellm_call_metadata(
if metadata is None:
return None
model_info: Final = metadata.get("model_info")
if model_info:
model_info_mapping: Final = cast( # cast-ok: model metadata is caller-provided and preserves its mapping shape
Mapping[str, object], model_info
)
return cast( # cast-ok: model metadata values are caller-provided
str | None, model_info_mapping.get("base_model")
)
return None
if not model_info:
return None
base_model: Final = cast( # cast-ok: model metadata is caller-provided and preserves its mapping shape
Mapping[str, object], model_info
).get("base_model")
return base_model if isinstance(base_model, str) else None
_get_base_model_from_litellm_call_metadata = get_base_model_from_litellm_call_metadata

View file

@ -989,6 +989,16 @@ class Logging(LiteLLMLoggingBaseClass):
return model_id
return None
def _routed_deployment_model(self) -> str | None:
litellm_params: Final = self._model_call_details_view.get("litellm_params")
metadata: Final = (
StandardLoggingPayloadSetup.merge_litellm_metadata(_LITELLM_PARAMS.validate_python(litellm_params))
if isinstance(litellm_params, dict)
else {}
)
deployment: Final = metadata.get("deployment")
return deployment if isinstance(deployment, str) else None
def get_deployment_model_for_cost(self) -> str | None:
"""The provider-qualified model to price against.
@ -2256,6 +2266,7 @@ class Logging(LiteLLMLoggingBaseClass):
and litellm_params.get(CallTypes.agenerate_content.value, False) is not True
and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True
and litellm_params.get(CallTypes.arealtime.value, False) is not True
and litellm_params.get(CallTypes.aresponses_websocket.value, False) is not True
)
_is_sync_litellm_request = is_sync_litellm_request
@ -2511,7 +2522,8 @@ class Logging(LiteLLMLoggingBaseClass):
elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
logging_result = ResponsesWebSocketTokenUsageProcessor.create_logging_object(
results=result # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
results=result, # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
litellm_model_name=self._routed_deployment_model(),
)
elif self.relays_llm_passthrough and isinstance(result, Response):

View file

@ -375,7 +375,7 @@ def get_azure_ad_token(
"Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
)
try:
azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
azure_ad_token_provider = _cached_azure_ad_token_refresh_provider(scope)
except ValueError:
verbose_logger.debug("Azure AD Token Provider could not be used.")
except Exception as e:

View file

@ -7,17 +7,21 @@ from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPC
from litellm.secret_managers.main import get_secret_str
def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str:
"""Azure serves WebRTC only on the GA v1 surface, which rejects any api-version query param."""
base: Final = api_base.rstrip("/").removesuffix("/v1").removesuffix("/openai")
return f"{base}/openai/v1/realtime/{endpoint}"
class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
def get_api_base(self, api_base: str | None, **kwargs) -> str:
def get_api_base(self, api_base: str | None, **kwargs: object) -> str:
return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or ""
def get_api_key(self, api_key: str | None, **kwargs) -> str:
def get_api_key(self, api_key: str | None, **kwargs: object) -> str:
return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or ""
def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
base: Final = self.get_api_base(api_base).rstrip("/")
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
return f"{base}/openai/realtime/client_secrets?api-version={version}"
return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets")
def validate_environment(
self,
@ -32,16 +36,14 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
}
def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
base: Final = self.get_api_base(api_base).rstrip("/")
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
return f"{base}/openai/realtime/calls?api-version={version}"
return _azure_realtime_v1_url(self.get_api_base(api_base), "calls")
def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
base: Final = self.get_api_base(api_base).rstrip("/")
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
return f"{base}/openai/realtime/transcription_sessions?api-version={version}"
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict:
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]:
return {
"api-key": ephemeral_key,
"Authorization": f"Bearer {ephemeral_key}",
}

View file

@ -6911,6 +6911,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider: str | None = None,
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:
@ -6973,12 +6974,12 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
first_message=first_message,
quota_callbacks=_ws_quota_callbacks,
turn_admission=turn_admission,
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
frame_guard=frame_guard,
**kwargs,
)
await handler.run()
return None
return await handler.run()
import websockets
from websockets.asyncio.client import ClientConnection
@ -7069,6 +7070,7 @@ class BaseLLMHTTPHandler:
authorized_model=model,
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
turn_admission=turn_admission,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

View file

@ -47779,6 +47779,34 @@
"mode": "embedding",
"output_cost_per_token": 0.0
},
"tsubasa/tsubasa-fast": {
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"input_cost_per_token": 2.0000000000000002e-07,
"output_cost_per_token": 1e-06,
"litellm_provider": "tsubasa",
"mode": "chat",
"supports_system_messages": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": false,
"supports_vision": false,
"supports_reasoning": false
},
"tsubasa/tsubasa-pro": {
"max_input_tokens": 32768,
"max_output_tokens": 16384,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 4e-06,
"litellm_provider": "tsubasa",
"mode": "chat",
"supports_system_messages": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": false,
"supports_vision": false,
"supports_reasoning": false
},
"together_ai/BAAI/bge-base-en-v1.5": {
"input_cost_per_token": 8e-09,
"litellm_provider": "together_ai",
@ -76083,32 +76111,22 @@
"supports_tool_choice": true,
"supports_vision": true
},
"tsubasa/tsubasa-fast": {
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"input_cost_per_token": 2.0000000000000002e-07,
"output_cost_per_token": 1e-06,
"litellm_provider": "tsubasa",
"mode": "chat",
"supports_system_messages": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": false,
"supports_vision": false,
"supports_reasoning": false
},
"tsubasa/tsubasa-pro": {
"max_input_tokens": 32768,
"max_output_tokens": 16384,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 4e-06,
"litellm_provider": "tsubasa",
"mode": "chat",
"supports_system_messages": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": false,
"supports_vision": false,
"supports_reasoning": false
"meta/muse-image-1.0": {
"litellm_provider": "meta",
"mode": "image_generation",
"output_cost_per_image": 0.01,
"source": "https://dev.meta.ai/docs/pricing-rate-limits",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
}
}

View file

@ -2,8 +2,11 @@ import asyncio
import traceback
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol, cast
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.batches.batch_utils import batch_cost_is_final
@ -17,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import (
get_key_object,
@ -100,6 +104,37 @@ def _proxy_stamped_used_client_oauth_token(
return stamped if isinstance(stamped, bool) else None
def _str_keyed_fields(value: object) -> Mapping[str, object]:
return value if is_str_keyed_mapping(value) else {}
def _is_model_group(value: object) -> bool:
return isinstance(value, str) and bool(value)
_TAG_LIST: Final = TypeAdapter(list[str])
def _is_tag_list(value: object) -> bool:
try:
return bool(_TAG_LIST.validate_python(value, strict=True))
except ValidationError:
return False
_ROUTED_METADATA_FIELD_CHECKS: Final = MappingProxyType({"model_group": _is_model_group, "tags": _is_tag_list})
def _routed_metadata_fallback(request_data: Mapping[str, object]) -> Mapping[str, object]:
caller_metadata: Final = _str_keyed_fields(request_data.get("metadata"))
routed_metadata: Final = _str_keyed_fields(request_data.get("litellm_metadata"))
return {
field: routed_metadata[field]
for field, is_valid in _ROUTED_METADATA_FIELD_CHECKS.items()
if not caller_metadata.get(field) and is_valid(routed_metadata.get(field))
}
def _failure_snapshot_with_guardrails_recorded_after_it(
failure_snapshot: object, recorded_guardrail_information: object
) -> Mapping[str, object] | None:
@ -220,10 +255,11 @@ class _ProxyDBLogger(CustomLogger):
metadata=_metadata, original_exception=original_exception
)
_metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_data, request_route)
request_fields: Final = _str_keyed_fields(request_data)
_metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route)
existing_metadata: Final[dict] = request_data.get("metadata", None) or {}
existing_metadata.update(_metadata)
existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata})
litellm_metadata_bucket: Final = request_data.get("litellm_metadata")
if (
@ -471,11 +507,7 @@ class _ProxyDBLogger(CustomLogger):
await _release_budget_reservation(budget_reservation=budget_reservation)
# Non-model call types (health checks, afile_delete) have no model or standard_logging_object.
# Use .get() for "stream" to avoid KeyError on health checks.
# WS session wrappers (_aresponses_websocket, _arealtime) also reach here with
# result=None; their per-turn costs are tracked on the inner aresponses/realtime calls.
if sl_object is None and (
not kwargs.get("model") or kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime")
):
if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") == "_arealtime"):
verbose_proxy_logger.warning(
"Cost tracking - skipping, no standard_logging_object for call_type=%s",
kwargs.get("call_type", "unknown"),

View file

@ -245,5 +245,4 @@ class GeminiPassthroughLoggingHandler:
logging_obj.model = litellm_model_response.model or model
logging_obj.model_call_details["model"] = logging_obj.model
logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
logging_obj.model_call_details["response_cost"] = response_cost
return kwargs

View file

@ -11,6 +11,7 @@ from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
InteractionsUsageObjectTransformation,
@ -225,7 +226,7 @@ class VertexPassthroughLoggingHandler:
start_time=start_time,
end_time=end_time,
logging_obj=logging_obj,
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route),
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider(url_route, logging_obj),
vertex_location=vertex_location,
)
@ -549,7 +550,7 @@ class VertexPassthroughLoggingHandler:
response_json=response_json,
)
custom_llm_provider: Final = VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route)
custom_llm_provider: Final = VertexPassthroughLoggingHandler._get_custom_llm_provider(url_route, logging_obj)
litellm_embedding_response.model = model
logging_obj.model = model
@ -621,7 +622,9 @@ class VertexPassthroughLoggingHandler:
start_time=start_time,
end_time=end_time,
logging_obj=litellm_logging_obj,
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route),
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider(
url_route, litellm_logging_obj
),
vertex_location=vertex_location,
)
@ -669,9 +672,16 @@ class VertexPassthroughLoggingHandler:
all_openai_chunks: Final = [parsed_chunk for parsed_chunk in parsed_chunks if parsed_chunk is not None]
complete_streaming_response: Final = litellm.stream_chunk_builder(chunks=all_openai_chunks)
VertexPassthroughLoggingHandler._drop_location_blind_stream_builder_cost(complete_streaming_response)
return complete_streaming_response
@staticmethod
def _drop_location_blind_stream_builder_cost(response: ModelResponse | TextCompletionResponse | None) -> None:
hidden_params: Final = get_hidden_params(response)
if hidden_params is not None:
hidden_params.pop("response_cost", None)
@staticmethod
def extract_model_from_url(url: str) -> str:
pattern: Final = r"/models/([^:]+)"
@ -724,9 +734,15 @@ class VertexPassthroughLoggingHandler:
return None
@staticmethod
def _get_custom_llm_provider_from_url(url: str) -> str:
parsed_url: Final = urlparse(url)
if parsed_url.hostname and parsed_url.hostname.endswith("generativelanguage.googleapis.com"):
def _get_custom_llm_provider(url: str, logging_obj: LiteLLMLoggingObj) -> str:
hostname: Final = urlparse(url).hostname
if logging_obj.model_call_details.get("custom_llm_provider") == litellm.LlmProviders.GEMINI.value or (
hostname is not None
and (
hostname == "generativelanguage.googleapis.com"
or hostname.endswith(".generativelanguage.googleapis.com")
)
):
return litellm.LlmProviders.GEMINI.value
return litellm.LlmProviders.VERTEX_AI.value

View file

@ -161,6 +161,12 @@ pass_through_endpoint_logging: Final = PassThroughEndpointLogging()
_METADATA_KEYS: Final = frozenset(("litellm_metadata", "metadata"))
_DECODED_JSON: Final = TypeAdapter(object)
_KEPT_OUT_OF_LITELLM_PARAMS: Final = _METADATA_KEYS | frozenset(wire_names(ProxyRequestState))
_PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",))
def _is_mergeable_client_metadata(key: str, value: object) -> bool:
return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
# Global registry to track registered pass-through routes and prevent memory leaks
_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
@ -633,8 +639,13 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
litellm_keys_in_body: Final = MappingProxyType(
{k: _parsed_body.pop(k) for k in types_utils.all_litellm_params if k in _parsed_body}
)
kept_out_of_litellm_params: Final = (
_KEPT_OUT_OF_LITELLM_PARAMS
if _key_or_team_allows_client_pricing_override(user_api_key_dict)
else _KEPT_OUT_OF_LITELLM_PARAMS | _PRICING_MODEL_OVERRIDE_PARAMS
)
litellm_params_in_body: Final = MappingProxyType(
{k: v for k, v in litellm_keys_in_body.items() if k not in _KEPT_OUT_OF_LITELLM_PARAMS}
{k: v for k, v in litellm_keys_in_body.items() if k not in kept_out_of_litellm_params}
)
_metadata = dict(
@ -645,7 +656,13 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
metadata: Final = litellm_keys_in_body.get("metadata")
for client_metadata in (litellm_metadata, metadata):
if isinstance(client_metadata, dict):
_metadata.update({k: v for k, v in client_metadata.items() if not k.startswith("user_api_key_")})
_metadata.update(
{
k: v
for k, v in cast(Mapping[str, object], client_metadata).items() # cast-ok: json
if _is_mergeable_client_metadata(k, v)
}
)
_metadata = _apply_key_team_project_controls(user_api_key_dict=user_api_key_dict, metadata=_metadata)
_metadata = _update_metadata_with_tags_in_header(

View file

@ -129,10 +129,11 @@ class PassThroughStreamingHandler:
usage: Final = getattr(partial_response, "usage", None)
if not isinstance(usage, Usage):
return
response_cost: Final = kwargs.get("response_cost")
logger_cost: Final = litellm_logging_obj.response_cost_calculator(result=partial_response)
handler_cost: Final = kwargs.get("response_cost")
fallback_cost: Final = float(handler_cost) if isinstance(handler_cost, (int, float)) else 0.0
litellm_logging_obj.record_partial_usage_for_failure(
usage=usage,
response_cost=float(response_cost) if isinstance(response_cost, (int, float)) else 0.0,
usage=usage, response_cost=fallback_cost if logger_cost is None else logger_cost
)
@staticmethod

View file

@ -945,6 +945,7 @@ from litellm.proxy.video_endpoints.endpoints import router as video_router
from litellm.repositories.base_repository import SupportsModelDump
from litellm.repositories.credentials_repository import CredentialsRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.responses.streaming_iterator import drain_logging_tasks
from litellm.router import (
AssistantsTypedDict,
Deployment,
@ -1239,6 +1240,19 @@ async def _flush_spend_logs_queue_on_shutdown() -> None:
verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e)
async def _flush_spend_on_shutdown() -> None:
await flush_spend_counters_on_shutdown()
await _flush_spend_logs_queue_on_shutdown()
async def _flush_spend_around_logging_drain_on_shutdown(
flush_spend: Callable[[], Awaitable[None]] = _flush_spend_on_shutdown,
) -> None:
await flush_spend()
await drain_logging_tasks()
await flush_spend()
async def _drain_audit_tasks_on_shutdown() -> None:
try:
from litellm.proxy.management_helpers.audit_logs import drain_audit_tasks
@ -1843,9 +1857,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
except Exception as e:
verbose_proxy_logger.error("Error stopping in-flight scheduled jobs: %s", e)
await flush_spend_counters_on_shutdown()
await _flush_spend_logs_queue_on_shutdown()
await _flush_spend_around_logging_drain_on_shutdown()
await proxy_config.stop_config_sync_subscriber()

View file

@ -78,7 +78,8 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
request_tags_from_request_data,
)
from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard
from litellm.proxy.utils import handle_exception_on_proxy
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task
from litellm.router import Router
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
@ -1861,12 +1862,14 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
def _responses_ws_failure_frame(failure: Exception) -> str:
raw_status: Final = getattr(failure, "status_code", None)
status: Final = raw_status if isinstance(raw_status, int) and not isinstance(raw_status, bool) else 500
proxy_exception: Final = handle_exception_on_proxy(failure)
status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500
error_type: Final = (
"rate_limit_exceeded" if status == 429 else "invalid_request_error" if 400 <= status < 500 else "server_error"
)
return json.dumps({"type": "error", "status": status, "error": {"type": error_type, "message": str(failure)}})
return json.dumps(
{"type": "error", "status": status, "error": {"type": error_type, "message": proxy_exception.message}}
)
async def _enforce_responses_ws_first_frame_model_auth(
@ -1934,7 +1937,10 @@ class _PreCallGuardrailWalk(Protocol):
def _responses_ws_frame_guard(
admitted: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, proxy_logging_obj: _PreCallGuardrailWalk
admitted: Mapping[str, object],
user_api_key_dict: UserAPIKeyAuth,
proxy_logging_obj: _PreCallGuardrailWalk,
record_refused_turn: Callable[[tuple[Mapping[str, object], ...], Exception], None],
) -> ResponsesWsFrameGuard:
"""Socket admission skips guardrails (the first frame's ``input`` is not part of the admitted request), so every
``response.create`` frame gets the pre_call guardrail walk the HTTP ``/v1/responses`` path runs on its body."""
@ -1993,10 +1999,12 @@ def _responses_ws_frame_guard(
if isinstance(nested, dict)
else {**kept, **guarded_fields}
)
guardrail_information: Final = _recorded_guardrail_information(guarded_data.get(metadata_key))
return GuardedResponsesWsFrame(
frame=json.dumps(rewritten),
guardrail_information=_recorded_guardrail_information(guarded_data.get(metadata_key)),
guardrail_information=guardrail_information,
pii_tokens=_recorded_pii_tokens(guarded_data.get("metadata")),
bill_refusal=partial(record_refused_turn, guardrail_information),
)
return guard
@ -2042,6 +2050,33 @@ def _recorded_pii_tokens(metadata: object) -> Mapping[str, str]:
_NO_PII_TOKENS: Final[Mapping[str, str]] = MappingProxyType({})
@dataclass(slots=True)
class _RefusedTurns:
"""Turns the deployment refused after their guardrails passed. The socket's close-time failure row bills their
guardrail records, so its failure hooks, model group and call type cover those turns too, and the first refusal
is that row's exception when the session limit cancels the relay before it returns one."""
first: Exception | None = None
records: tuple[Mapping[str, object], ...] = ()
def add(self, records: tuple[Mapping[str, object], ...], refusal: Exception) -> None:
self.records = (*self.records, *records)
if self.first is None:
self.first = refusal
def billed_on(self, request_data: Mapping[str, object]) -> Mapping[str, object]:
if not self.records:
return request_data
metadata_key: Final = get_metadata_variable_name_from_kwargs(request_data)
bucket: Final = request_data.get(metadata_key)
fields: Final = _WS_FRAME.validate_python(bucket) if isinstance(bucket, dict) else {}
recorded: Final = (*_recorded_guardrail_information(fields), *self.records)
return {
**request_data,
metadata_key: {**fields, _GUARDRAIL_INFORMATION_KEY: [dict(record) for record in recorded]},
}
def _recorded_guardrail_information(bucket: object) -> tuple[Mapping[str, object], ...]:
recorded: Final = (
_WS_FRAME.validate_python(bucket).get(_GUARDRAIL_INFORMATION_KEY) if isinstance(bucket, dict) else None
@ -2145,39 +2180,58 @@ async def _responses_websocket_session(
await websocket.close(code=1008, reason="Pre-call error")
return
refused_turns: Final = _RefusedTurns()
routed_data: Final = dict(
data,
user_api_key_dict=user_api_key_dict,
responses_ws_frame_guard=_responses_ws_frame_guard(data, user_api_key_dict, proxy_logging_obj),
responses_ws_frame_guard=_responses_ws_frame_guard(
data, user_api_key_dict, proxy_logging_obj, refused_turns.add
),
**_routing_hints_from_first_ws_frame(first_message),
)
# Phase 2: route to upstream provider
try:
llm_call: Final = await route_request(
data=routed_data,
route_type="_aresponses_websocket",
llm_router=llm_router,
user_model=user_model,
)
failure: Final = await llm_call
if isinstance(failure, Exception):
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=failure,
request_data=routed_data,
)
except Exception as e:
verbose_proxy_logger.exception("Responses WebSocket error")
with contextlib.suppress(Exception):
await websocket.send_text(_responses_ws_failure_frame(e))
async def write_failure_row(exception: Exception) -> None:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=routed_data,
original_exception=exception,
request_data=dict(refused_turns.billed_on(routed_data)),
)
await websocket.close(code=1011, reason="Internal server error")
def bill_socket_failure(exception: Exception) -> asyncio.Task[object]:
return spawn_logging_task(
asyncio.get_running_loop(), write_failure_row(exception), task_name="Responses WebSocket failure row"
)
async def relay_and_bill_failure() -> asyncio.Task[object] | None:
try:
llm_call: Final = await route_request(
data=routed_data,
route_type="_aresponses_websocket",
llm_router=llm_router,
user_model=user_model,
)
outcome: Final = await llm_call
except asyncio.CancelledError:
if refused_turns.first is not None:
_ = bill_socket_failure(refused_turns.first)
raise
except Exception as e:
crash_row: Final = bill_socket_failure(e)
with contextlib.suppress(Exception):
await websocket.send_text(_responses_ws_failure_frame(e))
with contextlib.suppress(Exception):
await websocket.close(code=1011, reason="Internal server error")
return crash_row
return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None
# Phase 2: route to upstream provider
try:
failure_row: Final = await relay_and_bill_failure()
finally:
await proxy_logging_obj.arelease_responses_websocket_slot(user_api_key_dict)
if failure_row is not None:
with contextlib.suppress(Exception):
await asyncio.shield(failure_row)
@router.websocket("/v1/responses")

View file

@ -1,7 +1,7 @@
import asyncio
import contextvars
import json
from collections.abc import Coroutine, Generator, Iterable, Mapping, Sequence
from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable, Mapping, Sequence
from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
@ -73,7 +73,7 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
from litellm.types.responses.main import *
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, all_litellm_params
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
from litellm.utils import (
ProviderConfigManager,
client,
@ -2609,6 +2609,7 @@ async def _aresponses_websocket(
api_base: str | None = None,
api_key: str | None = None,
timeout: float | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs,
) -> Exception | None:
@ -2624,7 +2625,7 @@ async def _aresponses_websocket(
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
user: Final = kwargs.get("user", None)
litellm_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params_dict: Final = get_litellm_params(**kwargs)
litellm_params_dict: Final = {**get_litellm_params(**kwargs), CallTypes.aresponses_websocket.value: True}
(
provider_model,
@ -2702,6 +2703,7 @@ async def _aresponses_websocket(
litellm_metadata=_build_litellm_metadata_for_ws(kwargs),
custom_llm_provider=_custom_llm_provider,
request_defaults=_build_responses_websocket_request_defaults(deployment_kwargs),
turn_admission=turn_admission,
frame_guard=responses_ws_frame_guard,
**remaining_kwargs,
)

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import contextlib
import copy
import json
import time
@ -198,7 +199,7 @@ def _running_loop() -> asyncio.AbstractEventLoop | None:
return None
def _spawn_logging_task(
def spawn_logging_task(
running_loop: asyncio.AbstractEventLoop, coroutine: Coroutine[object, object, object], *, task_name: str
) -> asyncio.Task[object]:
task: Final = running_loop.create_task(coroutine)
@ -208,6 +209,14 @@ def _spawn_logging_task(
return task
async def drain_logging_tasks(timeout: float = 5.0) -> None:
if not _PENDING_LOGGING_TASKS:
return
_, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout)
if pending:
verbose_logger.warning("Timed out draining %d logging tasks on shutdown, dropping them", len(pending))
_ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType(
{
"server_error": 500,
@ -930,7 +939,7 @@ class BaseResponsesAPIStreamingIterator:
running_loop: Final = _running_loop()
if running_loop is not None:
self._record_pending_logging_task(
_spawn_logging_task(
spawn_logging_task(
running_loop,
async_post_call_success_deployment_hook(
request_data=request_payload,
@ -966,7 +975,7 @@ class BaseResponsesAPIStreamingIterator:
running_loop: Final = _running_loop()
if running_loop is not None:
self._record_pending_logging_task(
_spawn_logging_task(
spawn_logging_task(
running_loop,
self._run_failure_handlers_in_order(exception, traceback_exception, end_time),
task_name="Responses stream failure logging",
@ -1947,6 +1956,38 @@ async def _enforce_frame_project_quota(
)
async def _admit_frame_turn(turn_admission: Callable[[], Awaitable[None]] | None, raw_message: str) -> Exception | None:
frame: Final = _json_object_or_none(raw_message)
if turn_admission is None or frame is None or frame.get("type") != "response.create":
return None
try:
await turn_admission()
except RateLimitError as refusal:
return refusal
except Exception as e:
verbose_logger.exception("Responses WS turn admission failed: %s", e)
return e
return None
def _turn_refusal_frame(refusal: RateLimitError) -> str:
return json.dumps(
{"type": "error", "status": 429, "error": {"type": "rate_limit_exceeded", "message": str(refusal)}}
)
def _turn_admission_error_frame(failure: Exception) -> str:
return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}})
def _turn_rejection_frame(rejection: Exception) -> str:
match rejection:
case RateLimitError():
return _turn_refusal_frame(rejection)
case _:
return _turn_admission_error_frame(rejection)
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
"response.created",
"response.completed",
@ -2019,16 +2060,30 @@ def _unmask_responses_event(response_str: str, pii_tokens: Mapping[str, str], un
@dataclass(frozen=True, slots=True)
class GuardedResponsesWsFrame:
"""A ``response.create`` frame after the proxy's pre_call guardrails ran on it: the frame to forward (guardrails
may rewrite ``input`` and ``instructions``), the guardrail records that bill against this turn only, and the
Presidio ``pii_tokens`` the walk masked with, which the handler needs to unmask the provider's reply."""
may rewrite ``input`` and ``instructions``), the guardrail records that bill against this turn only, the
Presidio ``pii_tokens`` the walk masked with, which the handler needs to unmask the provider's reply, and how
to carry those guardrail records onto the connection's failure row when the deployment then refuses the turn."""
frame: str
guardrail_information: tuple[Mapping[str, object], ...] = ()
pii_tokens: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({}))
bill_refusal: Callable[[Exception], None] | None = None
ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]]
async def _reject_guarded_turn(
websocket: ResponsesClientWebSocket, guarded: GuardedResponsesWsFrame, rejection: Exception
) -> None:
"""Answers the refused turn and leaves its guardrail records to the connection's close-time failure row, which
runs every failure hook once per socket, like the first refusal it records."""
if guarded.bill_refusal is not None:
guarded.bill_refusal(rejection)
with contextlib.suppress(Exception):
await websocket.send_text(_turn_rejection_frame(rejection))
_RESPONSES_WS_TERMINAL_EVENT_TYPES: Final = _HISTORY_TERMINAL_EVENT_TYPES | _RESPONSES_WS_FAILURE_EVENT_TYPES
@ -2212,6 +2267,7 @@ class ResponsesWebSocketStreaming:
authorized_model: str | None = None,
custom_llm_provider: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
frame_guard: ResponsesWsFrameGuard | None = None,
):
self.websocket = websocket
@ -2233,6 +2289,7 @@ class ResponsesWebSocketStreaming:
# response.create frame to prevent deployment-substitution attacks.
self.authorized_model: str | None = authorized_model
self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults
self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission
def _start_turn(
self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = ()
@ -2284,9 +2341,11 @@ class ResponsesWebSocketStreaming:
await self._send_error_event("invalid_request_error", str(e))
return None
async def _forward_guarded_frame(self, message: str) -> None:
async def _forward_guarded_frame(
self, message: str, turn_admission: Callable[[], Awaitable[None]] | None = None
) -> None:
guarded: Final = await self._guard_frame(message)
if guarded is None:
if guarded is None or not await self._admit_or_reject_turn(guarded, message, turn_admission):
return
if guarded.pii_tokens:
self._remember_pii_tokens(guarded.pii_tokens)
@ -2680,10 +2739,27 @@ class ResponsesWebSocketStreaming:
self.quota_callbacks, self.user_api_key_dict, self.authorized_model, message
)
except RateLimitError as e:
await self._send_error_event("rate_limit_exceeded", str(e))
await self._send_turn_refusal(e)
return False
return True
async def _admit_or_reject_turn(
self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: Callable[[], Awaitable[None]] | None
) -> bool:
rejection: Final = await _admit_frame_turn(turn_admission, message)
if rejection is None:
return True
if self._failure is None:
self._failure = rejection
await _reject_guarded_turn(self.websocket, guarded, rejection)
return False
async def _send_turn_refusal(self, refusal: RateLimitError) -> None:
try:
await self.websocket.send_text(_turn_refusal_frame(refusal))
except Exception: # noqa: BLE001, S110 # client may already be gone
pass
async def _send_error_event(self, error_type: str, message: str) -> None:
try:
await self.websocket.send_text(
@ -2702,7 +2778,7 @@ class ResponsesWebSocketStreaming:
message = await self.websocket.receive_text()
if not await self._enforce_or_reject_frame(message):
continue
await self._forward_guarded_frame(message)
await self._forward_guarded_frame(message, self.turn_admission)
except Exception as e:
verbose_logger.debug("Responses WS client_to_backend ended: %s", e)
@ -2779,6 +2855,7 @@ class ManagedResponsesWebSocketHandler:
custom_llm_provider: str | None = None,
first_message: str | None = None,
quota_callbacks: Sequence[ProjectQuotaCallback] | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
frame_guard: ResponsesWsFrameGuard | None = None,
guardrail_callbacks: Sequence[PresidioGuardrailCallback] | None = None,
**kwargs: object,
@ -2802,6 +2879,8 @@ class ManagedResponsesWebSocketHandler:
self._connection_provider = self._resolve_provider(model) or custom_llm_provider
self.first_message = first_message
self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else ()
self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission
self._failure: Exception | None = None
# Carry through safe pass-through kwargs (e.g. extra_headers)
self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS}
# In-memory session history: response_id → full accumulated message list.
@ -2830,12 +2909,11 @@ class ManagedResponsesWebSocketHandler:
return None
async def _send_error(self, message: str, error_type: str = "server_error") -> None:
try:
await self.websocket.send_text(
json.dumps({"type": "error", "error": {"type": error_type, "message": message}})
)
except Exception:
pass
await self._send_frame(json.dumps({"type": "error", "error": {"type": error_type, "message": message}}))
async def _send_frame(self, frame: str) -> None:
with contextlib.suppress(Exception):
await self.websocket.send_text(frame)
def _get_history_messages(self, previous_response_id: str) -> list[dict[str, object]]:
"""
@ -3197,7 +3275,9 @@ class ManagedResponsesWebSocketHandler:
# Core request handler
# ------------------------------------------------------------------
async def _process_response_create(self, raw_message: str) -> None:
async def _process_response_create(
self, raw_message: str, turn_admission: Callable[[], Awaitable[None]] | None = None
) -> None:
"""
Parse one ``response.create`` event, call ``litellm.aresponses(stream=True)``,
and forward every streaming event to the client.
@ -3238,12 +3318,17 @@ class ManagedResponsesWebSocketHandler:
self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message
)
except RateLimitError as e:
await self._send_error(str(e), error_type="rate_limit_exceeded")
await self._send_frame(_turn_refusal_frame(e))
return
guarded: Final = await self._guard_frame(raw_message)
if guarded is None:
return
rejection: Final = await _admit_frame_turn(turn_admission, raw_message)
if rejection is not None:
if self._failure is None:
self._failure = rejection
await _reject_guarded_turn(self.websocket, guarded, rejection)
return
if guarded.pii_tokens:
self._pii_tokens = {**self._pii_tokens, **guarded.pii_tokens}
@ -3287,7 +3372,7 @@ class ManagedResponsesWebSocketHandler:
# Main entry point
# ------------------------------------------------------------------
async def run(self) -> None:
async def run(self) -> Exception | None:
"""
Main loop: accept ``response.create`` events sequentially and handle
each one before waiting for the next message.
@ -3303,8 +3388,9 @@ class ManagedResponsesWebSocketHandler:
verbose_logger.debug("ManagedResponsesWS: client disconnected: %s", exc)
break
await self._process_response_create(message)
await self._process_response_create(message, self.turn_admission)
except Exception as exc:
verbose_logger.exception("ManagedResponsesWS: unexpected error: %s", exc)
await self._send_error(f"Internal server error: {exc}")
return self._failure

View file

@ -289,6 +289,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
deployment_has_io_token_limits,
refund_stale_reservation_before_retry,
set_io_token_rate_limit_request_kwargs,
without_io_token_limits,
)
from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
ModelRateLimitingCheck,
@ -5965,6 +5966,7 @@ class Router:
)
custom_llm_provider: Final = provider_for_generic_call(data)
generic_function_name: Final[str] = getattr(original_generic_function, "__name__", "")
response_kwargs: Final = {
**data,
@ -5972,6 +5974,11 @@ class Router:
**kwargs,
"model": model_name,
**_with_router_resolved_session_model(kwargs.get("session"), model_name),
**(
{"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs)}
if generic_function_name == "_aresponses_websocket"
else {}
),
}
# Only set custom_llm_provider if it's not None
if custom_llm_provider is not None:
@ -5981,7 +5988,7 @@ class Router:
self,
deployment,
response_kwargs,
surface_for_call(getattr(original_generic_function, "__name__", "")),
surface_for_call(generic_function_name),
)
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
response = await original_generic_function(**compacted_input)
@ -8936,8 +8943,7 @@ class Router:
Track remaining tpm/rpm quota for model in model_list
"""
try:
# WS session wrappers fire with result=None; per-turn costs tracked by inner calls.
if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"):
if kwargs.get("call_type") == "_arealtime":
return
if is_batch_retrieve_call_type(kwargs.get("call_type")):
return
@ -9605,6 +9611,71 @@ class Router:
)
raise e
def _deployment_as_configured_now(self, deployment: Mapping[str, object]) -> Mapping[str, object]:
model_info: Final = deployment.get("model_info")
model_id: Final = (
_MODEL_INFO_ADAPTER.validate_python(model_info).get("id") if isinstance(model_info, dict) else None
)
current: Final = self.get_deployment(model_id) if isinstance(model_id, str) else None
return deployment if current is None else current.model_dump(exclude_none=True)
async def async_admit_deployment_turn(
self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object]
) -> None:
deployment: Final = self._deployment_as_configured_now(opened_on)
budget_limiter: Final = self.get_router_deployment_budget_limiter()
if budget_limiter is not None:
try:
await budget_limiter.async_filter_deployments( # pyright: ignore[reportUnknownMemberType] # legacy signature takes untyped deployment dicts
model=model_group,
healthy_deployments=[dict(deployment)],
messages=None,
request_kwargs=dict(request_kwargs),
)
except ValueError as e:
raise litellm.RateLimitError(
message=str(e),
llm_provider="",
model=model_group,
category=litellm.RateLimitErrorCategory.LITELLM_RATE_LIMIT,
) from e
await self._admit_turn_through_routing_strategy(model_group, deployment, request_kwargs)
await self.async_routing_strategy_pre_call_checks( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict
deployment=dict(without_io_token_limits(deployment)), parent_otel_span=None
)
async def _admit_turn_through_routing_strategy(
self, model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object]
) -> None:
routing_kwargs: Final = {key: value for key, value in request_kwargs.items() if key != "litellm_logging_obj"}
strategy, selector = self._get_routing_context( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped request dict
model=model_group, request_kwargs=routing_kwargs
)
if strategy == "simple-shuffle" or selector is None:
return
selected: Final = await self._select_deployment_async( # pyright: ignore[reportUnknownMemberType] # legacy signature takes untyped deployment dicts
strategy=strategy,
selector=selector,
model=model_group,
healthy_deployments=[dict(deployment)],
messages=None,
input=None,
request_kwargs=routing_kwargs,
)
if selected is None:
no_deployment: Final = await async_raise_no_deployment_exception(
litellm_router_instance=self, model=model_group, parent_otel_span=None
)
raise litellm.RateLimitError(
message=str(no_deployment),
llm_provider="",
model=model_group,
category=litellm.RateLimitErrorCategory.LITELLM_RATE_LIMIT,
) from no_deployment
await self._async_override_selector_pre_call_check( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict
strategy, selector, dict(deployment), None
)
@contextlib.asynccontextmanager
async def _deployment_slot(
self, deployment: dict, kwargs: Mapping[str, object], parent_otel_span: Span | None

View file

@ -505,8 +505,7 @@ class RouterBudgetLimiting(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
"""Original method now uses helper functions"""
verbose_router_logger.debug("in RouterBudgetLimiting.async_log_success_event")
# WS session wrappers fire with result=None; per-turn costs tracked by inner calls.
if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"):
if kwargs.get("call_type") == "_arealtime":
return
standard_logging_payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
if standard_logging_payload is None:

View file

@ -23,6 +23,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.dual_cache import DualCache
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
from litellm.types.router import RouterCacheEnum, RouterErrors
from litellm.utils import get_utc_datetime
@ -96,6 +97,24 @@ def deployment_has_io_token_limits(deployment: dict) -> bool:
return itpm is not None or otpm is not None
_IO_TOKEN_LIMIT_KEYS: Final = frozenset({"itpm", "otpm"})
_IO_TOKEN_LIMIT_SECTIONS: Final = frozenset({"litellm_params", "model_info"})
def without_io_token_limits(deployment: Mapping[str, object]) -> Mapping[str, object]:
return {
key: _without_io_token_limit_keys(value) if key in _IO_TOKEN_LIMIT_SECTIONS else value
for key, value in deployment.items()
if key not in _IO_TOKEN_LIMIT_KEYS
}
def _without_io_token_limit_keys(section: object) -> object:
if not is_str_keyed_mapping(section):
return section
return {key: value for key, value in section.items() if key not in _IO_TOKEN_LIMIT_KEYS}
def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] | None:
model_id: Final = deployment.get("model_info", {}).get("id")
deployment_name: Final = deployment.get("litellm_params", {}).get("model")

View file

@ -8306,8 +8306,8 @@ def get_base_model_from_metadata(model_call_details: Mapping[str, object] | None
litellm_params: Final = cast(Mapping[str, object], model_call_details.get("litellm_params", {}))
if litellm_params is not None:
base_model: Final = litellm_params.get("base_model", None)
if base_model is not None:
return cast(str | None, base_model)
if isinstance(base_model, str):
return base_model
metadata: Final = cast(Mapping[str, object], litellm_params.get("metadata") or {})
base_model_metadata_getter: Final[_BaseModelFromMetadataGetter] = (
litellm_utils.get_base_model_from_litellm_call_metadata

View file

@ -47779,6 +47779,34 @@
"mode": "embedding",
"output_cost_per_token": 0.0
},
"tsubasa/tsubasa-fast": {
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"input_cost_per_token": 2.0000000000000002e-07,
"output_cost_per_token": 1e-06,
"litellm_provider": "tsubasa",
"mode": "chat",
"supports_system_messages": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": false,
"supports_vision": false,
"supports_reasoning": false
},
"tsubasa/tsubasa-pro": {
"max_input_tokens": 32768,
"max_output_tokens": 16384,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 4e-06,
"litellm_provider": "tsubasa",
"mode": "chat",
"supports_system_messages": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": false,
"supports_vision": false,
"supports_reasoning": false
},
"together_ai/BAAI/bge-base-en-v1.5": {
"input_cost_per_token": 8e-09,
"litellm_provider": "together_ai",
@ -76083,32 +76111,22 @@
"supports_tool_choice": true,
"supports_vision": true
},
"tsubasa/tsubasa-fast": {
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"input_cost_per_token": 2.0000000000000002e-07,
"output_cost_per_token": 1e-06,
"litellm_provider": "tsubasa",
"mode": "chat",
"supports_system_messages": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": false,
"supports_vision": false,
"supports_reasoning": false
},
"tsubasa/tsubasa-pro": {
"max_input_tokens": 32768,
"max_output_tokens": 16384,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 4e-06,
"litellm_provider": "tsubasa",
"mode": "chat",
"supports_system_messages": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": false,
"supports_vision": false,
"supports_reasoning": false
"meta/muse-image-1.0": {
"litellm_provider": "meta",
"mode": "image_generation",
"output_cost_per_image": 0.01,
"source": "https://dev.meta.ai/docs/pricing-rate-limits",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
}
}

View file

@ -134,6 +134,9 @@ ignored_function_names = [
"_provider_model_id", # Tested through get_router_model_info in test_router_model_cost_isolation.py
"_names_route_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py
"_declared_base_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py
"async_admit_deployment_turn", # Bound as the WebSocket turn_admission partial, tested through it in test_router_helper_utils.py
"_deployment_as_configured_now", # Read by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py
"_admit_turn_through_routing_strategy", # Run by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py
]

View file

@ -1544,6 +1544,73 @@ GEMINI_3_1_PRO_PASSTHROUGH_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY: Final = CostT
),
)
GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY: Final = CostTrackingTestCase(
name="gemini-3.1-pro-passthrough-stream_generate_content_priced_via_gemini_key",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
model="gemini/gemini-3.1-pro",
endpoint="/gemini/v1beta/models/$MODEL:streamGenerateContent?alt=sse",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "52b6a80ff038 summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": True,
"stream_options": {"include_usage": True},
"allowed_openai_params": [],
},
response=SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"candidates": [
{
"content": {"parts": [{"text": "scripted answer 52b6a80ff038"}], "role": "model"},
"finishReason": "STOP",
"index": 0,
}
],
"modelVersion": "gemini-3.1-pro",
},
{
"candidates": [],
"usageMetadata": {
"promptTokenCount": 1840,
"candidatesTokenCount": 412,
"totalTokenCount": 2252,
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
},
"modelVersion": "gemini-3.1-pro",
},
),
),
expected=ExactExpected(
spend=0.008624,
input_cost=0.00368,
output_cost=0.004944,
prompt_tokens=1840,
completion_tokens=412,
breakdown_persisted=False,
cost_header=False,
),
)
# gemini/gemini-3.8-flash
GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase(
@ -3475,6 +3542,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
GEMINI_GEMINI_3_8_FLASH_STREAM_FULL_USAGE,
GEMINI_EMBEDDINGS_002,
GEMINI_3_1_PRO_PASSTHROUGH_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY,
GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY,
GEMINI_GEMINI_3_8_FLASH_LITE_INPUT_BELOW_128K,
GEMINI_GEMINI_3_8_FLASH_LITE_INPUT_ABOVE_128K,
GEMINI_NANO_BANANA_2_1_INPUT_TEXT,

View file

@ -1659,73 +1659,6 @@ GEMINI_3_1_PRO_STREAM_FULL_USAGE: Final = CostTrackingTestCase(
),
)
GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_VERTEX_KEY: Final = CostTrackingTestCase(
name="gemini-3.1-pro-passthrough-stream_generate_content_priced_via_vertex_key",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
model="gemini-3.1-pro",
endpoint="/gemini/v1beta/models/$MODEL:streamGenerateContent?alt=sse",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "52b6a80ff038 summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": True,
"stream_options": {"include_usage": True},
"allowed_openai_params": [],
},
response=SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"candidates": [
{
"content": {"parts": [{"text": "scripted answer 52b6a80ff038"}], "role": "model"},
"finishReason": "STOP",
"index": 0,
}
],
"modelVersion": "gemini-3.1-pro",
},
{
"candidates": [],
"usageMetadata": {
"promptTokenCount": 1840,
"candidatesTokenCount": 412,
"totalTokenCount": 2252,
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
},
"modelVersion": "gemini-3.1-pro",
},
),
),
expected=ExactExpected(
spend=0.0090552,
input_cost=0.003864,
output_cost=0.0051912,
prompt_tokens=1840,
completion_tokens=412,
breakdown_persisted=False,
cost_header=False,
),
)
# gemini-3.8-flash
GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase(
@ -3647,7 +3580,6 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
GEMINI_3_8_FLASH_STREAM_FULL_USAGE,
IMAGEN_NEXT_IMAGES_ONE,
VERTEX_EMBEDDINGS_TEXT_006,
GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_VERTEX_KEY,
VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT,
VERTEX_AI_CLAUDE_SONNET_5_5_STREAM,
VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT,

View file

@ -0,0 +1,629 @@
import asyncio
import hashlib
import os
import threading
import uuid
from collections.abc import AsyncIterator, Callable, Iterator, Mapping
from contextlib import asynccontextmanager, contextmanager
from dataclasses import dataclass
from functools import partial
from datetime import UTC, datetime
from pathlib import Path
from types import MappingProxyType
from typing import Final
import httpx
import pytest
import websockets
import yaml
from integration._support.client import (
JSON_OBJECT,
Gateway,
Scenario,
eventually,
gateway_from_environment,
held,
list_value,
object_value,
string_value,
)
from integration._support.database import read_rows
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.responses_ws_peer import (
PROVIDER_MODEL,
USAGE,
Peer,
Script,
completed,
create,
drain,
events,
responses_peer,
turn,
until_terminal,
)
from integration._support.tls import write_self_signed_cert
from pydantic import JsonValue
from redis import Redis
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 240)
PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
INPUT_RATE: Final = 0.001
OUTPUT_RATE: Final = 0.002
TURN_SPEND: Final = USAGE["input_tokens"] * INPUT_RATE + USAGE["output_tokens"] * OUTPUT_RATE
TURN_TOKENS: Final = USAGE["total_tokens"]
MANAGED_PROVIDER: Final = "mistral"
SPEND_ROWS_FOR_KEY: Final = (
'SELECT request_id, spend, status, model_group, metadata, request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s'
)
TURN_CHECK_FAULT: Final = "turn_check_fault"
FAULTING_TURN_CHECK: Final = f"""
from litellm.integrations.custom_logger import CustomLogger
class FaultingTurnCheck(CustomLogger):
async def async_pre_call_check(self, deployment, parent_otel_span):
fault = (deployment.get("model_info") or {{}}).get("{TURN_CHECK_FAULT}")
if fault:
raise RuntimeError(fault)
return None
instance = FaultingTurnCheck()
"""
FrameReader = Callable[[], int]
@dataclass(frozen=True, slots=True)
class Deployment:
name: str
identity: str
forwarded_turns: FrameReader
def _key_hash(key: str) -> str:
return hashlib.sha256(key.encode()).hexdigest()
def _redis() -> Redis:
return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), decode_responses=True)
def _config(directory: Path, router_settings: Mapping[str, JsonValue], callbacks: tuple[str, ...]) -> Path:
base: Final = JSON_OBJECT.validate_python(yaml.safe_load(PROXY_CONFIG.read_text()))
path: Final = directory / "turn-admission.yaml"
path.write_text(
yaml.safe_dump(
{
**base,
"litellm_settings": {
**object_value(base["litellm_settings"]),
**({"callbacks": list(callbacks)} if callbacks else {}),
},
"router_settings": {
**object_value(base["router_settings"]),
"redis_host": os.environ["REDIS_HOST"],
"redis_port": int(os.environ["REDIS_PORT"]),
**router_settings,
},
}
)
)
return path
@contextmanager
def _admitting_proxy(
tmp_path_factory: pytest.TempPathFactory,
cert: tuple[Path, Path],
router_settings: Mapping[str, JsonValue],
callback_sources: Mapping[str, str] = MappingProxyType({}),
) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("responses-ws-turn-admission")
for module, source in callback_sources.items():
(directory / f"{module}.py").write_text(source)
config: Final = _config(directory, router_settings, tuple(f"{module}.instance" for module in callback_sources))
with gateway_from_environment() as base:
with owned_proxy(base, directory, {"SSL_CERT_FILE": str(cert[0])}, config=config) as proxy:
yield proxy
@pytest.fixture(scope="module")
def cert(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]:
return write_self_signed_cert(tmp_path_factory.mktemp("responses-ws-turn-admission-cert"))
@pytest.fixture(scope="module")
def limited(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]:
with _admitting_proxy(tmp_path_factory, cert, {"optional_pre_call_checks": ["enforce_model_rate_limits"]}) as proxy:
yield proxy
@pytest.fixture(scope="module")
def faulting(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]:
with _admitting_proxy(tmp_path_factory, cert, {}, {"faulting_turn_check": FAULTING_TURN_CHECK}) as proxy:
yield proxy
@pytest.fixture(scope="module")
def usage_routed(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]:
with _admitting_proxy(tmp_path_factory, cert, {"routing_strategy": "usage-based-routing-v2"}) as proxy:
yield proxy
def _native(scenario: Scenario, peer: Peer, **limits: JsonValue) -> Deployment:
identity: Final = f"ws-admission-{uuid.uuid4().hex}"
name: Final = scenario.model(
model_info={"id": identity},
model=f"openai/{PROVIDER_MODEL}",
api_base=peer.url,
input_cost_per_token=INPUT_RATE,
output_cost_per_token=OUTPUT_RATE,
**limits,
)
return Deployment(
name, identity, lambda: sum(frame.get("type") == "response.create" for frame in drain(peer.frames))
)
def _upstream_chats(gateway: Gateway, provider_model: str) -> int:
with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as upstream:
observed: Final = JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"]
assert isinstance(observed, list)
return sum(object_value(object_value(entry)["body"]).get("model") == provider_model for entry in observed)
def _managed(scenario: Scenario, **limits: JsonValue) -> Deployment:
identity: Final = f"ws-admission-{uuid.uuid4().hex}"
provider_model: Final = f"ws-managed-{uuid.uuid4().hex}"
name: Final = scenario.model(
model_info={"id": identity},
model=f"{MANAGED_PROVIDER}/{provider_model}",
input_cost_per_token=INPUT_RATE,
output_cost_per_token=OUTPUT_RATE,
**limits,
)
return Deployment(name, identity, lambda: _upstream_chats(scenario.gateway, provider_model))
def _socket_url(gateway: Gateway, model: str) -> str:
return f"{str(gateway.client.base_url).rstrip('/').replace('http://', 'ws://')}/v1/responses?model={model}"
@asynccontextmanager
async def _socket(
gateway: Gateway, key: str, model: str, headers: Mapping[str, str] | None = None
) -> AsyncIterator[websockets.ClientConnection]:
async with websockets.connect(
_socket_url(gateway, model),
additional_headers={"Authorization": f"Bearer {key}", **(headers or {})},
open_timeout=10,
) as connection:
yield connection
async def _say(connection: websockets.ClientConnection, model: str, text: str = "") -> tuple[dict[str, JsonValue], ...]:
return await turn(connection, create(model, text or f"turn {uuid.uuid4().hex}"))
def _assert_refused(received: tuple[dict[str, JsonValue], ...]) -> None:
assert [event.get("type") for event in received] == ["error"], received
assert received[0]["status"] == 429, received
assert object_value(received[0]["error"])["type"] == "rate_limit_exceeded", received
def _fresh_minute() -> str:
eventually(lambda: datetime.now(UTC), lambda now: now.second < 30, seconds=65)
return datetime.now(UTC).strftime("%H-%M")
def _deployment_spend(deployment: Deployment) -> float:
with _redis() as redis:
return float(redis.get(f"deployment_spend:{deployment.identity}:1d") or 0)
def _configured(gateway: Gateway, deployment: Deployment, section: str) -> dict[str, JsonValue]:
entries: Final = list_value(gateway.get("/model/info", {"litellm_model_id": deployment.identity})["data"])
return object_value(object_value(entries[0])[section])
def _configured_tpm(gateway: Gateway, deployment: Deployment) -> JsonValue:
return _configured(gateway, deployment, "litellm_params").get("tpm")
def _tokens_counted(identity: str, minute: str) -> int:
with _redis() as redis:
return sum(int(redis.get(key) or 0) for key in redis.scan_iter(f"{identity}:*:tpm:{minute}"))
def _settled_count(read: Callable[[], int]) -> int:
reached: Final = eventually(read, lambda tokens: tokens > 0, seconds=15)
return held(read, lambda tokens: tokens == reached, holding=3, seconds=20)
async def _turns_against_tpm(
gateway: Gateway, key: str, deployment: Deployment
) -> tuple[tuple[dict[str, JsonValue], ...], tuple[dict[str, JsonValue], ...], tuple[dict[str, JsonValue], ...], int]:
minute: Final = await asyncio.to_thread(_fresh_minute)
tokens: Final = partial(_tokens_counted, deployment.identity, minute)
async with _socket(gateway, key, deployment.name) as connection:
first: Final = await _say(connection, deployment.name)
counted_after_first: Final = await asyncio.to_thread(_settled_count, tokens)
second: Final = await _say(connection, deployment.name)
if second[-1].get("type") == "response.completed":
await _until(tokens, lambda counted: counted > counted_after_first)
return first, second, await _say(connection, deployment.name), counted_after_first
async def _until(read: Callable[[], float], satisfied: Callable[[float], bool]) -> float:
return await asyncio.to_thread(eventually, read, satisfied, 15)
def _failure_rows(key: str) -> list[dict[str, JsonValue]]:
rows: Final = eventually(
lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
lambda found: any(row["status"] == "failure" for row in found),
seconds=70,
)
return [row for row in rows if row["status"] == "failure"]
@pytest.mark.parametrize("relay", ("native", "managed"))
def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_logged_once_as_its_failure(
limited: Gateway, cert: tuple[Path, Path], relay: str
) -> None:
with responses_peer(cert) as peer, limited.scenario() as scenario:
budget: Final = {"max_budget": TURN_SPEND / 2, "budget_duration": "1d"}
deployment: Final = _native(scenario, peer, **budget) if relay == "native" else _managed(scenario, **budget)
key: Final = scenario.key(models=[deployment.name])
tag: Final = f"ws-turn-{uuid.uuid4().hex}"
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(limited, key, deployment.name, {"x-litellm-tags": tag}) as connection:
first: Final = await _say(connection, deployment.name)
await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2)
return (first, await _say(connection, deployment.name), await _say(connection, deployment.name))
first, second, third = asyncio.run(session())
completed(first)
_assert_refused(second)
_assert_refused(third)
assert deployment.forwarded_turns() == 1
failures: Final = _failure_rows(key)
assert len(failures) == 1, failures
assert failures[0]["model_group"] == deployment.name, failures
assert tag in list_value(failures[0]["request_tags"]), failures
assert float(failures[0]["spend"]) == 0.0, failures
error: Final = object_value(object_value(failures[0]["metadata"])["error_information"])
assert str(error["error_code"]) == "429", error
def _set_turn_check_fault(gateway: Gateway, deployment: Deployment, fault: str) -> None:
gateway.request(
"PATCH",
f"/model/{deployment.identity}/update",
{"model_info": {"id": deployment.identity, TURN_CHECK_FAULT: fault}},
).raise_for_status()
eventually(
lambda: _configured(gateway, deployment, "model_info").get(TURN_CHECK_FAULT) or "",
lambda configured: configured == fault,
seconds=15,
)
def _assert_turn_check_failed(received: tuple[dict[str, JsonValue], ...], fault: str) -> None:
assert [event.get("type") for event in received] == ["error"], received
assert received[0]["status"] == 500, received
error: Final = object_value(received[0]["error"])
assert error["type"] == "server_error", received
assert fault in string_value(error["message"]), received
def _first_turn_held_until(release: threading.Event) -> Script:
def script(response_id: str) -> tuple[dict[str, JsonValue], ...]:
if response_id == "resp_peer_1":
release.wait(timeout=60)
return events(f"resp_{uuid.uuid4().hex}", "seven")
return script
@pytest.mark.parametrize("relay", ("native", "managed"))
def test_a_turn_whose_admission_check_fails_gets_a_500_frame_and_the_socket_serves_the_next_turn(
faulting: Gateway, cert: tuple[Path, Path], relay: str
) -> None:
with responses_peer(cert) as peer, faulting.scenario() as scenario:
deployment: Final = _native(scenario, peer) if relay == "native" else _managed(scenario)
key: Final = scenario.key(models=[deployment.name])
fault: Final = f"turn check unavailable {uuid.uuid4().hex}"
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(faulting, key, deployment.name) as connection:
first: Final = await _say(connection, deployment.name)
await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, fault)
second: Final = await _say(connection, deployment.name)
third: Final = await _say(connection, deployment.name)
await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, "")
return first, second, third, await _say(connection, deployment.name)
first, second, third, fourth = asyncio.run(session())
completed(first)
_assert_turn_check_failed(second, fault)
_assert_turn_check_failed(third, fault)
completed(fourth)
assert deployment.forwarded_turns() == 2
failures: Final = _failure_rows(key)
assert len(failures) == 1, failures
assert failures[0]["model_group"] == deployment.name, failures
recorded: Final = object_value(object_value(failures[0]["metadata"])["error_information"])
assert fault in string_value(recorded["error_message"]), recorded
def test_a_turn_still_streaming_when_the_next_turns_check_fails_completes_and_is_billed(
faulting: Gateway, cert: tuple[Path, Path]
) -> None:
release: Final = threading.Event()
with responses_peer(cert, _first_turn_held_until(release)) as peer, faulting.scenario() as scenario:
deployment: Final = _native(scenario, peer)
key: Final = scenario.key(models=[deployment.name])
fault: Final = f"turn check unavailable {uuid.uuid4().hex}"
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(faulting, key, deployment.name) as connection:
try:
await connection.send(create(deployment.name, f"streaming {uuid.uuid4().hex}"))
await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, fault)
rejected: Final = await _say(connection, deployment.name)
finally:
release.set()
return rejected, await until_terminal(connection, ())
rejected, streamed = asyncio.run(session())
_assert_turn_check_failed(rejected, fault)
billed: Final = completed(streamed)
assert deployment.forwarded_turns() == 1
rows: Final = eventually(
lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
lambda found: any(row["request_id"] == billed["id"] for row in found),
seconds=70,
)
row: Final = next(row for row in rows if row["request_id"] == billed["id"])
assert row["status"] == "success", row
assert float(row["spend"]) == pytest.approx(TURN_SPEND), row
def test_a_socket_opened_after_websocket_turns_spent_the_budget_is_refused_and_so_is_http(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert) as peer, limited.scenario() as scenario:
deployment: Final = _native(scenario, peer, max_budget=TURN_SPEND / 2, budget_duration="1d")
key: Final = scenario.key(models=[deployment.name])
async def spend_then_reopen() -> tuple[tuple[dict[str, JsonValue], ...], int | None]:
async with _socket(limited, key, deployment.name) as spending:
completed(await _say(spending, deployment.name))
await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2)
async with _socket(limited, key, deployment.name) as refused:
received: Final = await _say(refused, deployment.name)
await asyncio.wait_for(refused.wait_closed(), timeout=20)
return received, refused.close_code
received, close_code = asyncio.run(spend_then_reopen())
_assert_refused(received)
assert close_code == 1011, close_code
assert peer.paths.qsize() == 1
assert deployment.forwarded_turns() == 1
http: Final = limited.request(
"POST", "/v1/responses", {"model": deployment.name, "input": f"after {uuid.uuid4().hex}"}, key=key
)
assert http.status_code == 429, http.text
assert peer.paths.qsize() == 1
def test_a_socket_the_router_refuses_to_open_gets_the_status_and_message_http_returns(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert) as peer, limited.scenario() as scenario:
name: Final = scenario.model(
model=f"openai/{PROVIDER_MODEL}",
api_base=peer.url,
tags=[f"granted-{uuid.uuid4().hex}"],
model_info={"enable_tag_filtering": True},
)
key: Final = scenario.key(models=[name])
foreign: Final = {"x-litellm-tags": f"foreign-{uuid.uuid4().hex}"}
http: Final = limited.request(
"POST", "/v1/responses", {"model": name, "input": "tag routing"}, key=key, headers=foreign
)
assert http.status_code != 200, http.text
http_error: Final = object_value(JSON_OBJECT.validate_json(http.content)["error"])
async def refused_open() -> tuple[dict[str, JsonValue], ...]:
async with _socket(limited, key, name, foreign) as connection:
return await _say(connection, name)
received: Final = asyncio.run(refused_open())
assert [event.get("type") for event in received] == ["error"], received
assert received[0]["status"] == http.status_code, (received, http.text)
assert object_value(received[0]["error"])["message"] == string_value(http_error["message"]), (
received,
http.text,
)
assert peer.paths.qsize() == 0
def test_each_turn_counts_as_one_request_toward_the_deployment_rpm(limited: Gateway, cert: tuple[Path, Path]) -> None:
with responses_peer(cert) as peer, limited.scenario() as scenario:
deployment: Final = _native(scenario, peer, rpm=1)
key: Final = scenario.key(models=[deployment.name])
_fresh_minute()
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(limited, key, deployment.name) as connection:
return (await _say(connection, deployment.name), await _say(connection, deployment.name))
first, second = asyncio.run(session())
completed(first)
_assert_refused(second)
assert deployment.forwarded_turns() == 1
def test_a_deployment_tpm_refuses_a_turn_only_once_earlier_turns_used_it_up(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert) as peer, limited.scenario() as scenario:
deployment: Final = _native(scenario, peer, tpm=TURN_TOKENS + 2)
key: Final = scenario.key(models=[deployment.name])
first, second, third, counted_after_first = asyncio.run(_turns_against_tpm(limited, key, deployment))
completed(first)
completed(second)
_assert_refused(third)
assert counted_after_first == TURN_TOKENS
assert deployment.forwarded_turns() == 2
def test_a_tpm_limit_lowered_while_the_socket_is_open_applies_to_its_next_turn(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert) as peer, limited.scenario() as scenario:
deployment: Final = _native(scenario, peer, tpm=100 * TURN_TOKENS)
key: Final = scenario.key(models=[deployment.name])
minute: Final = _fresh_minute()
def lower_tpm() -> None:
limited.request(
"PATCH", f"/model/{deployment.identity}/update", {"litellm_params": {"tpm": TURN_TOKENS - 2}}
).raise_for_status()
eventually(lambda: _configured_tpm(limited, deployment), lambda tpm: tpm == TURN_TOKENS - 2, seconds=15)
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(limited, key, deployment.name) as connection:
first: Final = await _say(connection, deployment.name)
await _until(partial(_tokens_counted, deployment.identity, minute), lambda tokens: tokens > 0)
await asyncio.to_thread(lower_tpm)
return (first, await _say(connection, deployment.name))
first, second = asyncio.run(session())
completed(first)
_assert_refused(second)
assert deployment.forwarded_turns() == 1
def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_turn(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert) as peer, limited.scenario() as scenario:
deployment: Final = _native(scenario, peer, otpm=10_000)
key: Final = scenario.key(models=[deployment.name])
_fresh_minute()
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(limited, key, deployment.name) as connection:
return tuple([await _say(connection, deployment.name) for _ in range(5)])
turns: Final = asyncio.run(session())
assert [completed(received)["status"] for received in turns] == ["completed"] * 5
assert deployment.forwarded_turns() == 5
def test_usage_based_routing_refuses_a_turn_only_once_earlier_turns_used_up_the_deployment_tpm(
usage_routed: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert) as peer, usage_routed.scenario() as scenario:
deployment: Final = _native(scenario, peer, tpm=TURN_TOKENS + 2)
key: Final = scenario.key(models=[deployment.name])
first, second, third, counted_after_first = asyncio.run(_turns_against_tpm(usage_routed, key, deployment))
completed(first)
completed(second)
_assert_refused(third)
assert counted_after_first == TURN_TOKENS
assert deployment.forwarded_turns() == 2
def _assert_http_chat_counted_once(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
identity: Final = f"ws-admission-{uuid.uuid4().hex}"
name: Final = scenario.model(model_info={"id": identity}, tpm=1_000_000)
key: Final = scenario.key(models=[name])
minute: Final = _fresh_minute()
answered: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": name, "messages": [{"role": "user", "content": f"http turn {uuid.uuid4().hex}"}]},
key=key,
)
assert answered.status_code == 200, answered.text
tokens: Final = object_value(JSON_OBJECT.validate_json(answered.content)["usage"])["total_tokens"]
assert isinstance(tokens, int) and tokens > 0, answered.text
assert _settled_count(partial(_tokens_counted, identity, minute)) == tokens
def test_an_http_call_counts_its_tokens_once_toward_the_deployment_tpm_under_model_rate_limits(
limited: Gateway,
) -> None:
_assert_http_chat_counted_once(limited)
def test_an_http_call_counts_its_tokens_once_toward_the_deployment_tpm_under_usage_based_routing(
usage_routed: Gateway,
) -> None:
_assert_http_chat_counted_once(usage_routed)
def test_usage_based_routing_serves_every_turn_while_the_deployment_has_headroom(
usage_routed: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert) as peer, usage_routed.scenario() as scenario:
deployment: Final = _native(scenario, peer, tpm=1_000_000, rpm=1_000)
key: Final = scenario.key(models=[deployment.name])
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(usage_routed, key, deployment.name) as connection:
return tuple([await _say(connection, deployment.name) for _ in range(3)])
turns: Final = asyncio.run(session())
assert [completed(received)["status"] for received in turns] == ["completed"] * 3
assert deployment.forwarded_turns() == 3
def test_a_frame_over_the_project_input_quota_gets_a_429_error_frame_and_never_reaches_the_provider(
limited: Gateway,
) -> None:
with limited.scenario() as scenario:
deployment: Final = _managed(scenario)
team: Final = scenario.team()
key: Final = scenario.key(
models=[deployment.name],
team_id=team,
project_id=scenario.project(team, model_itpm_limit={deployment.name: 100}),
)
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(limited, key, deployment.name) as connection:
return (
await _say(connection, deployment.name, "short"),
await _say(connection, deployment.name, "long " * 400),
await _say(connection, deployment.name, "long " * 400),
)
first, second, third = asyncio.run(session())
completed(first)
_assert_refused(third)
answered: Final = sum(received[-1].get("type") == "response.completed" for received in (first, second, third))
assert deployment.forwarded_turns() == answered

View file

@ -1,11 +1,13 @@
import json
import uuid
from collections.abc import Callable, Mapping
from typing import Final
import pytest
from integration._support.client import Gateway, eventually
from integration._support.client import Gateway, eventually, list_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue
@pytest.mark.covers("spend.failed_dispatch.failure_row_records_estimated_input_tokens")
@ -48,3 +50,79 @@ def test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_
assert row["status"] == "failure" and float(row["spend"]) == 0 and row["completion_tokens"] == 0, row
assert row["prompt_tokens"] > 0, f"failure row lost the dispatched input tokens: {row}"
assert row["total_tokens"] == row["prompt_tokens"], row
def _chat_body(model: str, text: str) -> Mapping[str, JsonValue]:
return {"model": model, "messages": [{"role": "user", "content": text}]}
def _responses_body(model: str, text: str) -> Mapping[str, JsonValue]:
return {"model": model, "input": text}
def _messages_body(model: str, text: str) -> Mapping[str, JsonValue]:
return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": text}]}
FAILURE_ROUTES: Final = pytest.mark.parametrize(
("route", "body"),
(
("/v1/chat/completions", _chat_body),
("/v1/responses", _responses_body),
("/v1/messages", _messages_body),
),
ids=("chat", "responses", "messages"),
)
def _provider_500_failure_row(
gateway: Gateway,
route: str,
body: Callable[[str, str], Mapping[str, JsonValue]],
headers: Mapping[str, str] | None = None,
) -> tuple[str, dict[str, JsonValue]]:
def provider(request: Request) -> Reply:
assert request.method == "POST"
return Reply(
status=500,
body=b'{"error":{"message":"synthetic provider outage","type":"server_error","code":"500"}}',
)
with wire_server(provider) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0)
key: Final = scenario.key(models=[model])
failed: Final = gateway.request(
"POST", route, body(model, f"failed dispatch group {uuid.uuid4().hex}"), key=key, headers=headers
)
assert failed.status_code == 500, failed.text
assert wire.drain()
rows: Final = eventually(
lambda: read_rows(
'SELECT status, model_group, request_tags FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
(failed.headers["x-litellm-call-id"],),
),
lambda values: len(values) == 1,
seconds=70,
)
assert rows[0]["status"] == "failure", rows
return model, rows[0]
@FAILURE_ROUTES
def test_a_provider_500_failure_row_names_the_model_group_the_caller_asked_for(
gateway: Gateway, route: str, body: Callable[[str, str], Mapping[str, JsonValue]]
) -> None:
model, row = _provider_500_failure_row(gateway, route, body)
assert row["model_group"] == model, row
@FAILURE_ROUTES
def test_a_provider_500_failure_row_keeps_the_tags_the_caller_sent(
gateway: Gateway, route: str, body: Callable[[str, str], Mapping[str, JsonValue]]
) -> None:
tag: Final = f"failed-dispatch-{uuid.uuid4().hex}"
_, row = _provider_500_failure_row(gateway, route, body, {"x-litellm-tags": tag})
assert tag in list_value(row["request_tags"]), row

View file

@ -0,0 +1,583 @@
import json
import uuid
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from hashlib import sha256
from pathlib import Path
from types import MappingProxyType
from typing import Final
from urllib.parse import urlsplit
import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, gateway_from_environment
from integration._support.database import read_rows
from integration._support.forward_proxy import tunnelling_forward_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.tls import server_context, write_self_signed_cert
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
_PROJECT: Final = "caller-project"
_CALLER_GOOGLE_TOKEN: Final = "caller-google-token"
_GEMINI: Final = "gemini-3.5-flash-lite"
_CLAUDE_ROUTE_MODEL: Final = "claude-haiku-4-5"
_CLAUDE_SERVED_MODEL: Final = "claude-haiku-4-5-20251001"
_OPENAI_MODEL: Final = "gpt-5.6"
_PROMPT_TOKENS: Final = 1200
_COMPLETION_TOKENS: Final = 300
_CUSTOM_INPUT_RATE: Final = 1e-05
_CUSTOM_OUTPUT_RATE: Final = 2e-05
_CUSTOM_PRICED_GEMINI_ID: Final = "audit-gemini-custom-priced"
_HOSTS: Final = (
"aiplatform.googleapis.com",
"aiplatform.us.rep.googleapis.com",
"us-east5-aiplatform.googleapis.com",
"generativelanguage.googleapis.com",
"api.openai.com",
)
_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json")
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_CUT_MARKER: Final = "cut this stream after the usage frame"
_GEMINI_USAGE: Final = {
"promptTokenCount": _PROMPT_TOKENS,
"candidatesTokenCount": _COMPLETION_TOKENS,
"totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS,
}
_DEPLOYMENT_LOCATIONS: Final = MappingProxyType({"audit-vertex-global": "global", "audit-vertex-us": "us"})
_GEMINI_ACTIONS: Final = ("generateContent", "streamGenerateContent?alt=sse")
_CONFIGURED_GATEWAY_PATH: Final = "/google-egress"
@dataclass(frozen=True, slots=True)
class _Rig:
proxy: Gateway
upstream: Wire
def _gemini_reply(request: Request) -> Reply:
body: Final = json.dumps(
{
"candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}, "finishReason": "STOP"}],
"usageMetadata": _GEMINI_USAGE,
"modelVersion": _GEMINI,
"responseId": uuid.uuid4().hex,
}
)
if ":streamGenerateContent" in request.target:
return Reply(chunks=(f"data: {body}\r\n\r\n".encode(),), content_type="text/event-stream")
return Reply(body=body.encode())
def _sse(event: str, payload: Mapping[str, JsonValue]) -> bytes:
return f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode()
def _claude_reply(request: Request) -> Reply:
message_id: Final = f"msg_vrtx_{uuid.uuid4().hex}"
if ":streamRawPredict" not in request.target:
return Reply(
body=json.dumps(
{
"model": _CLAUDE_SERVED_MODEL,
"id": message_id,
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "scripted"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": _PROMPT_TOKENS, "output_tokens": _COMPLETION_TOKENS},
}
).encode()
)
return Reply(
content_type="text/event-stream",
chunks=(
_sse(
"message_start",
{
"type": "message_start",
"message": {
"model": _CLAUDE_SERVED_MODEL,
"id": message_id,
"type": "message",
"role": "assistant",
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": _PROMPT_TOKENS, "output_tokens": 1},
},
},
),
_sse(
"content_block_start",
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
),
_sse(
"content_block_delta",
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "scripted"}},
),
_sse("content_block_stop", {"type": "content_block_stop", "index": 0}),
_sse(
"message_delta",
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": _COMPLETION_TOKENS},
},
),
_sse("message_stop", {"type": "message_stop"}),
),
)
def _openai_reply(request: Request) -> Reply:
identity: Final = f"chatcmpl-{uuid.uuid4().hex}"
chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": _OPENAI_MODEL}
frames: Final = (
{**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}}]},
{**chunk, "choices": [{"index": 0, "delta": {"content": "scripted"}}]},
{**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
{
**chunk,
"choices": [],
"usage": {
"prompt_tokens": _PROMPT_TOKENS,
"completion_tokens": _COMPLETION_TOKENS,
"total_tokens": _PROMPT_TOKENS + _COMPLETION_TOKENS,
},
},
)
encoded: Final = tuple(f"data: {json.dumps(frame)}\n\n".encode() for frame in frames)
if _CUT_MARKER.encode() in request.body:
return Reply(chunks=(*encoded, b"data: [DONE]\n\n"), content_type="text/event-stream", abort_after=len(encoded))
return Reply(chunks=(*encoded, b"data: [DONE]\n\n"), content_type="text/event-stream")
def _upstream(request: Request) -> Reply:
if request.target.startswith("/v1/chat/completions"):
return _openai_reply(request)
if "/publishers/anthropic/models/" in request.target:
assert f"/v1/projects/{_PROJECT}/locations/" in request.target, request.target
return _claude_reply(request)
return _gemini_reply(request)
def _port(url: str) -> int:
port: Final = urlsplit(url).port
assert port is not None, url
return port
def _cost_row(name: str) -> dict[str, JsonValue]:
return _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[name])
def _list_price(name: str) -> float:
row: Final = _cost_row(name)
input_rate: Final = row["input_cost_per_token"]
output_rate: Final = row["output_cost_per_token"]
assert isinstance(input_rate, float) and isinstance(output_rate, float), row
return _PROMPT_TOKENS * input_rate + _COMPLETION_TOKENS * output_rate
def _uplift(name: str) -> float:
multiplier: Final = _cost_row(name).get("regional_endpoint_uplift_multiplier")
assert isinstance(multiplier, int | float) and multiplier > 1, name
return float(multiplier)
def _config(directory: Path, token_url: str) -> Path:
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
existing: Final = base.get("model_list") or []
assert isinstance(existing, list), base
deployments: Final = [
{
"model_name": name,
"litellm_params": {
"model": f"vertex_ai/{_GEMINI}",
"vertex_project": _PROJECT,
"vertex_location": location,
"vertex_credentials": service_account_json(_PROJECT, token_url),
},
}
for name, location in _DEPLOYMENT_LOCATIONS.items()
]
custom_priced_gemini: Final = {
"model_name": _CUSTOM_PRICED_GEMINI_ID,
"litellm_params": {
"model": f"gemini/{_GEMINI}",
"api_key": "scripted-gemini-key",
"input_cost_per_token": _CUSTOM_INPUT_RATE,
"output_cost_per_token": _CUSTOM_OUTPUT_RATE,
},
"model_info": {"id": _CUSTOM_PRICED_GEMINI_ID},
}
path: Final = directory / "config.yaml"
path.write_text(yaml.safe_dump({**base, "model_list": [*existing, *deployments, custom_priced_gemini]}))
return path
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("passthrough-stream-pricing")
certificate: Final = write_self_signed_cert(directory, _HOSTS)
with (
wire_server(_upstream, tls=server_context(*certificate)) as upstream,
tunnelling_forward_proxy(_port(upstream.url)) as tunnel,
gateway_from_environment() as base,
owned_proxy(
base,
directory,
{
"HTTPS_PROXY": tunnel.url,
"NO_PROXY": "127.0.0.1,localhost",
"AIOHTTP_TRUST_ENV": "True",
"SSL_VERIFY": "False",
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
"OPENAI_API_KEY": "sk-scripted-openai",
"GEMINI_API_KEY": "scripted-gemini-key",
},
config=_config(directory, base.upstream_url.rstrip("/")),
workers=2,
) as proxy,
):
yield _Rig(proxy, upstream)
def _gateway_config(directory: Path, upstream_url: str) -> Path:
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
general_settings: Final = base.get("general_settings")
assert isinstance(general_settings, dict), base
configured_endpoint: Final = {
"path": _CONFIGURED_GATEWAY_PATH,
"target": f"{upstream_url}/egress",
"include_subpath": True,
"auth": True,
}
path: Final = directory / "config.yaml"
path.write_text(
yaml.safe_dump(
{**base, "general_settings": {**general_settings, "pass_through_endpoints": [configured_endpoint]}}
)
)
return path
@pytest.fixture(scope="module")
def gateway_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("passthrough-google-gateway")
with (
wire_server(_gemini_reply) as upstream,
gateway_from_environment() as base,
owned_proxy(
base,
directory,
{
"GEMINI_API_BASE": f"{upstream.url}/google",
"GEMINI_API_KEY": "scripted-gemini-key",
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
},
config=_gateway_config(directory, upstream.url),
workers=2,
) as proxy,
):
yield _Rig(proxy, upstream)
def _spend_row(key: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT spend, custom_llm_provider FROM "LiteLLM_SpendLogs" WHERE api_key=%s',
(sha256(key.encode()).hexdigest(),),
),
lambda found: len(found) == 1,
seconds=70,
)
return rows[0]
def _spend_for_key(key: str) -> float:
spend: Final = _spend_row(key)["spend"]
assert isinstance(spend, float), key
return spend
def _provider_for_key(key: str) -> JsonValue:
return _spend_row(key)["custom_llm_provider"]
def _post_and_read(rig: _Rig, path: str, body: Mapping[str, JsonValue], headers: Mapping[str, str]) -> None:
with rig.proxy.client.stream("POST", path, json=body, headers=dict(headers)) as response:
payload: Final = response.read()
assert response.status_code == 200, payload
def _claude_path(location: str, action: str) -> str:
return (
f"/vertex_ai/v1/projects/{_PROJECT}/locations/{location}/publishers/anthropic/models/"
f"{_CLAUDE_ROUTE_MODEL}:{action}"
)
def test_claude_on_vertex_passthrough_spend_carries_the_regional_uplift_whether_or_not_the_call_streams(
rig: _Rig,
) -> None:
body: Final[Mapping[str, JsonValue]] = {
"anthropic_version": "vertex-2023-10-16",
"max_tokens": 64,
"messages": [{"role": "user", "content": "hi"}],
}
cells: Final = (
("global", "rawPredict"),
("global", "streamRawPredict"),
("us-east5", "rawPredict"),
("us-east5", "streamRawPredict"),
)
with rig.proxy.scenario() as scenario:
keys: Final = {cell: scenario.key() for cell in cells}
for (location, action), key in keys.items():
_post_and_read(
rig,
_claude_path(location, action),
{**body, "stream": action == "streamRawPredict"},
{"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"},
)
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
standard: Final = _list_price(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}")
regional: Final = standard * _uplift(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}")
assert spend == pytest.approx(
{
("global", "rawPredict"): standard,
("global", "streamRawPredict"): standard,
("us-east5", "rawPredict"): regional,
("us-east5", "streamRawPredict"): regional,
}
)
def test_v1beta_generate_content_on_a_vertex_deployment_bills_the_deployment_location_in_both_modes(
rig: _Rig,
) -> None:
body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
actions: Final = ("generateContent", "streamGenerateContent?alt=sse")
cells: Final = tuple((deployment, action) for deployment in _DEPLOYMENT_LOCATIONS for action in actions)
with rig.proxy.scenario() as scenario:
keys: Final = {cell: scenario.key() for cell in cells}
for (deployment, action), key in keys.items():
_post_and_read(rig, f"/v1beta/models/{deployment}:{action}", body, {"x-goog-api-key": key})
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
standard: Final = _list_price(f"vertex_ai/{_GEMINI}")
regional: Final = standard * _uplift(f"vertex_ai/{_GEMINI}")
assert spend == pytest.approx(
{
("audit-vertex-global", "generateContent"): standard,
("audit-vertex-global", "streamGenerateContent?alt=sse"): standard,
("audit-vertex-us", "generateContent"): regional,
("audit-vertex-us", "streamGenerateContent?alt=sse"): regional,
}
)
def test_gemini_passthrough_bills_a_named_custom_priced_deployment_only_for_keys_that_allow_it_in_both_modes(
rig: _Rig,
) -> None:
body: Final[Mapping[str, JsonValue]] = {
"contents": [{"role": "user", "parts": [{"text": "hi"}]}],
"metadata": {
"model_info": {
"id": _CUSTOM_PRICED_GEMINI_ID,
"input_cost_per_token": _CUSTOM_INPUT_RATE,
"output_cost_per_token": _CUSTOM_OUTPUT_RATE,
}
},
}
actions: Final = ("generateContent", "streamGenerateContent?alt=sse")
with rig.proxy.scenario() as scenario:
allowed: Final = {action: scenario.key(metadata={"allow_client_pricing_override": True}) for action in actions}
plain: Final = {action: scenario.key() for action in actions}
for action in actions:
for key in (allowed[action], plain[action]):
_post_and_read(rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key})
spend: Final = {
**{("allowed", action): _spend_for_key(key) for action, key in allowed.items()},
**{("plain", action): _spend_for_key(key) for action, key in plain.items()},
}
custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE
list_price: Final = _list_price(f"gemini/{_GEMINI}")
assert spend == pytest.approx(
{
("allowed", "generateContent"): custom_price,
("allowed", "streamGenerateContent?alt=sse"): custom_price,
("plain", "generateContent"): list_price,
("plain", "streamGenerateContent?alt=sse"): list_price,
}
)
def test_openai_passthrough_stream_cut_off_after_its_usage_bills_what_the_finished_stream_bills(rig: _Rig) -> None:
def body(prompt: str) -> Mapping[str, JsonValue]:
return {
"model": _OPENAI_MODEL,
"stream": True,
"stream_options": {"include_usage": True},
"messages": [{"role": "user", "content": prompt}],
}
with rig.proxy.scenario() as scenario:
finished_key: Final = scenario.key()
cut_key: Final = scenario.key()
_post_and_read(rig, "/openai/v1/chat/completions", body("hi"), {"Authorization": f"Bearer {finished_key}"})
with rig.proxy.client.stream(
"POST",
"/openai/v1/chat/completions",
json=dict(body(_CUT_MARKER)),
headers={"Authorization": f"Bearer {cut_key}"},
) as response:
assert response.status_code == 200
with pytest.raises(httpx.RemoteProtocolError):
response.read()
spend: Final = {"finished": _spend_for_key(finished_key), "cut off": _spend_for_key(cut_key)}
list_price: Final = _list_price(_OPENAI_MODEL)
assert spend == pytest.approx({"finished": list_price, "cut off": list_price})
def _caller_headers(route: str, key: str) -> Mapping[str, str]:
if route == "vertex_ai":
return {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"}
return {"x-goog-api-key": key}
def test_a_body_base_model_reprices_passthrough_spend_only_for_keys_that_allow_client_pricing(rig: _Rig) -> None:
routes: Final = MappingProxyType(
{
"vertex_ai": (
f"/vertex_ai/v1/projects/{_PROJECT}/locations/global/publishers/google/models/{_GEMINI}",
"gemini-2.5-flash-lite",
),
"gemini": (f"/gemini/v1beta/models/{_GEMINI}", "gemini/gemini-2.5-flash-lite"),
}
)
cells: Final = tuple(
(route, action, grant) for route in routes for action in _GEMINI_ACTIONS for grant in ("allowed", "plain")
)
markers: Final = {cell: uuid.uuid4().hex for cell in cells}
with rig.proxy.scenario() as scenario:
keys: Final = {
cell: scenario.key(metadata={"allow_client_pricing_override": True})
if cell[2] == "allowed"
else scenario.key()
for cell in cells
}
for (route, action, grant), key in keys.items():
path, base_model = routes[route]
_post_and_read(
rig,
f"{path}:{action}",
{
"contents": [{"role": "user", "parts": [{"text": markers[(route, action, grant)]}]}],
"base_model": base_model,
},
_caller_headers(route, key),
)
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
forwarded: Final = rig.upstream.drain()
marked_bodies: Final = {
cell: tuple(request.body for request in forwarded if marker.encode() in request.body)
for cell, marker in markers.items()
}
assert all(len(bodies) == 1 and b"base_model" not in bodies[0] for bodies in marked_bodies.values()), marked_bodies
expected_price: Final = {
"vertex_ai": {"allowed": _list_price("gemini-2.5-flash-lite"), "plain": _list_price(f"vertex_ai/{_GEMINI}")},
"gemini": {"allowed": _list_price("gemini/gemini-2.5-flash-lite"), "plain": _list_price(f"gemini/{_GEMINI}")},
}
assert spend == pytest.approx({cell: expected_price[cell[0]][cell[2]] for cell in cells})
def test_a_non_string_body_base_model_bills_the_list_price_instead_of_nothing(rig: _Rig) -> None:
routes: Final = MappingProxyType(
{
"vertex_ai": f"/vertex_ai/v1/projects/{_PROJECT}/locations/global/publishers/google/models/{_GEMINI}",
"gemini": f"/gemini/v1beta/models/{_GEMINI}",
"v1beta": "/v1beta/models/audit-vertex-global",
}
)
cells: Final = tuple((route, action) for route in routes for action in _GEMINI_ACTIONS)
with rig.proxy.scenario() as scenario:
keys: Final = {
cell: scenario.key()
if cell[0] == "v1beta"
else scenario.key(metadata={"allow_client_pricing_override": True})
for cell in cells
}
for (route, action), key in keys.items():
_post_and_read(
rig,
f"{routes[route]}:{action}",
{"contents": [{"role": "user", "parts": [{"text": uuid.uuid4().hex}]}], "base_model": 123},
_caller_headers(route, key),
)
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
list_price: Final = {
"vertex_ai": _list_price(f"vertex_ai/{_GEMINI}"),
"gemini": _list_price(f"gemini/{_GEMINI}"),
"v1beta": _list_price(f"vertex_ai/{_GEMINI}"),
}
assert spend == pytest.approx({cell: list_price[cell[0]] for cell in cells})
def test_gemini_passthrough_through_a_gemini_api_base_gateway_logs_and_bills_as_gemini_in_both_modes(
gateway_rig: _Rig,
) -> None:
body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
with gateway_rig.proxy.scenario() as scenario:
keys: Final = {action: scenario.key() for action in _GEMINI_ACTIONS}
for action, key in keys.items():
_post_and_read(gateway_rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key})
providers: Final = {action: _provider_for_key(key) for action, key in keys.items()}
spend: Final = {action: _spend_for_key(key) for action, key in keys.items()}
forwarded: Final = tuple(
path
for path in (urlsplit(request.target).path for request in gateway_rig.upstream.drain())
if path.startswith("/google/")
)
assert sorted(forwarded) == sorted(
f"/google/v1beta/models/{_GEMINI}:{action.split('?')[0]}" for action in _GEMINI_ACTIONS
)
assert providers == {action: "gemini" for action in _GEMINI_ACTIONS}
assert spend == pytest.approx({action: _list_price(f"gemini/{_GEMINI}") for action in _GEMINI_ACTIONS})
def test_a_configured_passthrough_to_a_non_google_gateway_bills_as_vertex_whatever_the_path(gateway_rig: _Rig) -> None:
body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
routes: Final = MappingProxyType(
{
"vertex resource": f"/v1/projects/{_PROJECT}/locations/us/publishers/google/models/{_GEMINI}",
"short gateway path": f"/vx/models/{_GEMINI}",
}
)
cells: Final = tuple((route, action) for route in routes for action in _GEMINI_ACTIONS)
with gateway_rig.proxy.scenario() as scenario:
keys: Final = {cell: scenario.key(allowed_passthrough_routes=[_CONFIGURED_GATEWAY_PATH]) for cell in cells}
for (route, action), key in keys.items():
_post_and_read(
gateway_rig,
f"{_CONFIGURED_GATEWAY_PATH}{routes[route]}:{action}",
body,
{"Authorization": f"Bearer {key}"},
)
providers: Final = {cell: _provider_for_key(key) for cell, key in keys.items()}
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
short_path_spend: Final = {action: spend[("short gateway path", action)] for action in _GEMINI_ACTIONS}
assert providers == {cell: "vertex_ai" for cell in cells}
assert {action: spend[("vertex resource", action)] for action in _GEMINI_ACTIONS} == pytest.approx(
{action: _list_price(f"vertex_ai/{_GEMINI}") * _uplift(f"vertex_ai/{_GEMINI}") for action in _GEMINI_ACTIONS}
)
assert min(short_path_spend.values()) > 0
assert short_path_spend == pytest.approx(
{action: short_path_spend[_GEMINI_ACTIONS[0]] for action in _GEMINI_ACTIONS}
)

View file

@ -0,0 +1,200 @@
import itertools
import json
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Final
from urllib.parse import urlsplit
import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, gateway_from_environment
from integration._support.database import read_rows
from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.tls import server_context, write_self_signed_cert
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
_MODEL: Final = "gemini-3.5-flash-lite"
_PROJECT: Final = "caller-project"
_CALLER_GOOGLE_TOKEN: Final = "caller-google-token"
_GLOBAL_HOST: Final = "aiplatform.googleapis.com"
_US_HOST: Final = "aiplatform.us.rep.googleapis.com"
_GENERATE: Final = "generateContent"
_STREAM: Final = "streamGenerateContent?alt=sse"
_LOCATIONS: Final = ("global", "us")
_ACTIONS: Final = (_GENERATE, _STREAM)
_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json")
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_BODY: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
_CUT_PROMPT: Final = "end this stream early"
_CUT_BODY: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": _CUT_PROMPT}]}]}
_PROMPT_TOKENS: Final = 1200
_COMPLETION_TOKENS: Final = 300
_USAGE: Final = {
"promptTokenCount": _PROMPT_TOKENS,
"candidatesTokenCount": _COMPLETION_TOKENS,
"totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS,
}
_REPLY: Final = json.dumps(
{
"candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}, "finishReason": "STOP"}],
"usageMetadata": _USAGE,
"modelVersion": _MODEL,
"responseId": "scripted-response",
}
)
_UNFINISHED_FRAME: Final = json.dumps(
{
"candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}}],
"usageMetadata": _USAGE,
"modelVersion": _MODEL,
"responseId": "scripted-cut-response",
}
)
_CUSTOM_DEPLOYMENT: Final = "vertex-custom-priced"
_CUSTOM_INPUT_RATE: Final = 1e-06
_CUSTOM_OUTPUT_RATE: Final = 2e-06
@dataclass(frozen=True, slots=True)
class _Rig:
proxy: Gateway
tunnel: ForwardProxy
def _vertex(request: Request) -> Reply:
assert f"/v1/projects/{_PROJECT}/locations/" in request.target, request.target
if _CUT_PROMPT.encode() in request.body:
frame: Final = f"data: {_UNFINISHED_FRAME}\r\n\r\n".encode()
return Reply(chunks=(frame, frame), content_type="text/event-stream", abort_after=1)
if ":streamGenerateContent" in request.target:
return Reply(chunks=(f"data: {_REPLY}\r\n\r\n".encode(),), content_type="text/event-stream")
return Reply(body=_REPLY.encode())
def _port(url: str) -> int:
port: Final = urlsplit(url).port
assert port is not None, url
return port
def _uplift() -> float:
row: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[f"vertex_ai/{_MODEL}"])
multiplier: Final = row.get("regional_endpoint_uplift_multiplier")
assert isinstance(multiplier, int | float) and multiplier > 1, row
return float(multiplier)
def _config_with_custom_priced_deployment(directory: Path, token_url: str) -> Path:
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
existing: Final = base.get("model_list") or []
assert isinstance(existing, list), base
deployment: Final = {
"model_name": _CUSTOM_DEPLOYMENT,
"litellm_params": {
"model": f"vertex_ai/{_MODEL}",
"vertex_project": _PROJECT,
"vertex_location": "us",
"vertex_credentials": service_account_json(_PROJECT, token_url),
"use_in_pass_through": True,
"input_cost_per_token": _CUSTOM_INPUT_RATE,
"output_cost_per_token": _CUSTOM_OUTPUT_RATE,
},
}
path: Final = directory / "config.yaml"
path.write_text(yaml.safe_dump({**base, "model_list": [*existing, deployment]}))
return path
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("vertex-passthrough-uplift")
certificate: Final = write_self_signed_cert(directory, (_GLOBAL_HOST, _US_HOST))
with (
wire_server(_vertex, tls=server_context(*certificate)) as vertex,
tunnelling_forward_proxy(_port(vertex.url)) as tunnel,
gateway_from_environment() as base,
owned_proxy(
base,
directory,
{
"HTTPS_PROXY": tunnel.url,
"NO_PROXY": "127.0.0.1,localhost",
"AIOHTTP_TRUST_ENV": "True",
"SSL_VERIFY": "False",
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
},
config=_config_with_custom_priced_deployment(directory, base.upstream_url.rstrip("/")),
workers=2,
) as proxy,
):
yield _Rig(proxy, tunnel)
def _path(location: str, model: str, action: str) -> str:
return f"/vertex_ai/v1/projects/{_PROJECT}/locations/{location}/publishers/google/models/{model}:{action}"
def _headers(key: str) -> dict[str, str]:
return {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"}
def _spend(rig: _Rig, key: str, location: str, action: str, model: str = _MODEL) -> float:
with rig.proxy.client.stream("POST", _path(location, model, action), json=_BODY, headers=_headers(key)) as response:
body: Final = response.read()
assert response.status_code == 200, body
call_id: Final = response.headers["x-litellm-call-id"]
return _logged_spend(call_id)
def _logged_spend(call_id: str) -> float:
rows: Final = eventually(
lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)),
lambda found: len(found) == 1,
seconds=70,
)
spend: Final = rows[0]["spend"]
assert isinstance(spend, float), rows
return spend
def test_vertex_passthrough_spend_carries_the_regional_uplift_whether_or_not_the_call_streams(rig: _Rig) -> None:
with rig.proxy.scenario() as scenario:
key: Final = scenario.key()
spend: Final = {cell: _spend(rig, key, *cell) for cell in itertools.product(_LOCATIONS, _ACTIONS)}
standard: Final = spend[("global", _GENERATE)]
regional: Final = standard * _uplift()
assert standard > 0, spend
assert spend == pytest.approx(
{
("global", _GENERATE): standard,
("global", _STREAM): standard,
("us", _GENERATE): regional,
("us", _STREAM): regional,
}
)
assert set(rig.tunnel.targets()) == {f"{_GLOBAL_HOST}:443", f"{_US_HOST}:443"}
def test_custom_priced_vertex_passthrough_bills_the_deployment_price_even_when_the_stream_is_cut_off(rig: _Rig) -> None:
with rig.proxy.scenario() as scenario:
key: Final = scenario.key()
finished: Final = {action: _spend(rig, key, "us", action, _CUSTOM_DEPLOYMENT) for action in _ACTIONS}
with rig.proxy.client.stream(
"POST", _path("us", _CUSTOM_DEPLOYMENT, _STREAM), json=_CUT_BODY, headers=_headers(key)
) as response:
assert response.status_code == 200
call_id: Final = response.headers["x-litellm-call-id"]
with pytest.raises(httpx.RemoteProtocolError):
response.read()
cut_off: Final = _logged_spend(call_id)
custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE
assert {**finished, "cut off": cut_off} == pytest.approx(
{_GENERATE: custom_price, _STREAM: custom_price, "cut off": custom_price}
)

View file

@ -4,6 +4,8 @@ Test to verify the Google GenAI generate_content adapter functionality
"""
import json
from collections.abc import Iterator
from typing import Final
import pytest
@ -286,3 +288,97 @@ def test_vertex_generate_content_routes_by_deployment_base_model(base_model, use
)
assert (setup.generate_content_provider_config is not None) is uses_native_api
_UPLIFT_PROBE: Final = "gemini-regional-uplift-probe"
_UPLIFT_INPUT_RATE: Final = 1e-06
_UPLIFT_OUTPUT_RATE: Final = 2e-06
_UPLIFT_MULTIPLIER: Final = 1.1
@pytest.fixture
def vertex_regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
rates: Final = {
"input_cost_per_token": _UPLIFT_INPUT_RATE,
"output_cost_per_token": _UPLIFT_OUTPUT_RATE,
"mode": "chat",
}
monkeypatch.setitem(litellm.model_cost, _UPLIFT_PROBE, {**rates, "litellm_provider": "vertex_ai-language-models"})
monkeypatch.setattr(litellm, "vertex_language_models", {*litellm.vertex_language_models, _UPLIFT_PROBE})
monkeypatch.setitem(
litellm.model_cost,
f"vertex_ai/{_UPLIFT_PROBE}",
{
**rates,
"litellm_provider": "vertex_ai-language-models",
"regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER,
},
)
monkeypatch.delenv("VERTEXAI_LOCATION", raising=False)
monkeypatch.delenv("VERTEX_LOCATION", raising=False)
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
async def _logged_vertex_generate_content_cost(location: str, *, stream: bool) -> float:
import asyncio
import datetime
from litellm.google_genai.main import GenerateContentHelper
from litellm.google_genai.streaming_iterator import BaseGoogleGenAIGenerateContentStreamingIterator
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.google_genai.main import GenerateContentResponse
reply: Final = {
"candidates": [{"content": {"role": "model", "parts": [{"text": "hi"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8},
"modelVersion": _UPLIFT_PROBE,
}
logging_obj: Final = Logging(
model=f"vertex_ai/{_UPLIFT_PROBE}",
messages=[],
stream=stream,
call_type="agenerate_content_stream" if stream else "agenerate_content",
start_time=datetime.datetime(2026, 1, 1),
litellm_call_id="uplift-test",
function_id="uplift-test",
)
setup: Final = GenerateContentHelper.setup_generate_content_call(
model=f"vertex_ai/{_UPLIFT_PROBE}",
contents=[{"role": "user", "parts": [{"text": "hi"}]}],
litellm_logging_obj=logging_obj,
vertex_location=location,
stream=stream,
)
if stream:
iterator: Final = BaseGoogleGenAIGenerateContentStreamingIterator(
litellm_logging_obj=logging_obj,
request_body={},
model=setup.model,
custom_llm_provider=setup.custom_llm_provider,
)
iterator.collected_chunks.append(f"data: {json.dumps(reply)}\n\n".encode())
await iterator._handle_async_streaming_logging()
await asyncio.gather(*(task for task in asyncio.all_tasks() if task is not asyncio.current_task()))
else:
logging_obj._process_hidden_params_and_response_cost(
GenerateContentResponse(**reply),
datetime.datetime(2026, 1, 1),
datetime.datetime(2026, 1, 1),
build_logging_payload=False,
)
return logging_obj.model_call_details["response_cost"]
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us", _UPLIFT_MULTIPLIER)])
async def test_vertex_generate_content_spend_follows_the_deployment_location(
vertex_regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float
) -> None:
expected: Final = (7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * multiplier
recorded: Final = await _logged_vertex_generate_content_cost(location, stream=stream)
assert recorded == pytest.approx(expected), (location, stream)

View file

@ -6,9 +6,12 @@ Ensures backward compatibility after sparse kwargs extraction optimization.
from typing import Final
import httpx
import pytest
from openai import OpenAI
from pydantic import ValidationError
import litellm
from litellm.constants import CONTROL_OPTIONS_KEY
from litellm.litellm_core_utils.get_litellm_params import (
_OPTIONAL_KWARGS_KEYS,
@ -75,10 +78,39 @@ class TestGetBaseModelFromLitellmCallMetadata:
)
assert result == "gpt-4"
def test_returns_non_string_base_model_without_filtering(self):
result = get_base_model_from_litellm_call_metadata({"model_info": {"base_model": 123}})
@pytest.mark.parametrize(
"model_info",
[{"base_model": 123}, {"base_model": ["gpt-4"]}, {"base_model": {"model": "gpt-4"}}],
)
def test_ignores_a_non_string_base_model(self, model_info: object):
assert get_base_model_from_litellm_call_metadata({"model_info": model_info}) is None
assert result == 123
@pytest.mark.parametrize("model_info", ["malformed", ["malformed"], 123])
def test_a_call_with_a_non_object_model_info_fails_before_reaching_the_provider(model_info: object) -> None:
sent: Final[list[httpx.Request]] = [] # mutable-ok: the transport records each outbound request
def record(request: httpx.Request) -> httpx.Response:
sent.append(request)
return httpx.Response(500)
client: Final = OpenAI(
api_key="sk-test",
base_url="https://api.openai.test/v1",
http_client=httpx.Client(transport=httpx.MockTransport(record)),
max_retries=0,
)
with pytest.raises(AttributeError):
litellm.completion(
model="openai/gpt-5.4-mini",
messages=[{"role": "user", "content": "hi"}],
metadata={"model_info": model_info},
client=client,
num_retries=0,
)
assert sent == []
class TestGetLitellmParamsKwargsExtraction:

View file

@ -2066,6 +2066,56 @@ async def test_arealtime_marks_litellm_params_async(monkeypatch):
assert LitellmLogging.is_sync_litellm_request(captured["litellm_params"]) is False
@pytest.mark.asyncio
async def test_aresponses_websocket_turn_reaches_a_custom_logger_through_the_async_hook_only(
monkeypatch: pytest.MonkeyPatch,
):
from litellm.litellm_core_utils import thread_pool_executor
from litellm.responses.main import base_llm_http_handler
class InlineExecutor:
def submit(self, fn, *args, **kwargs):
fn(*args, **kwargs)
sessions: list[LitellmLogging] = []
async def open_session(**kwargs):
sessions.append(kwargs["logging_obj"])
async_logged = asyncio.Event()
sync_call_ids: list[str] = []
async_call_ids: list[str] = []
class CaptureLogger(CustomLogger):
def log_success_event(self, kwargs, response_obj, start_time, end_time):
sync_call_ids.append(kwargs["litellm_call_id"])
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
async_call_ids.append(kwargs["litellm_call_id"])
async_logged.set()
logger = CaptureLogger()
monkeypatch.setattr(litellm, "callbacks", [logger])
monkeypatch.setattr(litellm, "success_callback", ["cache", logger])
monkeypatch.setattr(litellm, "_async_success_callback", [logger])
monkeypatch.setattr(thread_pool_executor, "executor", InlineExecutor())
with patch.object( # test-quality-ok: the provider socket is the seam; how a turn of the session logs is under test
base_llm_http_handler, "async_responses_websocket", AsyncMock(side_effect=open_session)
):
await litellm._aresponses_websocket(model="openai/gpt-4o", websocket=MagicMock(), api_key="sk-test")
turn = sessions[0].copy_for_new_call("ws-turn-2", dt_object.now())
await turn.dispatch_success_handlers(
litellm.ResponsesAPIResponse(
id="resp_turn_2", created_at=0, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[]
),
prefer_async_handlers=True,
)
await asyncio.wait_for(async_logged.wait(), timeout=10)
assert (async_call_ids, sync_call_ids) == (["ws-turn-2"], [])
@pytest.mark.asyncio
async def test_aresponses_websocket_hands_back_the_provider_failure_without_a_success_log(
monkeypatch: pytest.MonkeyPatch,

View file

@ -0,0 +1,57 @@
from typing import Final
import httpx
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
_DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview"
_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai")
class _AzureThatRecordsTheRequest:
def __init__(self) -> None:
self.requests: Final[list[httpx.Request]] = []
def __call__(self, request: httpx.Request) -> httpx.Response:
self.requests.append(request)
return httpx.Response(200, json={"value": "ek_upstream"})
def client(self) -> AsyncHTTPHandler:
return AsyncHTTPHandler(transport=httpx.MockTransport(self))
@pytest.mark.asyncio
@pytest.mark.parametrize("api_base", _API_BASES)
async def test_azure_client_secret_targets_the_ga_v1_surface_even_with_a_dated_api_version(api_base: str) -> None:
azure: Final = _AzureThatRecordsTheRequest()
await litellm.acreate_realtime_client_secret(
model="azure/gpt-realtime-deployment",
api_base=api_base,
api_key="azure-key",
api_version=_DATED_PREVIEW_API_VERSION,
client=azure.client(),
)
(request,) = azure.requests
assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/client_secrets"
assert request.headers.get("api-key") == "azure-key"
@pytest.mark.asyncio
@pytest.mark.parametrize("api_base", _API_BASES)
async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_key_as_bearer(api_base: str) -> None:
azure: Final = _AzureThatRecordsTheRequest()
await litellm.arealtime_calls(
openai_ephemeral_key="ek_upstream",
sdp_body=b"v=0",
model="azure/gpt-realtime-deployment",
api_base=api_base,
api_key="azure-key",
api_version=_DATED_PREVIEW_API_VERSION,
client=azure.client(),
)
(request,) = azure.requests
assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls"
assert request.headers.get("authorization") == "Bearer ek_upstream"
assert "api-key" not in request.headers

View file

@ -1410,6 +1410,26 @@ def test_get_azure_ad_token_with_token_refresh(setup_mocks, monkeypatch):
assert token == "mock-default-token"
def test_get_azure_ad_token_with_token_refresh_reuses_provider_per_scope(setup_mocks):
"""Repeated requests reuse one token provider per scope instead of building a
new credential, and fetching a new token, on every call (#46419)"""
setup_mocks["litellm"].enable_azure_ad_token_refresh = True
default_scope = "https://cognitiveservices.azure.com/.default"
other_scope = "api://other-resource/.default"
tokens = [
get_azure_ad_token(GenericLiteLLMParams()),
get_azure_ad_token(GenericLiteLLMParams()),
get_azure_ad_token(GenericLiteLLMParams(azure_scope=other_scope)),
]
assert tokens == ["mock-default-token"] * 3
assert [c.kwargs for c in setup_mocks["token_provider"].call_args_list] == [
{"azure_scope": default_scope},
{"azure_scope": other_scope},
]
def test_get_azure_ad_token_with_token_refresh_error(setup_mocks):
"""Test get_azure_ad_token with token refresh enabled but raising an error."""
# Reset mocks to ensure clean state

View file

@ -4,7 +4,7 @@ import logging
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, MagicMock, create_autospec, patch
import httpx
import pytest
@ -136,6 +136,77 @@ async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_
assert metadata["original_key"] == "original_value"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("metadata_buckets", "expected_model_group"),
[
({"metadata": {}, "litellm_metadata": {"model_group": "ws-budget-sock"}}, "ws-budget-sock"),
({"metadata": {"model_group": "caller-group"}, "litellm_metadata": {"model_group": "routed"}}, "caller-group"),
({"metadata": {}, "litellm_metadata": {}}, ""),
],
)
async def test_async_post_call_failure_hook_logs_the_routed_model_group_on_the_failure_row(
metadata_buckets: dict[str, dict[str, str]], expected_model_group: str
):
request_data: Final = {
"model": "ws-budget-sock",
"input": "Reply with exactly the word two",
"proxy_server_request": {"request_id": "test_request_id"},
**metadata_buckets,
}
spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook(
request_data=request_data,
original_exception=litellm.RateLimitError(message="crossed budget", llm_provider="", model="ws-budget-sock"),
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
)
row: Final = get_logging_payload(
kwargs=spend_writer.update_database.call_args.kwargs["kwargs"],
response_obj={},
start_time=datetime.now(),
end_time=datetime.now(),
)
assert row["model_group"] == expected_model_group
@pytest.mark.asyncio
@pytest.mark.parametrize(
("metadata_buckets", "expected_tags"),
[
({"metadata": {}, "litellm_metadata": {"tags": ["ws-tag"]}}, ["ws-tag"]),
({"metadata": {"tags": ["caller-tag"]}, "litellm_metadata": {"tags": ["routed-tag"]}}, ["caller-tag"]),
({"metadata": {}, "litellm_metadata": {"tags": "not-a-list"}}, []),
({"metadata": {}, "litellm_metadata": {"tags": ["ws-tag", 1, {"a": 1}]}}, []),
],
)
async def test_async_post_call_failure_hook_logs_the_routed_tags_on_the_failure_row(
metadata_buckets: dict[str, dict[str, object]], expected_tags: list[str]
):
request_data: Final = {
"model": "ws-budget-sock",
"input": "Reply with exactly the word two",
"proxy_server_request": {"request_id": "test_request_id"},
**metadata_buckets,
}
spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook(
request_data=request_data,
original_exception=litellm.RateLimitError(message="crossed budget", llm_provider="", model="ws-budget-sock"),
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
)
row: Final = get_logging_payload(
kwargs=spend_writer.update_database.call_args.kwargs["kwargs"],
response_obj={},
start_time=datetime.now(),
end_time=datetime.now(),
)
assert json.loads(row["request_tags"]) == expected_tags
@pytest.mark.asyncio
async def test_async_post_call_failure_hook_does_not_clobber_guardrail_info_in_metadata():
logger = ProxyDBLogger()
@ -1105,6 +1176,38 @@ async def test_track_cost_callback_skips_when_no_standard_logging_object():
mock_proxy_logging.failed_tracking_alert.assert_not_called()
class _FailedTrackingAlerts:
def __init__(self) -> None:
self.messages: tuple[str, ...] = ()
async def failed_tracking_alert(self, error_message: str, failing_model: str) -> None:
self.messages = (*self.messages, error_message)
@pytest.mark.asyncio
@pytest.mark.parametrize(("call_type", "alerted"), [("_aresponses_websocket", True), ("_arealtime", False)])
async def test_track_cost_callback_alerts_on_a_websocket_turn_with_no_logging_object(call_type: str, alerted: bool):
kwargs: Final = {
"call_type": call_type,
"model": "gpt-5-mini",
"litellm_call_id": "turn-call-id",
"litellm_params": {},
"stream": False,
}
alerts: Final = _FailedTrackingAlerts()
with patch("litellm.proxy.proxy_server.proxy_logging_obj", alerts):
await ProxyDBLogger()._PROXY_track_cost_callback(
kwargs=kwargs,
completion_response=None,
start_time=datetime.now(),
end_time=datetime.now(),
)
await asyncio.sleep(0)
assert any(f"call_type: {call_type}" in message for message in alerts.messages) is alerted
@pytest.mark.asyncio
async def test_track_cost_callback_defers_in_progress_background_interaction(): # test-quality-ok: writing no spend row and raising no alert is the whole observable contract of the deferral path
"""

View file

@ -1,4 +1,6 @@
import asyncio
import json
from collections.abc import Iterator
from datetime import datetime
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -11,6 +13,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.gemini_passthrough_logging_handler import (
GeminiPassthroughLoggingHandler,
)
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
@ -186,7 +190,6 @@ class TestGeminiPassthroughLoggingHandler:
mock_completion_cost.assert_called_once()
# Verify logging object was updated
assert mock_logging_obj.model_call_details["response_cost"] == 0.000045
assert mock_logging_obj.model_call_details["model"] == "gemini-1.5-flash"
assert mock_logging_obj.model_call_details["custom_llm_provider"] == "gemini"
@ -310,8 +313,6 @@ class TestGeminiPassthroughLoggingHandler:
# Assert - The success handler returns None on success (following the pattern from other tests)
assert result is None
# Verify that the logging object has the cost set (from Gemini handler)
assert mock_logging_obj.model_call_details["response_cost"] == 0.000050
assert mock_logging_obj.model_call_details["model"] == "gemini-2.0-flash"
assert mock_logging_obj.model_call_details["custom_llm_provider"] == "gemini"
@ -479,3 +480,114 @@ def test_build_complete_streaming_response_assembles_gemini_sse_chunks():
assert response.choices[0].message.content == "Hello there!"
assert response.choices[0].finish_reason == "stop"
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (10, 8, 18)
_PROBE_MODEL: Final = "gemini-pricing-probe"
_LIST_INPUT_RATE: Final = 1e-06
_LIST_OUTPUT_RATE: Final = 4e-06
_CUSTOM_RATES: Final = {"input_cost_per_token": 2e-09, "output_cost_per_token": 3e-09}
_CUSTOM_PRICED_DEPLOYMENT: Final = "gemini-custom-priced-deployment"
_PROBE_REPLY: Final = {
"candidates": [{"content": {"parts": [{"text": "pong"}], "role": "model"}, "finishReason": "STOP", "index": 0}],
"usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8},
"modelVersion": _PROBE_MODEL,
}
@pytest.fixture
def probe_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
rates: Final = {
"input_cost_per_token": _LIST_INPUT_RATE,
"output_cost_per_token": _LIST_OUTPUT_RATE,
"litellm_provider": "gemini",
"mode": "chat",
}
monkeypatch.setitem(litellm.model_cost, f"gemini/{_PROBE_MODEL}", rates)
monkeypatch.setitem(
litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**_CUSTOM_RATES, "litellm_provider": "gemini", "mode": "chat"}
)
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
def _cost_the_logger_records_for_gemini(*, stream: bool, model_info: dict[str, object] | None) -> float:
method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent"
url_route: Final = f"https://generativelanguage.googleapis.com/v1beta/models/{_PROBE_MODEL}:{method}"
logging_obj: Final = LiteLLMLoggingObj(
model="unknown",
messages=[{"role": "user", "content": "ping"}],
stream=stream,
call_type="pass_through_endpoint",
start_time=datetime(2026, 1, 1),
litellm_call_id="call-1",
function_id="fn-1",
)
logging_obj.update_environment_variables(
litellm_params={"metadata": {"model_info": model_info}} if model_info is not None else {}, optional_params={}
)
if stream:
asyncio.run(
PassThroughStreamingHandler.route_streaming_logging_to_handler(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=PassThroughEndpointLogging(),
url_route=url_route,
request_body={},
endpoint_type=HttpPassThroughEndpointHelpers.get_endpoint_type(url_route),
start_time=datetime(2026, 1, 1),
raw_bytes=[f"data: {json.dumps(_PROBE_REPLY)}\n\n".encode()],
end_time=datetime(2026, 1, 1),
)
)
return logging_obj.model_call_details["response_cost"]
response: Final = httpx.Response(200, json=_PROBE_REPLY)
result: Final = GeminiPassthroughLoggingHandler.gemini_passthrough_handler(
httpx_response=response,
response_body=_PROBE_REPLY,
logging_obj=logging_obj,
url_route=url_route,
result=response.text,
start_time=datetime(2026, 1, 1),
end_time=datetime(2026, 1, 1),
cache_hit=False,
request_body={},
)
logging_obj._process_hidden_params_and_response_cost(
result["result"], datetime(2026, 1, 1), datetime(2026, 1, 1), build_logging_payload=False
)
return logging_obj.model_call_details["response_cost"]
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
@pytest.mark.parametrize(
"model_info, input_rate, output_rate",
[
(None, _LIST_INPUT_RATE, _LIST_OUTPUT_RATE),
(
{"id": _CUSTOM_PRICED_DEPLOYMENT, **_CUSTOM_RATES},
_CUSTOM_RATES["input_cost_per_token"],
_CUSTOM_RATES["output_cost_per_token"],
),
],
ids=["list-price", "custom-price"],
)
def test_gemini_passthrough_spend_matches_whether_or_not_the_call_streams(
probe_rate_card: None,
stream: bool,
model_info: dict[str, object] | None,
input_rate: float,
output_rate: float,
) -> None:
recorded: Final = _cost_the_logger_records_for_gemini(stream=stream, model_info=model_info)
assert recorded == pytest.approx(7 * input_rate + 1 * output_rate), stream
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
@pytest.mark.parametrize("base_model", [123, [f"gemini/{_PROBE_MODEL}"], {"model": f"gemini/{_PROBE_MODEL}"}])
def test_gemini_passthrough_bills_the_list_price_when_the_model_info_base_model_is_not_a_string(
probe_rate_card: None, stream: bool, base_model: object
) -> None:
recorded: Final = _cost_the_logger_records_for_gemini(stream=stream, model_info={"base_model": base_model})
assert recorded == pytest.approx(7 * _LIST_INPUT_RATE + 1 * _LIST_OUTPUT_RATE), stream

View file

@ -1,4 +1,6 @@
import asyncio
import json
from collections.abc import Iterator
from datetime import datetime
from typing import Final
@ -6,11 +8,15 @@ import httpx
import pytest
from pydantic import ValidationError
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import (
VertexPassthroughLoggingHandler,
)
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
from litellm.types.utils import EmbeddingResponse, ModelResponse
_PREDICT_ROUTE = "/v1/projects/p/locations/us-central1/publishers/google/models/text-embedding-004:predict"
@ -97,6 +103,359 @@ def test_interactions_usage_object_is_read_into_prompt_and_completion_tokens():
assert result["kwargs"]["custom_llm_provider"] == "vertex_ai"
_UPLIFT_MODEL: Final = "gemini-regional-uplift-probe"
_UPLIFT_INPUT_RATE: Final = 1e-06
_UPLIFT_OUTPUT_RATE: Final = 4e-06
_UPLIFT_MULTIPLIER: Final = 1.1
@pytest.fixture
def regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
rates: Final = {
"input_cost_per_token": _UPLIFT_INPUT_RATE,
"output_cost_per_token": _UPLIFT_OUTPUT_RATE,
"litellm_provider": "vertex_ai-language-models",
"mode": "chat",
}
monkeypatch.setitem(litellm.model_cost, _UPLIFT_MODEL, rates)
monkeypatch.setattr(litellm, "vertex_language_models", {*litellm.vertex_language_models, _UPLIFT_MODEL})
monkeypatch.setitem(
litellm.model_cost,
f"vertex_ai/{_UPLIFT_MODEL}",
{**rates, "regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER},
)
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
def _generate_content_payload() -> dict[str, object]:
return {
"candidates": [{"content": {"parts": [{"text": "pong"}], "role": "model"}, "finishReason": "STOP", "index": 0}],
"usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8},
"modelVersion": _UPLIFT_MODEL,
}
def _cost_the_logger_records_for(
url_route: str,
*,
stream: bool,
reply: dict[str, object],
stream_events: list[dict[str, object]],
model_info: dict[str, object] | None = None,
route_provider: str | None = None,
base_model: object = None,
) -> float:
logging_obj: Final = Logging(
model="unknown",
messages=[{"role": "user", "content": "ping"}],
stream=stream,
call_type="pass_through_endpoint",
start_time=datetime(2026, 1, 1),
litellm_call_id="call-1",
function_id="fn-1",
)
logging_obj.update_environment_variables(
litellm_params={"metadata": {"model_info": model_info}} if model_info is not None else {}, optional_params={}
)
if route_provider is not None:
logging_obj.model_call_details["custom_llm_provider"] = route_provider
if base_model is not None:
logging_obj.model_call_details["litellm_params"] = {"base_model": base_model}
if stream:
asyncio.run(
PassThroughStreamingHandler.route_streaming_logging_to_handler(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=PassThroughEndpointLogging(),
url_route=url_route,
request_body={},
endpoint_type=EndpointType.VERTEX_AI,
start_time=datetime(2026, 1, 1),
raw_bytes=[f"data: {json.dumps(event)}\n\n".encode() for event in stream_events],
end_time=datetime(2026, 1, 1),
)
)
return logging_obj.model_call_details["response_cost"]
response: Final = httpx.Response(200, json=reply)
result: Final = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
httpx_response=response,
logging_obj=logging_obj,
url_route=url_route,
result=response.text,
start_time=datetime(2026, 1, 1),
end_time=datetime(2026, 1, 1),
cache_hit=False,
request_body={},
)
logging_obj._process_hidden_params_and_response_cost(
result["result"], datetime(2026, 1, 1), datetime(2026, 1, 1), build_logging_payload=False
)
return logging_obj.model_call_details["response_cost"]
def _cost_the_logger_records(location: str, *, stream: bool, model_info: dict[str, object] | None = None) -> float:
method: Final = "streamGenerateContent" if stream else "generateContent"
url_route: Final = (
f"https://aiplatform.googleapis.com/v1/projects/p/locations/{location}"
f"/publishers/google/models/{_UPLIFT_MODEL}:{method}"
)
return _cost_the_logger_records_for(
url_route,
stream=stream,
reply=_generate_content_payload(),
stream_events=[_generate_content_payload()],
model_info=model_info,
)
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us", _UPLIFT_MULTIPLIER)])
def test_generate_content_spend_follows_the_vertex_location(
regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float
) -> None:
expected: Final = (7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * multiplier
assert _cost_the_logger_records(location, stream=stream) == pytest.approx(expected), (location, stream)
@pytest.mark.parametrize(
"gemini_api_base", ["https://generativelanguage.googleapis.com", "https://llm-egress.internal.example/google"]
)
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
def test_gemini_route_spend_has_no_vertex_uplift_whatever_host_serves_it(
regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, gemini_api_base: str, stream: bool
) -> None:
monkeypatch.setitem(
litellm.model_cost,
f"gemini/{_UPLIFT_MODEL}",
{
"input_cost_per_token": _UPLIFT_INPUT_RATE,
"output_cost_per_token": _UPLIFT_OUTPUT_RATE,
"litellm_provider": "gemini",
"mode": "chat",
},
)
method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent"
recorded: Final = _cost_the_logger_records_for(
f"{gemini_api_base}/v1beta/models/{_UPLIFT_MODEL}:{method}",
stream=stream,
reply=_generate_content_payload(),
stream_events=[_generate_content_payload()],
route_provider="gemini",
)
assert recorded == pytest.approx(7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE), (gemini_api_base, stream)
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
def test_a_vertex_host_that_only_ends_with_the_gemini_host_name_keeps_the_vertex_uplift(
regional_uplift_rate_card: None, stream: bool
) -> None:
method: Final = "streamGenerateContent" if stream else "generateContent"
recorded: Final = _cost_the_logger_records_for(
"https://vertex-generativelanguage.googleapis.com/v1/projects/p/locations/us/publishers/google/models/"
f"{_UPLIFT_MODEL}:{method}",
stream=stream,
reply=_generate_content_payload(),
stream_events=[_generate_content_payload()],
)
assert recorded == pytest.approx((7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * _UPLIFT_MULTIPLIER), stream
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
def test_a_subdomain_of_the_gemini_host_bills_the_gemini_price(
regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool
) -> None:
monkeypatch.setitem(
litellm.model_cost,
f"gemini/{_UPLIFT_MODEL}",
{
"input_cost_per_token": _UPLIFT_INPUT_RATE,
"output_cost_per_token": _UPLIFT_OUTPUT_RATE,
"litellm_provider": "gemini",
"mode": "chat",
},
)
method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent"
recorded: Final = _cost_the_logger_records_for(
f"https://eu.generativelanguage.googleapis.com/v1beta/models/{_UPLIFT_MODEL}:{method}",
stream=stream,
reply=_generate_content_payload(),
stream_events=[_generate_content_payload()],
)
assert recorded == pytest.approx(7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE), stream
@pytest.mark.parametrize("value", [123, ["gemini-2.5-flash-lite"], {"model": "gemini-2.5-flash-lite"}])
@pytest.mark.parametrize("carrier", ["root", "model_info"])
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
def test_a_non_string_base_model_bills_the_list_price(
regional_uplift_rate_card: None, stream: bool, carrier: str, value: object
) -> None:
method: Final = "streamGenerateContent" if stream else "generateContent"
recorded: Final = _cost_the_logger_records_for(
f"https://aiplatform.googleapis.com/v1/projects/p/locations/us/publishers/google/models/{_UPLIFT_MODEL}:{method}",
stream=stream,
reply=_generate_content_payload(),
stream_events=[_generate_content_payload()],
model_info={"base_model": value} if carrier == "model_info" else None,
base_model=value if carrier == "root" else None,
)
assert recorded == pytest.approx((7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * _UPLIFT_MULTIPLIER), stream
_CUSTOM_PRICED_DEPLOYMENT: Final = "vertex-custom-priced-deployment"
_CUSTOM_INPUT_RATE: Final = 2e-09
_CUSTOM_OUTPUT_RATE: Final = 3e-09
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
@pytest.mark.parametrize("location", ["global", "us"])
def test_generate_content_spend_uses_the_deployment_custom_price(
regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool, location: str
) -> None:
custom_rates: Final = {"input_cost_per_token": _CUSTOM_INPUT_RATE, "output_cost_per_token": _CUSTOM_OUTPUT_RATE}
monkeypatch.setitem(
litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"}
)
expected: Final = 7 * _CUSTOM_INPUT_RATE + 1 * _CUSTOM_OUTPUT_RATE
recorded: Final = _cost_the_logger_records(
location, stream=stream, model_info={"id": _CUSTOM_PRICED_DEPLOYMENT, **custom_rates}
)
assert recorded == pytest.approx(expected), (location, stream)
_CLAUDE_UPLIFT_MODEL: Final = "claude-regional-uplift-probe"
_CLAUDE_INPUT_RATE: Final = 3e-06
_CLAUDE_OUTPUT_RATE: Final = 1.5e-05
@pytest.fixture
def claude_regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
rates: Final = {
"input_cost_per_token": _CLAUDE_INPUT_RATE,
"output_cost_per_token": _CLAUDE_OUTPUT_RATE,
"mode": "chat",
}
monkeypatch.setitem(litellm.model_cost, _CLAUDE_UPLIFT_MODEL, {**rates, "litellm_provider": "anthropic"})
monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, _CLAUDE_UPLIFT_MODEL})
monkeypatch.setitem(
litellm.model_cost,
f"vertex_ai/{_CLAUDE_UPLIFT_MODEL}",
{
**rates,
"litellm_provider": "vertex_ai-anthropic_models",
"regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER,
},
)
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
def _claude_message() -> dict[str, object]:
return {
"id": "msg_probe",
"type": "message",
"role": "assistant",
"model": _CLAUDE_UPLIFT_MODEL,
"content": [{"type": "text", "text": "pong"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 7, "output_tokens": 1},
}
def _claude_stream_events() -> list[dict[str, object]]:
message_start: Final = {**_claude_message(), "content": [], "usage": {"input_tokens": 7, "output_tokens": 0}}
return [
{"type": "message_start", "message": message_start},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}},
{"type": "message_stop"},
]
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us-east5", _UPLIFT_MULTIPLIER)])
def test_claude_raw_predict_spend_follows_the_vertex_location(
claude_regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float
) -> None:
method: Final = "streamRawPredict" if stream else "rawPredict"
url_route: Final = (
f"https://aiplatform.googleapis.com/v1/projects/p/locations/{location}"
f"/publishers/anthropic/models/{_CLAUDE_UPLIFT_MODEL}:{method}"
)
expected: Final = (7 * _CLAUDE_INPUT_RATE + 1 * _CLAUDE_OUTPUT_RATE) * multiplier
recorded: Final = _cost_the_logger_records_for(
url_route, stream=stream, reply=_claude_message(), stream_events=_claude_stream_events()
)
assert recorded == pytest.approx(expected), (location, stream)
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
def test_claude_raw_predict_spend_uses_the_deployment_custom_price(
claude_regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool
) -> None:
custom_rates: Final = {"input_cost_per_token": _CUSTOM_INPUT_RATE, "output_cost_per_token": _CUSTOM_OUTPUT_RATE}
monkeypatch.setitem(
litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"}
)
method: Final = "streamRawPredict" if stream else "rawPredict"
url_route: Final = (
"https://aiplatform.googleapis.com/v1/projects/p/locations/us-east5"
f"/publishers/anthropic/models/{_CLAUDE_UPLIFT_MODEL}:{method}"
)
recorded: Final = _cost_the_logger_records_for(
url_route,
stream=stream,
reply=_claude_message(),
stream_events=_claude_stream_events(),
model_info={"id": _CUSTOM_PRICED_DEPLOYMENT, **custom_rates},
)
assert recorded == pytest.approx(7 * _CUSTOM_INPUT_RATE + 1 * _CUSTOM_OUTPUT_RATE), stream
def test_a_claude_stream_through_a_configured_vertex_gateway_with_a_short_path_bills_the_vertex_price(
claude_regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setitem(
litellm.model_cost,
f"vertex_ai/{_CLAUDE_UPLIFT_MODEL}",
{
"input_cost_per_token": _CLAUDE_INPUT_RATE,
"output_cost_per_token": _CLAUDE_OUTPUT_RATE,
"litellm_provider": "vertex_ai-anthropic_models",
"mode": "chat",
},
)
litellm.get_model_info.cache_clear()
recorded: Final = _cost_the_logger_records_for(
f"https://vertex-gateway.internal.example/vxc/models/{_CLAUDE_UPLIFT_MODEL}:streamRawPredict",
stream=True,
reply=_claude_message(),
stream_events=_claude_stream_events(),
)
assert recorded == pytest.approx(7 * _CLAUDE_INPUT_RATE + 1 * _CLAUDE_OUTPUT_RATE)
def test_build_complete_streaming_response_assembles_stream_generate_content_sse_chunks():
logging_obj: Final = Logging(
model="gemini-2.5-flash",

View file

@ -8452,7 +8452,9 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_PROXY_SERVER_REQUEST: Final = TypeAdapter(dict[str, object])
def _split_pass_through_body(body: str) -> _PassThroughSplit:
def _split_pass_through_body(
body: str, user_api_key_dict: UserAPIKeyAuth = UserAPIKeyAuth(api_key="hashed-key")
) -> _PassThroughSplit:
mock_request: Final = MagicMock(spec=Request)
mock_request.method = "POST"
mock_request.url = httpx.URL("http://0.0.0.0:4000/gemini/v1beta/models/gemini-2.5-flash:generateContent")
@ -8462,7 +8464,7 @@ def _split_pass_through_body(body: str) -> _PassThroughSplit:
init_kwargs_for_pass_through_endpoint: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # untyped legacy helper
kwargs: Final = init_kwargs_for_pass_through_endpoint(
request=mock_request,
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
user_api_key_dict=user_api_key_dict,
passthrough_logging_payload=MagicMock(),
logging_obj=MagicMock(),
_parsed_body=json.loads(body),
@ -8500,6 +8502,7 @@ def test_passthrough_moves_every_litellm_owned_key_from_the_forwarded_body_into_
assert split.forwarded_body == json.loads(GEMINI_BODY)
PRICING_MODEL_OVERRIDE_NAMES: Final = frozenset(("base_model",))
PROXY_STAMPED_NAMES: Final = frozenset(
(
"proxy_server_request",
@ -8515,7 +8518,12 @@ PROXY_STAMPED_NAMES: Final = frozenset(
@pytest.mark.parametrize(
"name",
sorted(frozenset(litellm.all_litellm_params) - frozenset(("metadata", "litellm_metadata")) - PROXY_STAMPED_NAMES),
sorted(
frozenset(litellm.all_litellm_params)
- frozenset(("metadata", "litellm_metadata"))
- PROXY_STAMPED_NAMES
- PRICING_MODEL_OVERRIDE_NAMES
),
)
def test_passthrough_keeps_each_registered_litellm_owned_name_out_of_the_forwarded_body(name: str) -> None:
split: Final = _split_pass_through_body(json.dumps({name: "owned", **json.loads(GEMINI_BODY)}))
@ -8525,6 +8533,41 @@ def test_passthrough_keeps_each_registered_litellm_owned_name_out_of_the_forward
assert split.forwarded_body == json.loads(GEMINI_BODY)
@pytest.mark.parametrize("allows_client_pricing", [True, False], ids=["key-allows-client-pricing", "plain-key"])
def test_passthrough_prices_with_a_client_base_model_only_for_keys_that_allow_client_pricing(
allows_client_pricing: bool,
) -> None:
key: Final = UserAPIKeyAuth(
api_key="hashed-key", metadata={"allow_client_pricing_override": True} if allows_client_pricing else {}
)
split: Final = _split_pass_through_body(
json.dumps({"base_model": "vertex_ai/gemini-2.5-flash-lite", **json.loads(GEMINI_BODY)}), key
)
expected: Final = "vertex_ai/gemini-2.5-flash-lite" if allows_client_pricing else None
assert split.litellm_params.get("base_model") == expected
assert split.forwarded_body == json.loads(GEMINI_BODY)
@pytest.mark.parametrize("model_info", ["malformed", ["malformed"], 123])
def test_passthrough_drops_a_client_model_info_that_is_not_an_object(model_info: object) -> None:
key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"allow_client_pricing_override": True})
plain_metadata: Final = _metadata_of(_split_pass_through_body(json.dumps({"metadata": {"tag": "a"}}), key))
split: Final = _split_pass_through_body(json.dumps({"metadata": {"tag": "a", "model_info": model_info}}), key)
assert _metadata_of(split) == plain_metadata
def test_passthrough_keeps_a_client_model_info_object_for_keys_that_allow_client_pricing() -> None:
key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"allow_client_pricing_override": True})
split: Final = _split_pass_through_body(json.dumps({"metadata": {"model_info": {"base_model": "gemini/x"}}}), key)
assert _metadata_of(split)["model_info"] == {"base_model": "gemini/x"}
@pytest.mark.parametrize("name", sorted(PROXY_STAMPED_NAMES))
def test_passthrough_drops_a_client_supplied_proxy_stamped_name(name: str) -> None:
split: Final = _split_pass_through_body(json.dumps({name: {"forged": "by-client"}, **json.loads(GEMINI_BODY)}))

View file

@ -2,6 +2,7 @@ import json
import logging
from collections.abc import Iterator
from datetime import datetime
from typing import Final
from unittest.mock import MagicMock
import pytest
@ -13,6 +14,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrou
VertexPassthroughLoggingHandler,
)
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamContext,
PassThroughStreamingHandler,
)
from litellm.proxy.pass_through_endpoints.success_handler import (
@ -235,3 +237,40 @@ async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(
partial_usage = logging_obj.record_partial_usage_for_failure.call_args.kwargs["usage"]
assert partial_usage.completion_tokens > 100_000
assert_loop_stayed_free(took, lags)
@pytest.mark.asyncio
async def test_interrupted_vertex_stream_bills_the_deployment_custom_price(monkeypatch: pytest.MonkeyPatch) -> None:
deployment_id: Final = "vertex-custom-priced-deployment"
custom_rates: Final = {"input_cost_per_token": 2e-09, "output_cost_per_token": 3e-09}
monkeypatch.setitem(
litellm.model_cost, deployment_id, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"}
)
logging_obj: Final = LiteLLMLoggingObj(
model="unknown",
messages=[{"role": "user", "content": "ping"}],
stream=True,
call_type="pass_through_endpoint",
start_time=datetime.now(),
litellm_call_id="interrupted-stream",
function_id="interrupted-stream",
)
logging_obj.optional_params = {}
logging_obj.litellm_params = {"metadata": {"model_info": {"id": deployment_id, **custom_rates}}}
url_route: Final = (
f"https://aiplatform.googleapis.com/v1/projects/p/locations/us"
f"/publishers/google/models/{MODEL}:streamGenerateContent?alt=sse"
)
await PassThroughStreamingHandler.schedule_stream_failure_logging(
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.VERTEX_AI,
request_body={},
raw_bytes=[chunk.encode("utf-8") for chunk in _chunks()],
exception=RuntimeError("upstream closed the stream"),
stream_context=PassThroughStreamContext(PassThroughEndpointLogging(), url_route, datetime.now()),
)
recorded: Final = logging_obj.model_call_details["response_cost"]
await GLOBAL_LOGGING_WORKER.flush()
assert recorded == pytest.approx(PROMPT_TOKENS * 2e-09 + COMPLETION_TOKENS * 3e-09)

View file

@ -972,6 +972,54 @@ async def test_proxy_startup_event_honors_disable_model_info_refresh(
)
@pytest.mark.asyncio
async def test_proxy_shutdown_commits_spend_before_waiting_on_a_slow_logging_task_and_again_after_it() -> None:
"""A worker killed while shutdown waits on a slow logging task keeps the spend it had already counted,
and a worker that finishes the wait also commits the spend that task adds."""
from litellm.responses import streaming_iterator as streaming_module
task_may_finish: Final = asyncio.Event()
task_finished: Final = asyncio.Event()
flushes: Final[list[tuple[str, bool]]] = [] # mutable-ok: records what each shutdown flush saw, which is the assertion
async def slow_logging_task() -> None:
await task_may_finish.wait()
task_finished.set()
def recording_flush(name: str) -> Callable[[], Awaitable[None]]:
async def flush() -> None:
flushes.append((name, task_finished.is_set()))
task_may_finish.set()
return flush
scheduler: Final = AsyncIOScheduler()
clean_env: Final = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | {
"LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true"
}
with (
patch.dict(os.environ, clean_env, clear=True),
patch.object(ps, "scheduler", scheduler),
patch.object(ps, "flush_spend_counters_on_shutdown", recording_flush("spend counters")),
patch.object(ps, "_flush_spend_logs_queue_on_shutdown", recording_flush("spend logs")),
):
try:
async with proxy_startup_event(app=None):
streaming_module.spawn_logging_task(
asyncio.get_running_loop(), slow_logging_task(), task_name="slow spend row"
)
finally:
if scheduler.running:
scheduler.shutdown(wait=False)
assert flushes == [
("spend counters", False),
("spend logs", False),
("spend counters", True),
("spend logs", True),
]
def test_otel_global_provider_published_after_callback_init():
"""The OTel V2 global-provider publish must run after callback
initialization in ``proxy_startup_event``.

View file

@ -9,17 +9,20 @@ import json
import unittest
from collections.abc import Mapping
from typing import Any, Final, Literal, cast
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, MagicMock, call, patch
import httpx
import pytest
import respx
from fastapi import HTTPException
from fastapi.testclient import TestClient
from httpx import Response
import litellm
from litellm.proxy._types import ProxyException
from litellm.proxy.proxy_server import app
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.router import RouterErrors
from tests._master_key import MASTER_KEY
@ -1648,7 +1651,7 @@ class TestManagedResponsesWSFirstMessage:
processed: list = []
async def fake_process(msg: str) -> None:
async def fake_process(msg: str, turn_admission: object = None) -> None:
processed.append(msg)
handler = ManagedResponsesWebSocketHandler(
@ -1693,30 +1696,32 @@ class TestResponsesWSStreamingFirstMessage:
backend_ws.send.assert_awaited_once_with(first)
@pytest.mark.parametrize(
("failure", "expected_status", "expected_type"),
[
(ValueError("No deployments available - crossed budget: Exceeded budget for deployment"), 429, "rate_limit_exceeded"),
(litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error"),
(RuntimeError("boom"), 500, "server_error"),
(ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error"),
(ProxyException(message="Authentication Error", type="auth_error", param=None, code=401), 401, "invalid_request_error"),
(HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error"),
],
ids=["crossed-budget", "bad-request", "unexpected", "tag-routing", "proxy-auth", "http-exception"],
)
def test_responses_ws_failure_frame_uses_the_http_status_for_the_same_failure(
failure: Exception, expected_status: int, expected_type: str
):
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame
from litellm.proxy.utils import handle_exception_on_proxy
frame: Final = json.loads(_responses_ws_failure_frame(failure))
assert frame["status"] == expected_status
assert frame["error"]["type"] == expected_type
assert frame["error"]["message"] == handle_exception_on_proxy(failure).message
class TestWSSessionCostTracking:
@pytest.mark.asyncio
async def test_router_budget_limiter_skips_aresponses_websocket_call_type(self):
"""
RouterBudgetLimiting.async_log_success_event must not raise when
call_type='_aresponses_websocket', even when standard_logging_object is None.
Per-turn costs are tracked by individual aresponses calls inside the session;
the outer session wrapper fires with result=None.
"""
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
limiter = RouterBudgetLimiting.__new__(RouterBudgetLimiting)
kwargs = {
"call_type": "_aresponses_websocket",
"standard_logging_object": None,
"litellm_params": {"custom_llm_provider": "vertex_ai"},
}
await limiter.async_log_success_event(
kwargs=kwargs,
response_obj=None,
start_time=None,
end_time=None,
)
@pytest.mark.asyncio
async def test_router_budget_limiter_skips_arealtime_call_type(self):
"""Same guard applies to _arealtime WS session wrappers."""
@ -1996,7 +2001,7 @@ class TestResponsesWSFirstFrameModelAuth:
"standard_logging_guardrail_information": [{"guardrail_name": "admission"}],
},
}
guard = _responses_ws_frame_guard(admitted, MagicMock(), proxy_logging_obj)
guard = _responses_ws_frame_guard(admitted, MagicMock(), proxy_logging_obj, MagicMock())
first = await guard(json.dumps({"type": "response.create", "model": "alias", "input": "hi"}))
second = await guard(json.dumps({"type": "response.create", "input": "again"}))
@ -2027,7 +2032,7 @@ class TestResponsesWSFirstFrameModelAuth:
rate_limiter.async_post_call_failure_hook = AsyncMock()
user_api_key_dict = MagicMock()
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, user_api_key_dict, proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, user_api_key_dict, proxy_logging_obj, MagicMock()
)
with (
@ -2059,7 +2064,7 @@ class TestResponsesWSFirstFrameModelAuth:
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
tools = [{"type": "function", "name": "a"}, {"type": "function", "name": "b"}]
@ -2078,7 +2083,7 @@ class TestResponsesWSFirstFrameModelAuth:
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
metadata = {
"user_api_key_metadata": {"disable_global_guardrails": True},
@ -2109,6 +2114,7 @@ class TestResponsesWSFirstFrameModelAuth:
{"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}},
MagicMock(),
proxy_logging_obj,
MagicMock(),
)
await guard(json.dumps(frame))
@ -2131,6 +2137,7 @@ class TestResponsesWSFirstFrameModelAuth:
{"model": "gpt-4o-mini", "litellm_metadata": {"litellm_parent_otel_span": span}},
MagicMock(),
proxy_logging_obj,
MagicMock(),
)
guarded = await guard(json.dumps({"type": "response.create", "input": "hi"}))
@ -2156,7 +2163,7 @@ class TestResponsesWSFirstFrameModelAuth:
spend_logger = MagicMock(spec=ProxyDBLogger)
spend_logger.async_post_call_failure_hook = AsyncMock()
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
with (
@ -2189,7 +2196,7 @@ class TestResponsesWSFirstFrameModelAuth:
spend_logger = MagicMock(spec=ProxyDBLogger)
spend_logger.async_post_call_failure_hook = AsyncMock()
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
body = {
"input": "forbidden",
@ -2215,7 +2222,7 @@ class TestResponsesWSFirstFrameModelAuth:
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
body = {"instructions": "SECRET system prompt", "input": "hi", "store": False}
frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body}
@ -2240,7 +2247,7 @@ class TestResponsesWSFirstFrameModelAuth:
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
with patch.object(litellm, "callbacks", []):
@ -2264,7 +2271,7 @@ class TestResponsesWSFirstFrameModelAuth:
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
with patch.object(litellm, "callbacks", []), pytest.raises(ValueError, match="both at the top level"):
@ -2283,7 +2290,7 @@ class TestResponsesWSFirstFrameModelAuth:
spend_logger = MagicMock(spec=ProxyDBLogger)
spend_logger.async_post_call_failure_hook = AsyncMock(side_effect=RuntimeError("db down"))
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
)
with patch.object(litellm, "callbacks", [spend_logger]), pytest.raises(ValueError, match="blocked"):
@ -2360,6 +2367,345 @@ class TestResponsesWSFirstFrameModelAuth:
assert booked["user_api_key_dict"] is user_api_key_dict
assert booked["request_data"]["model"] == "gpt-4o-mini"
@pytest.mark.asyncio
@pytest.mark.parametrize("ending", ["refusal", "crash", "session_limit"])
async def test_close_time_failure_row_bills_the_guardrails_of_the_turns_the_deployment_refused(self, ending: str):
from litellm.exceptions import RateLimitError
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
ws = MagicMock()
ws.headers = {}
ws.query_params = {}
ws.scope = {"headers": []}
ws.url = "ws://testserver/v1/responses"
ws.accept = AsyncMock()
ws.receive_text = AsyncMock(
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
)
ws.close = AsyncMock()
processor = MagicMock()
processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {"model_group": "gpt-4o-mini"}}, MagicMock())
)
guardrail_costs = {"two": 0.2, "three": 0.3, "four": 0.4}
refusal = RateLimitError(message="crossed budget", llm_provider="", model="")
later_refusal = RateLimitError(message="still over budget", llm_provider="", model="")
crash = RuntimeError("relay crashed")
async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only):
recorded = [{"guardrail_cost": guardrail_costs[data["input"]]}]
return {
**data,
"litellm_metadata": {**data["litellm_metadata"], "standard_logging_guardrail_information": recorded},
}
def route_request(data, **kwargs):
async def relay():
for turn in ("two", "three", "four"):
guarded = await data["responses_ws_frame_guard"](
json.dumps({"type": "response.create", "input": turn})
)
if turn != "three":
guarded.bill_refusal(refusal if turn == "two" else later_refusal)
if ending == "crash":
raise crash
if ending == "session_limit":
await asyncio.Event().wait()
return refusal
return relay()
row_written = asyncio.Event()
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=lambda **kwargs: row_written.set())
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None)
with (
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
new_callable=AsyncMock,
),
patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
return_value=processor,
),
patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome
"litellm.proxy.route_llm_request.route_request",
new_callable=AsyncMock,
side_effect=route_request,
),
patch( # test-quality-ok: a short session limit stands in for the configured one so the cap fires mid-session
"litellm.proxy.response_api_endpoints.endpoints._resolve_responses_ws_session_limit_seconds",
return_value=0.05 if ending == "session_limit" else 3600.0,
),
patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row
"litellm.proxy.proxy_server.proxy_logging_obj",
proxy_logging_obj,
),
):
await responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock())
await asyncio.wait_for(row_written.wait(), timeout=1)
proxy_logging_obj.post_call_failure_hook.assert_awaited_once()
proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once()
booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs
assert booked["original_exception"] is (crash if ending == "crash" else refusal)
assert booked["request_data"]["litellm_metadata"] == {
"model_group": "gpt-4o-mini",
"standard_logging_guardrail_information": [{"guardrail_cost": 0.2}, {"guardrail_cost": 0.4}],
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"relay_outcome",
["still_relaying", "already_returned", "returns_despite_the_cancel", "crashes_despite_the_cancel"],
)
async def test_session_limit_frees_the_slot_at_once_and_never_cuts_the_failure_row(self, relay_outcome: str):
from litellm.exceptions import RateLimitError
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
ws = MagicMock()
ws.headers = {}
ws.query_params = {}
ws.scope = {"headers": []}
ws.url = "ws://testserver/v1/responses"
ws.accept = AsyncMock()
ws.receive_text = AsyncMock(
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
)
async def close_over_the_network(**kwargs):
await asyncio.sleep(0)
ws.close = AsyncMock(side_effect=close_over_the_network)
processor = MagicMock()
processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
)
refusal = RateLimitError(message="crossed budget", llm_provider="", model="")
crash = RuntimeError("relay crashed")
row_may_finish = asyncio.Event()
row_written = asyncio.Event()
async def slow_failure_hook(**kwargs):
await row_may_finish.wait()
row_written.set()
def route_request(data, **kwargs):
async def relay():
guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"}))
guarded.bill_refusal(refusal)
if relay_outcome == "still_relaying":
await asyncio.Event().wait()
if relay_outcome in ("returns_despite_the_cancel", "crashes_despite_the_cancel"):
with contextlib.suppress(asyncio.CancelledError):
await asyncio.Event().wait()
if relay_outcome == "crashes_despite_the_cancel":
raise crash
return refusal
return relay()
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(
side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data
)
proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=slow_failure_hook)
async def release_over_the_network(user_api_key_dict):
await asyncio.sleep(0.01)
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(side_effect=release_over_the_network)
with (
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
new_callable=AsyncMock,
),
patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
return_value=processor,
),
patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome
"litellm.proxy.route_llm_request.route_request",
new_callable=AsyncMock,
side_effect=route_request,
),
patch( # test-quality-ok: a short session limit stands in for the configured one so the cap fires mid-session
"litellm.proxy.response_api_endpoints.endpoints._resolve_responses_ws_session_limit_seconds",
return_value=0.05,
),
patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row
"litellm.proxy.proxy_server.proxy_logging_obj",
proxy_logging_obj,
),
):
endpoint = asyncio.ensure_future(
responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock())
)
finished, _ = await asyncio.wait({endpoint}, timeout=1)
assert endpoint in finished
limit_close = call(code=1000, reason="Session duration limit reached")
crash_close = call(code=1011, reason="Internal server error")
crashed = relay_outcome == "crashes_despite_the_cancel"
assert ws.close.await_args_list == ([limit_close, crash_close] if crashed else [limit_close])
proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once()
assert not row_written.is_set()
row_may_finish.set()
await asyncio.wait_for(row_written.wait(), timeout=1)
assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"] is (crash if crashed else refusal)
@pytest.mark.asyncio
@pytest.mark.parametrize("ending", ["client_closed", "crash"])
async def test_a_closed_socket_frees_its_slot_before_its_failure_row_and_waits_for_the_row(self, ending: str):
from litellm.exceptions import RateLimitError
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
ws = MagicMock()
ws.headers = {}
ws.query_params = {}
ws.scope = {"headers": []}
ws.url = "ws://testserver/v1/responses"
ws.accept = AsyncMock()
ws.receive_text = AsyncMock(
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
)
ws.close = AsyncMock()
processor = MagicMock()
processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
)
refusal = RateLimitError(message="crossed budget", llm_provider="", model="")
crash = RuntimeError("relay crashed")
row_started = asyncio.Event()
row_may_finish = asyncio.Event()
row_written = asyncio.Event()
async def slow_failure_hook(**kwargs):
row_started.set()
await row_may_finish.wait()
row_written.set()
def route_request(data, **kwargs):
async def relay():
guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"}))
guarded.bill_refusal(refusal)
if ending == "crash":
raise crash
return refusal
return relay()
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(
side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data
)
proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=slow_failure_hook)
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None)
with (
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
new_callable=AsyncMock,
),
patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
return_value=processor,
),
patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome
"litellm.proxy.route_llm_request.route_request",
new_callable=AsyncMock,
side_effect=route_request,
),
patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row
"litellm.proxy.proxy_server.proxy_logging_obj",
proxy_logging_obj,
),
):
endpoint = asyncio.ensure_future(
responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock())
)
await asyncio.wait_for(row_started.wait(), timeout=1)
proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once()
finished_early, _ = await asyncio.wait({endpoint}, timeout=0.05)
assert endpoint not in finished_early
row_may_finish.set()
finished, _ = await asyncio.wait({endpoint}, timeout=1)
assert endpoint in finished
assert row_written.is_set()
booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs
assert booked["original_exception"] is (crash if ending == "crash" else refusal)
@pytest.mark.asyncio
async def test_a_crashed_socket_whose_failure_row_cannot_be_built_still_gets_its_error_frame_and_close(self):
from litellm.exceptions import RateLimitError
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
ws = MagicMock()
ws.headers = {}
ws.query_params = {}
ws.scope = {"headers": []}
ws.url = "ws://testserver/v1/responses"
ws.accept = AsyncMock()
ws.receive_text = AsyncMock(
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
)
sent = []
closes = []
ws.send_text = AsyncMock(side_effect=sent.append)
ws.close = AsyncMock(side_effect=lambda code, reason: closes.append((code, reason)))
processor = MagicMock()
processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
)
refusal = RateLimitError(message="crossed budget", llm_provider="", model="")
def route_request(data, **kwargs):
async def relay():
guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"}))
guarded.bill_refusal(refusal)
data["litellm_metadata"]["standard_logging_guardrail_information"] = "written by a broken callback"
raise RuntimeError("relay crashed")
return relay()
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(
side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: {
**data,
"litellm_metadata": {"standard_logging_guardrail_information": [{"guardrail_name": "turn-check"}]},
}
)
proxy_logging_obj.post_call_failure_hook = AsyncMock()
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None)
with (
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
new_callable=AsyncMock,
),
patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
return_value=processor,
),
patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome
"litellm.proxy.route_llm_request.route_request",
new_callable=AsyncMock,
side_effect=route_request,
),
patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row
"litellm.proxy.proxy_server.proxy_logging_obj",
proxy_logging_obj,
),
):
await asyncio.wait_for(
responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()), timeout=1
)
assert [json.loads(frame)["error"]["message"] for frame in sent] == ["relay crashed"]
assert closes == [(1011, "Internal server error")]
@pytest.mark.asyncio
async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection(self):
from litellm.proxy.response_api_endpoints.endpoints import (

View file

@ -235,6 +235,29 @@ async def test_proxy_shutdown_closes_admin_connector_when_drain_is_cancelled(
# Your bearer token
@pytest.mark.asyncio
async def test_shutdown_commits_spend_a_logging_task_queued_while_the_first_commit_was_writing():
from litellm.responses.streaming_iterator import spawn_logging_task
queued: list[str] = []
committed: list[str] = []
async def commit_spend() -> None:
committed.extend(queued)
queued.clear()
await asyncio.sleep(0.05)
async def failure_row() -> None:
await asyncio.sleep(0.01)
queued.append("failure row spend")
spawn_logging_task(asyncio.get_running_loop(), failure_row(), task_name="failure row")
await litellm.proxy.proxy_server._flush_spend_around_logging_drain_on_shutdown(flush_spend=commit_spend)
assert committed == ["failure row spend"]
token = MASTER_KEY
headers = {"Authorization": f"Bearer {token}"}

View file

@ -7,6 +7,7 @@ Tests that:
3. Providers without native websocket support use ManagedResponsesWebSocketHandler
"""
import asyncio
import json
from collections.abc import Mapping
from typing import Final
@ -1331,7 +1332,7 @@ class TestWebSocketProjectQuotaEnforcement:
callback, not just the first one."""
@pytest.mark.asyncio
async def test_managed_handler_blocks_frame_rejected_by_quota_callback(self, monkeypatch):
async def test_managed_handler_blocks_frame_rejected_by_quota_callback(self, monkeypatch: pytest.MonkeyPatch):
from unittest.mock import AsyncMock, MagicMock
import litellm
@ -1377,11 +1378,11 @@ class TestWebSocketProjectQuotaEnforcement:
quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
assert aresponses_called is False
mock_websocket.send_text.assert_called_once()
error_event = mock_websocket.send_text.call_args[0][0]
assert "rate_limit_exceeded" in error_event
error_event = json.loads(mock_websocket.send_text.call_args[0][0])
assert (error_event["status"], error_event["error"]["type"]) == (429, "rate_limit_exceeded")
@pytest.mark.asyncio
async def test_managed_handler_forwards_frame_allowed_by_quota_callback(self, monkeypatch):
async def test_managed_handler_forwards_frame_allowed_by_quota_callback(self, monkeypatch: pytest.MonkeyPatch):
from unittest.mock import AsyncMock, MagicMock
import litellm
@ -1462,7 +1463,8 @@ class TestWebSocketProjectQuotaEnforcement:
assert allowed is False
mock_backend_ws.send.assert_not_called()
mock_websocket.send_text.assert_called_once()
assert "rate_limit_exceeded" in mock_websocket.send_text.call_args[0][0]
error_event = json.loads(mock_websocket.send_text.call_args[0][0])
assert (error_event["status"], error_event["error"]["type"]) == (429, "rate_limit_exceeded")
@pytest.mark.asyncio
async def test_native_handler_runs_the_frame_guard_on_every_turn_and_bills_each_turn_its_own_guardrail_records(
@ -1580,6 +1582,461 @@ class TestWebSocketProjectQuotaEnforcement:
quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
def _turn_frames() -> tuple[str, str, str]:
return tuple(json.dumps({"type": "response.create", "input": word}) for word in ("one", "two", "three"))
def _admission_refusing_the_next_turn():
from unittest.mock import AsyncMock
from litellm.exceptions import RateLimitError
return AsyncMock(side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), None])
class _ClientSocket:
"""A client socket that, like Starlette's, stops receiving once the server closes it."""
def __init__(self, *frames: str) -> None:
self._frames: Final = iter(frames)
self.sent: Final[list[dict[str, object]]] = []
self.close_code: int | None = None
async def send_text(self, data: str) -> None:
self.sent.append(json.loads(data))
async def receive_text(self) -> str:
frame: Final = next(self._frames, None)
if self.close_code is not None or frame is None:
raise RuntimeError("WebSocket is not connected")
return frame
async def close(self, code: int = 1000, reason: str | None = None) -> None:
self.close_code = code
class _StalledClientSocket(_ClientSocket):
"""A client socket whose sends never finish, like a client that stopped reading."""
def __init__(self, *frames: str) -> None:
super().__init__(*frames)
self.send_started: Final = asyncio.Event()
async def send_text(self, data: str) -> None:
self.send_started.set()
await asyncio.Event().wait()
def _admission_crashing_the_next_turn():
from unittest.mock import AsyncMock
return AsyncMock(side_effect=[ValueError("custom check failed"), None])
def _guard_blocking(word: str):
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame
async def guard(frame: str) -> GuardedResponsesWsFrame:
if word in json.loads(frame)["input"]:
raise ValueError("blocked by guardrail")
return GuardedResponsesWsFrame(frame=frame)
return guard
def _guard_recording(bill_refusal: MagicMock):
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame
async def guard(frame: str) -> GuardedResponsesWsFrame:
return GuardedResponsesWsFrame(
frame=frame, guardrail_information=({"guardrail_cost": 0.5},), bill_refusal=bill_refusal
)
return guard
def _errors(websocket: _ClientSocket) -> list[tuple[object, object]]:
return [(frame.get("status"), frame["error"]) for frame in websocket.sent if frame.get("type") == "error"]
class TestWebSocketTurnAdmission:
"""A turn after the connection's first is admitted by the deployment again before it reaches the provider."""
@pytest.mark.asyncio
async def test_native_relay_drops_a_later_turn_the_deployment_refuses(self):
from unittest.mock import AsyncMock
first, second, third = _turn_frames()
cancel: Final = json.dumps({"type": "response.cancel"})
turn_admission: Final = _admission_refusing_the_next_turn()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
websocket: Final = MagicMock()
websocket.send_text = AsyncMock()
websocket.receive_text = AsyncMock(side_effect=[second, cancel, third, Exception("stop")])
handler: Final = _make_streaming(
websocket=websocket,
backend_ws=backend_ws,
request_data={},
first_message=first,
turn_admission=turn_admission,
)
await handler.client_to_backend()
assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [
json.loads(first),
json.loads(cancel),
json.loads(third),
]
assert turn_admission.await_count == 2
refusal: Final = json.loads(websocket.send_text.await_args.args[0])
assert refusal["status"] == 429
assert refusal["error"]["type"] == "rate_limit_exceeded"
assert "crossed budget" in refusal["error"]["message"]
@pytest.mark.asyncio
async def test_native_relay_reports_a_refused_turn_as_the_socket_failure(self):
from unittest.mock import AsyncMock
from litellm.exceptions import RateLimitError
first, second, _ = _turn_frames()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
backend_ws.close = AsyncMock()
backend_ws.recv = AsyncMock(side_effect=Exception("stop"))
websocket: Final = MagicMock()
websocket.send_text = AsyncMock()
websocket.receive_text = AsyncMock(side_effect=[second, Exception("client gone")])
handler: Final = _make_streaming(
websocket=websocket,
backend_ws=backend_ws,
request_data={},
first_message=first,
turn_admission=_admission_refusing_the_next_turn(),
)
failure: Final = await handler.bidirectional_forward()
assert isinstance(failure, RateLimitError)
assert "crossed budget" in str(failure)
@pytest.mark.asyncio
async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch):
from unittest.mock import AsyncMock
import litellm
from litellm.exceptions import RateLimitError
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler
answered: Final[list[object]] = []
async def fake_aresponses(*args: object, **kwargs: object):
answered.append(kwargs.get("input"))
async def _empty():
return
yield
return _empty()
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
first, second, third = _turn_frames()
turn_admission: Final = _admission_refusing_the_next_turn()
websocket: Final = MagicMock()
websocket.send_text = AsyncMock()
websocket.receive_text = AsyncMock(side_effect=[second, third, Exception("stop")])
handler: Final = ManagedResponsesWebSocketHandler(
websocket=websocket,
model="test-model",
logging_obj=Logging(
model="test-model",
messages=[],
stream=True,
call_type="aresponses",
start_time=0,
litellm_call_id="test-id",
function_id="test-func",
),
first_message=first,
turn_admission=turn_admission,
)
failure: Final = await handler.run()
assert len(answered) == 2
assert "two" not in json.dumps(answered)
assert turn_admission.await_count == 2
assert isinstance(failure, RateLimitError)
assert "crossed budget" in str(failure)
refusals: Final = [
json.loads(call.args[0])
for call in websocket.send_text.await_args_list
if json.loads(call.args[0]).get("type") == "error"
]
assert [(refusal["status"], refusal["error"]["type"]) for refusal in refusals] == [(429, "rate_limit_exceeded")]
@pytest.mark.asyncio
async def test_native_relay_runs_the_guardrail_before_the_deployment_counts_a_turn(self):
from unittest.mock import AsyncMock
first, second, third = _turn_frames()
turn_admission: Final = _admission_refusing_the_next_turn()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
websocket: Final = _ClientSocket(second, third)
handler: Final = _make_streaming(
websocket=websocket,
backend_ws=backend_ws,
request_data={},
first_message=first,
frame_guard=_guard_blocking("two"),
turn_admission=turn_admission,
)
await handler.client_to_backend()
assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [json.loads(first)]
assert turn_admission.await_count == 1
assert [(status, error["type"]) for status, error in _errors(websocket)] == [
(None, "invalid_request_error"),
(429, "rate_limit_exceeded"),
]
@pytest.mark.asyncio
async def test_managed_relay_runs_the_guardrail_before_the_deployment_counts_a_turn(
self, monkeypatch: pytest.MonkeyPatch
):
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler
answered: Final[list[object]] = []
async def fake_aresponses(*args: object, **kwargs: object):
answered.append(kwargs.get("input"))
async def _empty():
return
yield
return _empty()
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
first, second, third = _turn_frames()
turn_admission: Final = _admission_refusing_the_next_turn()
websocket: Final = _ClientSocket(second, third)
handler: Final = ManagedResponsesWebSocketHandler(
websocket=websocket,
model="test-model",
logging_obj=Logging(
model="test-model",
messages=[],
stream=True,
call_type="aresponses",
start_time=0,
litellm_call_id="test-id",
function_id="test-func",
),
first_message=first,
frame_guard=_guard_blocking("two"),
turn_admission=turn_admission,
)
await handler.run()
assert answered == ["one"]
assert turn_admission.await_count == 1
assert [(status, error["type"]) for status, error in _errors(websocket)] == [
(None, "invalid_request_error"),
(429, "rate_limit_exceeded"),
]
@pytest.mark.asyncio
async def test_native_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row(self):
from unittest.mock import AsyncMock
from litellm.exceptions import RateLimitError
first, second, third = _turn_frames()
refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="")
bill_refusal: Final = MagicMock()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
backend_ws.close = AsyncMock()
backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait)
handler: Final = _make_streaming(
websocket=_ClientSocket(second, third),
backend_ws=backend_ws,
request_data={},
first_message=first,
frame_guard=_guard_recording(bill_refusal),
turn_admission=AsyncMock(side_effect=[refusal, None]),
)
close_time_failure: Final = await handler.bidirectional_forward()
assert [json.loads(sent.args[0])["input"] for sent in backend_ws.send.await_args_list] == ["one", "three"]
bill_refusal.assert_called_once_with(refusal)
assert close_time_failure is refusal
@pytest.mark.asyncio
async def test_a_refused_turn_is_recorded_even_when_the_relay_is_cancelled_while_answering_it(self):
from unittest.mock import AsyncMock
from litellm.exceptions import RateLimitError
first, second, third = _turn_frames()
refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="")
bill_refusal: Final = MagicMock()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
backend_ws.close = AsyncMock()
backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait)
client: Final = _StalledClientSocket(second, third)
handler: Final = _make_streaming(
websocket=client,
backend_ws=backend_ws,
request_data={},
first_message=first,
frame_guard=_guard_recording(bill_refusal),
turn_admission=AsyncMock(side_effect=[refusal, None]),
)
relay: Final = asyncio.ensure_future(handler.bidirectional_forward())
await asyncio.wait_for(client.send_started.wait(), timeout=1)
relay.cancel()
with pytest.raises(asyncio.CancelledError):
await relay
bill_refusal.assert_called_once_with(refusal)
@pytest.mark.asyncio
async def test_managed_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row(
self, monkeypatch: pytest.MonkeyPatch
):
from unittest.mock import AsyncMock
import litellm
from litellm.exceptions import RateLimitError
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler
answered: Final[list[object]] = []
async def fake_aresponses(*args: object, **kwargs: object):
answered.append(kwargs.get("input"))
async def _empty():
return
yield
return _empty()
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
first, second, third = _turn_frames()
refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="")
bill_refusal: Final = MagicMock()
handler: Final = ManagedResponsesWebSocketHandler(
websocket=_ClientSocket(second, third),
model="test-model",
logging_obj=Logging(
model="test-model",
messages=[],
stream=True,
call_type="aresponses",
start_time=0,
litellm_call_id="test-id",
function_id="test-func",
),
first_message=first,
frame_guard=_guard_recording(bill_refusal),
turn_admission=AsyncMock(side_effect=[refusal, None]),
)
close_time_failure: Final = await handler.run()
assert answered == ["one", "three"]
bill_refusal.assert_called_once_with(refusal)
assert close_time_failure is refusal
@pytest.mark.asyncio
async def test_native_relay_answers_a_crashed_turn_check_with_a_500_and_keeps_serving(self):
from unittest.mock import AsyncMock
first, second, third = _turn_frames()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
backend_ws.close = AsyncMock()
backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait)
websocket: Final = _ClientSocket(second, third)
handler: Final = _make_streaming(
websocket=websocket,
backend_ws=backend_ws,
request_data={},
first_message=first,
turn_admission=_admission_crashing_the_next_turn(),
)
failure: Final = await handler.bidirectional_forward()
assert isinstance(failure, ValueError)
assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [
json.loads(first),
json.loads(third),
]
assert _errors(websocket) == [(500, {"type": "server_error", "message": "custom check failed"})]
assert websocket.close_code is None
@pytest.mark.asyncio
async def test_managed_relay_answers_a_crashed_turn_check_with_a_500_and_keeps_serving(
self, monkeypatch: pytest.MonkeyPatch
):
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler
answered: Final[list[object]] = []
async def fake_aresponses(*args: object, **kwargs: object):
answered.append(kwargs.get("input"))
async def _empty():
return
yield
return _empty()
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
first, second, third = _turn_frames()
websocket: Final = _ClientSocket(second, third)
handler: Final = ManagedResponsesWebSocketHandler(
websocket=websocket,
model="test-model",
logging_obj=Logging(
model="test-model",
messages=[],
stream=True,
call_type="aresponses",
start_time=0,
litellm_call_id="test-id",
function_id="test-func",
),
first_message=first,
turn_admission=_admission_crashing_the_next_turn(),
)
failure: Final = await handler.run()
assert isinstance(failure, ValueError)
assert answered == ["one", "three"]
assert _errors(websocket) == [(500, {"type": "server_error", "message": "custom check failed"})]
assert websocket.close_code is None
def _deployment_defaults():
from types import MappingProxyType

View file

@ -2594,3 +2594,28 @@ def test_faked_stream_builds_each_refusal_summary_and_custom_tool_input_once_for
(1, "content", 1): "No forecasts for Mars.",
(2, "input", 0): "curl wttr.in/Berlin",
}
@pytest.mark.asyncio
async def test_shutdown_drain_waits_for_a_pending_logging_task():
row_written = asyncio.Event()
async def write_row():
await asyncio.sleep(0.05)
row_written.set()
streaming_module.spawn_logging_task(asyncio.get_running_loop(), write_row(), task_name="row")
await streaming_module.drain_logging_tasks(timeout=1)
assert row_written.is_set()
@pytest.mark.asyncio
async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout():
stuck = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck")
await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1)
assert not stuck.done()
stuck.cancel()

View file

@ -60,7 +60,9 @@ async def _log_success(limiter: RouterBudgetLimiting, kwargs: dict[str, object])
@pytest.mark.asyncio
@pytest.mark.parametrize("call_type", ["aresponses", "anthropic_messages", "aembedding", "arerank"])
@pytest.mark.parametrize(
"call_type", ["aresponses", "_aresponses_websocket", "anthropic_messages", "aembedding", "arerank"]
)
async def test_provider_spend_tracked_when_litellm_params_omits_provider(disable_budget_sync, call_type):
"""Non-chat surfaces carry the provider only on the standard logging payload."""
limiter = RouterBudgetLimiting(

View file

@ -1,6 +1,7 @@
import json
import os
from datetime import datetime, timezone
from collections.abc import Mapping
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -13,6 +14,7 @@ from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
from litellm.types.utils import (
CallTypes,
ModelResponse,
StandardLoggingHiddenParams,
StandardLoggingMetadata,
@ -482,6 +484,227 @@ async def test_deployment_callback_on_success(sync_mode):
assert tpm_key is not None
@pytest.mark.asyncio
async def test_deployment_callback_on_success_counts_a_responses_websocket_turn():
import time
model_list = [
{
"model_name": "gpt-5-mini",
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-fake", "rpm": 100},
"model_info": {"id": "ws-deployment"},
}
]
router = Router(model_list=model_list)
standard_logging_payload = create_standard_logging_payload()
standard_logging_payload["total_tokens"] = 17
def turn_kwargs(call_type: str) -> dict:
return {
"call_type": call_type,
"litellm_params": {
"metadata": {"model_group": "gpt-5-mini"},
"model_info": {"id": "ws-deployment"},
},
"standard_logging_object": standard_logging_payload,
}
tpm_key = await router.deployment_callback_on_success(
kwargs=turn_kwargs("_aresponses_websocket"),
completion_response=None,
start_time=time.time(),
end_time=time.time(),
)
assert tpm_key is not None
assert await router.cache.async_get_cache(key=tpm_key) == 17
realtime_key = await router.deployment_callback_on_success(
kwargs=turn_kwargs("_arealtime"),
completion_response=None,
start_time=time.time(),
end_time=time.time(),
)
assert realtime_key is None
assert await router.cache.async_get_cache(key=tpm_key) == 17
async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]:
routed: Final[dict[str, object]] = {}
async def call(**kwargs: object) -> None:
routed.update(kwargs)
call.__name__ = function_name
await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call)
return routed
async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None:
import time
from litellm.litellm_core_utils.litellm_logging import Logging
turn: Final = Logging(
model="gpt-5-mini",
messages=[],
stream=False,
call_type=CallTypes.aresponses_websocket.value,
start_time=time.time(),
litellm_call_id=call_id,
function_id=call_id,
)
turn.update_environment_variables(litellm_params={"litellm_metadata": routed["litellm_metadata"]}, optional_params={})
await turn.dispatch_success_handlers(
[{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35}}}],
prefer_async_handlers=True,
)
@pytest.mark.asyncio
async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_budget():
router = Router(
model_list=[
{
"model_name": "ws-budget",
"litellm_params": {
"model": "openai/gpt-5-mini",
"api_key": "sk-fake",
"max_budget": 0.001,
"budget_duration": "1d",
},
"model_info": {"id": "ws-budget-dep"},
}
]
)
assert "turn_admission" not in await _kwargs_routed_to(router, "ws-budget", "aresponses")
admit_turn: Final = (await _kwargs_routed_to(router, "ws-budget", "_aresponses_websocket"))["turn_admission"]
assert callable(admit_turn)
await admit_turn()
await router.cache.async_set_cache("deployment_spend:ws-budget-dep:1d", 0.002)
with pytest.raises(litellm.RateLimitError, match="crossed budget"):
await admit_turn()
@pytest.mark.asyncio
async def test_responses_websocket_turn_after_the_first_counts_toward_deployment_rpm():
router = Router(
model_list=[
{
"model_name": "ws-rpm",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "rpm": 1},
"model_info": {"id": "ws-rpm-dep"},
}
],
optional_pre_call_checks=["enforce_model_rate_limits"],
)
admit_turn: Final = (await _kwargs_routed_to(router, "ws-rpm", "_aresponses_websocket"))["turn_admission"]
assert callable(admit_turn)
with pytest.raises(litellm.RateLimitError, match="RPM limit=1"):
await admit_turn()
@pytest.mark.asyncio
async def test_responses_websocket_turn_is_checked_against_limits_changed_after_the_socket_opened():
router = Router(
model_list=[
{
"model_name": "ws-rpm-lowered",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "rpm": 100},
"model_info": {"id": "ws-rpm-lowered-dep"},
}
],
optional_pre_call_checks=["enforce_model_rate_limits"],
)
admit_turn: Final = (await _kwargs_routed_to(router, "ws-rpm-lowered", "_aresponses_websocket"))["turn_admission"]
assert callable(admit_turn)
await admit_turn()
router.upsert_deployment(
Deployment(
model_name="ws-rpm-lowered",
litellm_params=LiteLLM_Params(model="openai/gpt-5-mini", api_key="sk-fake", rpm=1),
model_info=ModelInfo(id="ws-rpm-lowered-dep"),
)
)
with pytest.raises(litellm.RateLimitError, match="RPM limit=1"):
await admit_turn()
@pytest.mark.asyncio
async def test_responses_websocket_turn_counts_toward_deployment_tpm(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "callbacks", [])
router = Router(
model_list=[
{
"model_name": "ws-tpm",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": 30},
"model_info": {"id": "ws-tpm-dep"},
}
],
optional_pre_call_checks=["enforce_model_rate_limits"],
)
monkeypatch.setattr(litellm, "_async_success_callback", list(litellm.callbacks))
routed: Final = await _kwargs_routed_to(router, "ws-tpm", "_aresponses_websocket")
admit_turn: Final = routed["turn_admission"]
assert callable(admit_turn)
await admit_turn()
await _finish_35_token_websocket_turn(routed, "ws-tpm-turn")
with pytest.raises(litellm.RateLimitError, match="TPM limit=30, current usage=35"):
await admit_turn()
@pytest.mark.asyncio
async def test_responses_websocket_turn_is_refused_by_usage_based_routing_once_deployment_tpm_is_used(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "callbacks", [])
router = Router(
model_list=[
{
"model_name": "ws-tpm-v2",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": 30},
"model_info": {"id": "ws-tpm-v2-dep"},
}
],
routing_strategy="usage-based-routing-v2",
)
monkeypatch.setattr(litellm, "_async_success_callback", list(litellm.callbacks))
routed: Final = await _kwargs_routed_to(router, "ws-tpm-v2", "_aresponses_websocket")
admit_turn: Final = routed["turn_admission"]
assert callable(admit_turn)
await admit_turn()
await _finish_35_token_websocket_turn(routed, "ws-tpm-v2-turn")
with pytest.raises(litellm.RateLimitError):
await admit_turn()
@pytest.mark.asyncio
async def test_responses_websocket_turns_do_not_reserve_deployment_otpm():
router = Router(
model_list=[
{
"model_name": "ws-otpm",
"litellm_params": {"model": "openai/ws-otpm-unpriced", "api_key": "sk-fake", "otpm": 5000},
"model_info": {"id": "ws-otpm-dep"},
}
],
optional_pre_call_checks=["enforce_model_rate_limits"],
)
admit_turn: Final = (await _kwargs_routed_to(router, "ws-otpm", "_aresponses_websocket"))["turn_admission"]
assert callable(admit_turn)
for _ in range(3):
await admit_turn()
@pytest.mark.asyncio
async def test_deployment_callback_on_success_tracks_tpm_for_io_deployment():
"""

View file

@ -114,10 +114,22 @@ from litellm.utils import (
)
def test_get_base_model_from_metadata_returns_unvalidated_root_value():
@pytest.mark.parametrize("root_base_model", [42, ["gemini-3.5-flash"], {"model": "gemini-3.5-flash"}])
def test_get_base_model_from_metadata_ignores_a_non_string_root_value(root_base_model: object):
from litellm.utils import get_base_model_from_metadata
assert get_base_model_from_metadata({"litellm_params": {"base_model": 42}}) == 42
assert get_base_model_from_metadata({"litellm_params": {"base_model": root_base_model}}) is None
assert (
get_base_model_from_metadata(
{
"litellm_params": {
"base_model": root_base_model,
"metadata": {"model_info": {"base_model": "vertex_ai/gemini-3.5-flash"}},
}
}
)
== "vertex_ai/gemini-3.5-flash"
)
def _reject_short_post_call_response(input: str, model: str | None = None) -> dict[str, object]: