mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
chore: merge main and avoid catalog insertion conflicts
This commit is contained in:
commit
6635fb0b1d
89 changed files with 6414 additions and 2554 deletions
|
|
@ -6,9 +6,11 @@ use litellm_llms::{
|
|||
bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
|
||||
openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
|
||||
};
|
||||
use litellm_llms_types::formats::chat_completions::ChatMessage;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::Error;
|
||||
use crate::types::{ChatCompletionsCall, ResolvedChatCompletionsRequest};
|
||||
|
||||
const HEADER_CONTEXT: &str = "chat completions";
|
||||
|
||||
|
|
@ -42,3 +44,58 @@ pub(super) fn string_headers(
|
|||
) -> Result<Vec<(String, String)>, Error> {
|
||||
shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_request(
|
||||
call: ChatCompletionsCall,
|
||||
config: &'static dyn BaseConfig,
|
||||
) -> Result<ResolvedChatCompletionsRequest, Error> {
|
||||
let messages = serde_json::from_value::<Vec<ChatMessage>>(call.messages).map_err(|err| {
|
||||
Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
|
||||
"chat completions messages",
|
||||
err,
|
||||
))
|
||||
})?;
|
||||
if messages.is_empty() {
|
||||
return Err(Error::InvalidRequest(
|
||||
"chat completions requires at least one message".into(),
|
||||
));
|
||||
}
|
||||
if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) {
|
||||
return Err(Error::Unsupported(reason.0));
|
||||
}
|
||||
Ok(ResolvedChatCompletionsRequest {
|
||||
messages,
|
||||
optional_params: call.optional_params,
|
||||
api_key: call.api_key,
|
||||
api_base: call.api_base,
|
||||
extra_headers: call.extra_headers,
|
||||
timeout: call.timeout,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::chat_completions_provider;
|
||||
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
|
||||
|
||||
#[rstest]
|
||||
#[case::anthropic(LlmProviders::Anthropic, ("anthropic-version", "2023-06-01"))]
|
||||
#[case::bedrock(LlmProviders::Bedrock, ("Content-Type", "application/json"))]
|
||||
fn resolves_supported_provider_configs(
|
||||
#[case] provider: LlmProviders,
|
||||
#[case] expected_header: (&str, &str),
|
||||
) {
|
||||
let config = chat_completions_provider(provider)
|
||||
.expect("provider has a chat completions config")
|
||||
.config();
|
||||
assert!(config.default_headers().contains(&expected_header));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::openai(LlmProviders::Openai)]
|
||||
fn rejects_providers_without_a_chat_config(#[case] provider: LlmProviders) {
|
||||
assert!(chat_completions_provider(provider).is_none());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,44 +3,66 @@ use std::time::Duration;
|
|||
|
||||
use litellm_auth::AuthServices;
|
||||
use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
|
||||
use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body};
|
||||
use litellm_http::{
|
||||
Client,
|
||||
outbound::OutboundRequest,
|
||||
request::{truncate_error_body, with_default_headers},
|
||||
};
|
||||
use litellm_llms::base_llm::{
|
||||
auth::{Authenticated, resolve_auth},
|
||||
auth::{Authenticated, ValidatedEnvironment, resolve_auth},
|
||||
chat::transformation::BaseConfig,
|
||||
chat::transformation::ProviderChatResponseData,
|
||||
};
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
|
||||
use litellm_secrets::source::Secrets;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::Error;
|
||||
use crate::{constants::CHAT_COMPLETIONS_TIMEOUT_SECS, types::ProviderChatCompletionsRequest};
|
||||
use crate::{
|
||||
common_utils::string_headers, constants::CHAT_COMPLETIONS_TIMEOUT_SECS,
|
||||
types::ResolvedChatCompletionsRequest,
|
||||
};
|
||||
|
||||
#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately
|
||||
pub(super) async fn execute(
|
||||
http: &Client,
|
||||
auth: &AuthServices,
|
||||
request: ProviderChatCompletionsRequest,
|
||||
config: &'static dyn BaseConfig,
|
||||
provider: litellm_inference::provider::ResolvedProvider<'_>,
|
||||
call: ResolvedChatCompletionsRequest,
|
||||
secrets: Secrets,
|
||||
cache: Option<litellm_cache_response::ScopedCache>,
|
||||
cache_options: Option<litellm_cache_response::CachePolicy>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<ChatCompletionsResponse, Error> {
|
||||
let ProviderChatCompletionsRequest {
|
||||
model,
|
||||
custom_llm_provider,
|
||||
config,
|
||||
url,
|
||||
body,
|
||||
optional_params,
|
||||
environment,
|
||||
secrets,
|
||||
timeout,
|
||||
api_key,
|
||||
} = request;
|
||||
let validated = config.validate_environment(
|
||||
string_headers(call.extra_headers)?,
|
||||
call.api_key.as_deref(),
|
||||
provider.model,
|
||||
&call.optional_params,
|
||||
&|key| secrets.get(key),
|
||||
)?;
|
||||
let url = config.get_complete_url(
|
||||
call.api_base.as_deref(),
|
||||
provider.model,
|
||||
&call.optional_params,
|
||||
&|key| secrets.get(key),
|
||||
)?;
|
||||
let body = config
|
||||
.transform_request(provider.model, call.messages, call.optional_params.clone())?
|
||||
.body;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: with_default_headers(validated.headers, config.default_headers()),
|
||||
auth: validated.auth,
|
||||
};
|
||||
let provider_name = <&'static str>::from(provider.provider);
|
||||
let context = RequestContext {
|
||||
model: model.clone(),
|
||||
custom_llm_provider,
|
||||
optional_params: Value::Object(optional_params),
|
||||
model: provider.model.to_owned(),
|
||||
custom_llm_provider: provider_name.to_owned(),
|
||||
optional_params: Value::Object(call.optional_params),
|
||||
secret_fields: Vec::new(),
|
||||
api_key,
|
||||
api_key: call.api_key.map(litellm_auth::SecretValue::new),
|
||||
};
|
||||
let authenticated = resolve_auth(auth, environment, &|key| secrets.get(key)).await?;
|
||||
let identity = litellm_host::interceptors::ProviderIdentity {
|
||||
|
|
@ -76,7 +98,7 @@ pub(super) async fn execute(
|
|||
},
|
||||
wire.url,
|
||||
&wire.body,
|
||||
timeout,
|
||||
call.timeout,
|
||||
)?;
|
||||
|
||||
let response = litellm_inference::outbound::send(outbound, http)
|
||||
|
|
@ -121,7 +143,7 @@ pub(super) async fn execute(
|
|||
))
|
||||
})?;
|
||||
config
|
||||
.transform_response(&model, ProviderChatResponseData { body })
|
||||
.transform_response(provider.model, ProviderChatResponseData { body })
|
||||
.map_err(Error::from)
|
||||
.map_err(as_response_error)
|
||||
},
|
||||
|
|
@ -134,7 +156,7 @@ pub(super) async fn execute(
|
|||
///
|
||||
/// A config reports the same variants on either side of the call: a missing
|
||||
/// field or an unsupported block can mean "this request cannot be translated"
|
||||
/// during prepare and "this response cannot be normalized" here. Only the
|
||||
/// before the request is sent and "this response cannot be normalized" here. Only the
|
||||
/// second kind has already been billed, and a host that keeps a reference
|
||||
/// implementation must not retry those, so collapse them to one variant that
|
||||
/// can only mean the provider was already called.
|
||||
|
|
@ -178,9 +200,10 @@ mod tests {
|
|||
|
||||
use super::*;
|
||||
use crate::{
|
||||
prepare::{prepare_provider_request, resolve_request},
|
||||
types::ChatCompletionsRequest,
|
||||
common_utils::{chat_completions_provider, resolve_request},
|
||||
types::{ChatCompletionsCall, ResolvedChatCompletionsRequest},
|
||||
};
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
|
||||
const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
|
||||
|
||||
|
|
@ -217,22 +240,33 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
fn prepared(api_base: &str) -> ProviderChatCompletionsRequest {
|
||||
prepare_provider_request(
|
||||
resolve_request(ChatCompletionsRequest {
|
||||
model: "anthropic/claude-sonnet-4-5",
|
||||
fn prepared(
|
||||
api_base: &str,
|
||||
) -> (
|
||||
litellm_inference::provider::ResolvedProvider<'static>,
|
||||
&'static dyn BaseConfig,
|
||||
ResolvedChatCompletionsRequest,
|
||||
) {
|
||||
let provider =
|
||||
resolve_llm_provider("anthropic/claude-sonnet-4-5", None, "chat completions").unwrap();
|
||||
let config = chat_completions_provider(provider.provider)
|
||||
.expect("provider has a chat completions config")
|
||||
.config();
|
||||
let request = resolve_request(
|
||||
ChatCompletionsCall {
|
||||
model: "anthropic/claude-sonnet-4-5".into(),
|
||||
messages: json!([{"role": "user", "content": "hi"}]),
|
||||
optional_params: json!({"max_tokens": 16}).as_object().unwrap().clone(),
|
||||
api_key: Some("sk-test"),
|
||||
api_base: Some(api_base),
|
||||
api_key: Some("sk-test".into()),
|
||||
api_base: Some(api_base.into()),
|
||||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
timeout: None,
|
||||
})
|
||||
.unwrap(),
|
||||
std::sync::Arc::new(|_: &str| None),
|
||||
},
|
||||
config,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
(provider, config, request)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -247,10 +281,14 @@ mod tests {
|
|||
.await;
|
||||
let interceptors = RecordingHooks::default();
|
||||
|
||||
let (provider, config, request) = prepared(&upstream.uri());
|
||||
execute(
|
||||
&Client::plain_for_test(),
|
||||
&AuthServices::default(),
|
||||
prepared(&upstream.uri()),
|
||||
config,
|
||||
provider,
|
||||
request,
|
||||
std::sync::Arc::new(|_: &str| None),
|
||||
None,
|
||||
None,
|
||||
&interceptors,
|
||||
|
|
@ -288,10 +326,14 @@ mod tests {
|
|||
.await;
|
||||
let interceptors = RecordingHooks::default();
|
||||
|
||||
let (provider, config, request) = prepared(&upstream.uri());
|
||||
let error = execute(
|
||||
&Client::plain_for_test(),
|
||||
&AuthServices::default(),
|
||||
prepared(&upstream.uri()),
|
||||
config,
|
||||
provider,
|
||||
request,
|
||||
std::sync::Arc::new(|_: &str| None),
|
||||
None,
|
||||
None,
|
||||
&interceptors,
|
||||
|
|
|
|||
|
|
@ -5,14 +5,13 @@ pub use litellm_inference::RouteError as Error;
|
|||
mod common_utils;
|
||||
pub mod constants;
|
||||
pub(crate) mod handler;
|
||||
mod prepare;
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
|
||||
use prepare::{prepare_provider_request, resolve_request};
|
||||
|
||||
use litellm_auth::AuthServices;
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use std::sync::Arc;
|
||||
use types::ChatCompletionsRequest;
|
||||
use types::{ChatCompletionsCall, ChatCompletionsRequest};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ChatCompletionsRoute {
|
||||
|
|
@ -55,7 +54,7 @@ impl ChatCompletionsRoute {
|
|||
} = options.into();
|
||||
litellm_host::lifecycle::observe_unary(
|
||||
observers.clone(),
|
||||
self.run_call(
|
||||
self.run(
|
||||
request.into(),
|
||||
cache_options,
|
||||
interceptors,
|
||||
|
|
@ -65,30 +64,52 @@ impl ChatCompletionsRoute {
|
|||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.route", skip_all, fields(
|
||||
route = "chat_completions",
|
||||
model = %call.model,
|
||||
provider,
|
||||
resolved_model,
|
||||
stream = false,
|
||||
outcome
|
||||
))]
|
||||
async fn run(
|
||||
&self,
|
||||
request: ChatCompletionsRequest<'_>,
|
||||
call: ChatCompletionsCall,
|
||||
cache_options: Option<litellm_cache_response::CachePolicy>,
|
||||
interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<ChatCompletionsResponse, Error> {
|
||||
let resolved = resolve_request(request)?;
|
||||
let snapshot = self
|
||||
.secrets
|
||||
.resolve(&resolved.config.secret_names())
|
||||
.await?;
|
||||
let prepared = prepare_provider_request(resolved, snapshot)?;
|
||||
litellm_inference::diagnostic::provider(&prepared.model, &prepared.custom_llm_provider);
|
||||
let execute: futures_util::future::BoxFuture<'_, Result<ChatCompletionsResponse, Error>> =
|
||||
Box::pin(handler::execute(
|
||||
litellm_inference::diagnostic::unary(async {
|
||||
let model = call.model.clone();
|
||||
let custom_llm_provider = call.custom_llm_provider.clone();
|
||||
let provider =
|
||||
resolve_llm_provider(&model, custom_llm_provider.as_deref(), "chat completions")?;
|
||||
let config = common_utils::chat_completions_provider(provider.provider)
|
||||
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?
|
||||
.config();
|
||||
let call = common_utils::resolve_request(call, config)?;
|
||||
litellm_inference::diagnostic::provider(
|
||||
provider.model,
|
||||
<&'static str>::from(provider.provider),
|
||||
);
|
||||
let secrets = self.secrets.resolve(&config.secret_names()).await?;
|
||||
let execute: futures_util::future::BoxFuture<
|
||||
'_,
|
||||
Result<ChatCompletionsResponse, Error>,
|
||||
> = Box::pin(handler::execute(
|
||||
&self.http,
|
||||
&self.auth,
|
||||
prepared,
|
||||
config,
|
||||
provider,
|
||||
call,
|
||||
secrets,
|
||||
self.cache.clone(),
|
||||
cache_options,
|
||||
interceptors,
|
||||
observers,
|
||||
));
|
||||
execute.await
|
||||
execute.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,638 +0,0 @@
|
|||
use litellm_auth::SecretValue;
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_http::request::with_default_headers;
|
||||
use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
|
||||
use litellm_llms_types::formats::chat_completions::ChatMessage;
|
||||
use litellm_secrets::source::Secrets;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{
|
||||
Error,
|
||||
common_utils::{chat_completions_provider, string_headers},
|
||||
};
|
||||
use crate::types::{
|
||||
ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
|
||||
};
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
|
||||
pub(super) struct ResolvedProvider {
|
||||
pub(super) model: String,
|
||||
pub(super) custom_llm_provider: String,
|
||||
pub(super) config: &'static dyn BaseConfig,
|
||||
}
|
||||
|
||||
pub(super) fn resolve_provider_config<'a>(
|
||||
model: &'a str,
|
||||
custom_llm_provider: Option<&'a str>,
|
||||
) -> Result<ResolvedProvider, Error> {
|
||||
let provider_info = resolve_llm_provider(model, custom_llm_provider, "chat completions")?;
|
||||
let config = chat_completions_provider(provider_info.provider)
|
||||
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?
|
||||
.config();
|
||||
Ok(ResolvedProvider {
|
||||
model: provider_info.model.to_string(),
|
||||
custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
|
||||
config,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn parse_messages(messages: Value) -> Result<Vec<ChatMessage>, Error> {
|
||||
serde_json::from_value(messages).map_err(|err| {
|
||||
Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
|
||||
"chat completions messages",
|
||||
err,
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn resolve_request(
|
||||
request: ChatCompletionsRequest<'_>,
|
||||
) -> Result<ResolvedChatCompletionsRequest<'_>, Error> {
|
||||
let ResolvedProvider {
|
||||
model,
|
||||
custom_llm_provider,
|
||||
config,
|
||||
} = resolve_provider_config(request.model, request.custom_llm_provider)?;
|
||||
let messages = parse_messages(request.messages)?;
|
||||
if messages.is_empty() {
|
||||
return Err(Error::InvalidRequest(
|
||||
"chat completions requires at least one message".into(),
|
||||
));
|
||||
}
|
||||
if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) {
|
||||
return Err(Error::Unsupported(reason.0));
|
||||
}
|
||||
Ok(ResolvedChatCompletionsRequest {
|
||||
model,
|
||||
custom_llm_provider,
|
||||
config,
|
||||
messages,
|
||||
optional_params: request.optional_params,
|
||||
api_key: request.api_key,
|
||||
api_base: request.api_base,
|
||||
extra_headers: request.extra_headers,
|
||||
timeout: request.timeout,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_environment(
|
||||
request: &ResolvedChatCompletionsRequest<'_>,
|
||||
model: &str,
|
||||
config: &dyn BaseConfig,
|
||||
secrets: &dyn Lookup,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
let env_lookup = |key: &str| secrets.get(key);
|
||||
let forwarded = string_headers(request.extra_headers.clone())?;
|
||||
let validated = config.validate_environment(
|
||||
forwarded,
|
||||
request.api_key,
|
||||
model,
|
||||
&request.optional_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
Ok(ValidatedEnvironment {
|
||||
headers: with_default_headers(validated.headers, config.default_headers()),
|
||||
auth: validated.auth,
|
||||
})
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
|
||||
pub(super) fn prepare_provider_request(
|
||||
request: ResolvedChatCompletionsRequest<'_>,
|
||||
secrets: Secrets,
|
||||
) -> Result<ProviderChatCompletionsRequest, Error> {
|
||||
let environment =
|
||||
validate_environment(&request, &request.model, request.config, secrets.as_ref())?;
|
||||
let model = request.model;
|
||||
let config = request.config;
|
||||
let env_lookup = |key: &str| secrets.get(key);
|
||||
let url = config.get_complete_url(
|
||||
request.api_base,
|
||||
&model,
|
||||
&request.optional_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
let transformed =
|
||||
config.transform_request(&model, request.messages, request.optional_params.clone())?;
|
||||
|
||||
Ok(ProviderChatCompletionsRequest {
|
||||
model,
|
||||
custom_llm_provider: request.custom_llm_provider,
|
||||
config,
|
||||
url,
|
||||
body: transformed.body,
|
||||
optional_params: request.optional_params,
|
||||
environment,
|
||||
secrets,
|
||||
timeout: request.timeout,
|
||||
api_key: request.api_key.map(|key| SecretValue::new(key.to_string())),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_auth::CredentialPlacement;
|
||||
use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use super::{prepare_provider_request, resolve_request};
|
||||
use crate::{
|
||||
Error,
|
||||
types::{ChatCompletionsRequest, ProviderChatCompletionsRequest},
|
||||
};
|
||||
|
||||
fn prepare_chat_completions_call(
|
||||
request: ChatCompletionsRequest<'_>,
|
||||
) -> Result<ProviderChatCompletionsRequest, Error> {
|
||||
prepare_provider_request(
|
||||
resolve_request(request)?,
|
||||
std::sync::Arc::new(|_: &str| None),
|
||||
)
|
||||
}
|
||||
|
||||
/// The headers as they go on the wire, credential applied.
|
||||
fn wire_headers(prepared: &ProviderChatCompletionsRequest) -> Vec<(String, String)> {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(resolve_auth(
|
||||
&litellm_auth::AuthServices::default(),
|
||||
prepared.environment.clone(),
|
||||
&|_| None,
|
||||
))
|
||||
.unwrap()
|
||||
.headers
|
||||
}
|
||||
|
||||
fn request<'a>(
|
||||
model: &'a str,
|
||||
provider: Option<&'a str>,
|
||||
messages: Value,
|
||||
optional_params: Value,
|
||||
) -> ChatCompletionsRequest<'a> {
|
||||
ChatCompletionsRequest {
|
||||
model,
|
||||
messages,
|
||||
optional_params: match optional_params {
|
||||
Value::Object(map) => map,
|
||||
other => panic!("params must be an object, got {other}"),
|
||||
},
|
||||
api_key: Some("sk-test"),
|
||||
api_base: None,
|
||||
custom_llm_provider: provider,
|
||||
extra_headers: None,
|
||||
timeout: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn preparation_error(request: ChatCompletionsRequest<'_>) -> Error {
|
||||
prepare_chat_completions_call(request)
|
||||
.err()
|
||||
.expect("request preparation should fail")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_the_provider_from_the_model_prefix() {
|
||||
let prepared = prepare_chat_completions_call(request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({"max_tokens": 16}),
|
||||
))
|
||||
.expect("prepares");
|
||||
assert_eq!(prepared.model, "claude-sonnet-4-5");
|
||||
assert_eq!(prepared.url, "https://api.anthropic.com/v1/messages");
|
||||
assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_an_explicit_provider_prefix_from_the_model() {
|
||||
let prepared = prepare_chat_completions_call(request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
))
|
||||
.expect("prepares");
|
||||
assert_eq!(prepared.model, "claude-sonnet-4-5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adds_the_auth_and_default_headers() {
|
||||
let prepared = prepare_chat_completions_call(request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
))
|
||||
.expect("prepares");
|
||||
assert!(
|
||||
wire_headers(&prepared).contains(&("x-api-key".to_string(), "sk-test".to_string()))
|
||||
);
|
||||
assert!(
|
||||
wire_headers(&prepared)
|
||||
.contains(&("anthropic-version".to_string(), "2023-06-01".to_string()))
|
||||
);
|
||||
assert!(matches!(
|
||||
prepared.environment.auth,
|
||||
AuthScheme::Credential {
|
||||
placement: CredentialPlacement::Header("x-api-key"),
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
|
||||
// Python builds `{**headers, **anthropic_headers}`, so the deployment's key
|
||||
// overwrites a forwarded one. Honouring the caller's would let whoever sends
|
||||
// the request choose the Anthropic principal it bills to.
|
||||
let mut call = request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
);
|
||||
call.extra_headers = Some(Map::from_iter([(
|
||||
"X-Api-Key".to_string(),
|
||||
json!("sk-caller"),
|
||||
)]));
|
||||
let prepared = prepare_chat_completions_call(call).expect("prepares");
|
||||
let headers = wire_headers(&prepared);
|
||||
let keys: Vec<_> = headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
|
||||
.collect();
|
||||
assert_eq!(keys.len(), 1, "got {:?}", headers);
|
||||
assert_eq!(keys[0].1, "sk-test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
|
||||
// Anthropic's `validate_environment` pops `x-api-key` and sets `authorization`
|
||||
// for an OAuth token, so re-adding the key here would put the credential into
|
||||
// a header the host removed on purpose.
|
||||
let mut call = request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
);
|
||||
call.extra_headers = Some(Map::from_iter([
|
||||
(
|
||||
"Authorization".to_string(),
|
||||
json!("Bearer sk-ant-oat01-token"),
|
||||
),
|
||||
("X-Api-Key".to_string(), json!("sk-caller")),
|
||||
]));
|
||||
let prepared = prepare_chat_completions_call(call).expect("prepares");
|
||||
assert!(
|
||||
!wire_headers(&prepared)
|
||||
.iter()
|
||||
.any(|(name, value)| name.eq_ignore_ascii_case("x-api-key") && value == "sk-test"),
|
||||
"the resolved key must not be applied over an OAuth bearer, got {:?}",
|
||||
wire_headers(&prepared)
|
||||
);
|
||||
assert!(
|
||||
wire_headers(&prepared)
|
||||
.iter()
|
||||
.any(|(name, value)| name.eq_ignore_ascii_case("authorization")
|
||||
&& value == "Bearer sk-ant-oat01-token")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
|
||||
// Only an OAuth bearer replaces the credential. Python sends the deployment's
|
||||
// `x-api-key` alongside any other forwarded `authorization`, so deferring on
|
||||
// the mere presence of that header would drop the deployment's auth.
|
||||
let mut call = request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
);
|
||||
call.extra_headers = Some(Map::from_iter([
|
||||
("Authorization".to_string(), json!("Bearer unrelated")),
|
||||
("X-Api-Key".to_string(), json!("sk-caller")),
|
||||
]));
|
||||
let prepared = prepare_chat_completions_call(call).expect("prepares");
|
||||
let headers = wire_headers(&prepared);
|
||||
let keys: Vec<_> = headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
|
||||
.collect();
|
||||
assert_eq!(keys.len(), 1, "got {:?}", headers);
|
||||
assert_eq!(keys[0].1, "sk-test");
|
||||
assert!(
|
||||
wire_headers(&prepared)
|
||||
.iter()
|
||||
.any(|(name, value)| name.eq_ignore_ascii_case("authorization")
|
||||
&& value == "Bearer unrelated"),
|
||||
"the unrelated authorization must survive, got {:?}",
|
||||
wire_headers(&prepared)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
fn rejects_empty_messages_before_resolving_credentials() {
|
||||
let call = ChatCompletionsRequest {
|
||||
api_key: None,
|
||||
..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
|
||||
};
|
||||
assert!(matches!(preparation_error(call), Error::InvalidRequest(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_an_unknown_provider() {
|
||||
assert_eq!(
|
||||
preparation_error(request(
|
||||
"openai/gpt-4o",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)),
|
||||
Error::InvalidProvider("openai".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_a_model_with_no_resolvable_provider() {
|
||||
assert!(matches!(
|
||||
preparation_error(request(
|
||||
"claude-sonnet-4-5",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)),
|
||||
Error::InvalidProvider(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_an_empty_or_malformed_message_list() {
|
||||
assert_eq!(
|
||||
preparation_error(request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
None,
|
||||
json!([]),
|
||||
json!({}),
|
||||
)),
|
||||
Error::InvalidRequest(
|
||||
"chat completions requires at least one message"
|
||||
.to_string()
|
||||
.into()
|
||||
)
|
||||
);
|
||||
assert!(matches!(
|
||||
preparation_error(request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
None,
|
||||
json!("not a list"),
|
||||
json!({}),
|
||||
)),
|
||||
Error::InvalidRequest(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_string_extra_headers() {
|
||||
let mut call = request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
);
|
||||
call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
|
||||
assert_eq!(
|
||||
preparation_error(call),
|
||||
Error::Headers(litellm_http::request::HeaderError {
|
||||
context: "chat completions",
|
||||
name: "x-trace".to_string(),
|
||||
actual: "number",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepares_a_bedrock_call_without_resolving_credentials() {
|
||||
let mut call = request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({"maxTokens": 16}),
|
||||
);
|
||||
call.api_key = None;
|
||||
let prepared = prepare_chat_completions_call(call).expect("prepares");
|
||||
assert_eq!(
|
||||
prepared.url,
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
|
||||
);
|
||||
assert!(matches!(
|
||||
&prepared.environment.auth,
|
||||
AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == "us-east-1"
|
||||
));
|
||||
// SigV4 signs the serialized body, so prepare must not have added an
|
||||
// Authorization header; the signer does it over the bytes sent.
|
||||
assert!(
|
||||
!prepared
|
||||
.environment
|
||||
.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("authorization"))
|
||||
);
|
||||
assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
|
||||
// Python signs only the AWS header set and reattaches the rest, so a header
|
||||
// the caller forwarded rides along without joining the canonical request.
|
||||
// Signing it makes Converse 403 on a deployment that works on Python.
|
||||
let mut call = request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({
|
||||
"maxTokens": 16,
|
||||
"aws_access_key_id": "AKIDEXAMPLE",
|
||||
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
|
||||
}),
|
||||
);
|
||||
// A key would resolve to a bearer token and never reach the signer.
|
||||
call.api_key = None;
|
||||
call.extra_headers = Some(Map::from_iter([(
|
||||
"x-request-id".to_string(),
|
||||
json!("abc-123"),
|
||||
)]));
|
||||
let prepared = prepare_chat_completions_call(call).expect("prepares");
|
||||
let authenticated = resolve_auth(
|
||||
&litellm_auth::AuthServices::default(),
|
||||
prepared.environment,
|
||||
&|_| None,
|
||||
)
|
||||
.await
|
||||
.expect("resolves");
|
||||
let signed = crate::handler::outbound_request(
|
||||
authenticated,
|
||||
prepared.url,
|
||||
&prepared.body,
|
||||
prepared.timeout,
|
||||
)
|
||||
.expect("signs");
|
||||
|
||||
let authorization = signed
|
||||
.header("authorization")
|
||||
.expect("carries an authorization header")
|
||||
.to_string();
|
||||
assert!(
|
||||
authorization.starts_with("AWS4-HMAC-SHA256"),
|
||||
"expected a SigV4 signature, got {authorization}"
|
||||
);
|
||||
assert!(
|
||||
!authorization.contains("x-request-id"),
|
||||
"forwarded header reached SignedHeaders: {authorization}"
|
||||
);
|
||||
// It still goes on the wire, it is just not part of the signature.
|
||||
assert!(
|
||||
signed
|
||||
.headers()
|
||||
.iter()
|
||||
.any(|(name, value)| name == "x-request-id" && value == "abc-123"),
|
||||
"forwarded header was dropped instead of reattached"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::authorization("Authorization")]
|
||||
#[case::amz_date("x-amz-date")]
|
||||
#[case::security_token("x-amz-security-token")]
|
||||
#[case::date("Date")]
|
||||
#[tokio::test]
|
||||
async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
|
||||
let call = ChatCompletionsRequest {
|
||||
api_key: None,
|
||||
extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
|
||||
..request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({
|
||||
"maxTokens": 16,
|
||||
"aws_access_key_id": "AKIDEXAMPLE",
|
||||
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
|
||||
}),
|
||||
)
|
||||
};
|
||||
let prepared = prepare_chat_completions_call(call).expect("prepares");
|
||||
let authenticated = resolve_auth(
|
||||
&litellm_auth::AuthServices::default(),
|
||||
prepared.environment,
|
||||
&|_| None,
|
||||
)
|
||||
.await
|
||||
.expect("resolves");
|
||||
let error = crate::handler::outbound_request(
|
||||
authenticated,
|
||||
prepared.url,
|
||||
&prepared.body,
|
||||
prepared.timeout,
|
||||
)
|
||||
.expect_err("conflicting signing headers must fail");
|
||||
assert!(
|
||||
matches!(error, Error::Unsupported(_)),
|
||||
"{forwarded} returned {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
|
||||
// `get_request_headers` assigns `headers["Authorization"]` unconditionally
|
||||
// once a bearer token resolves, so the deployment's identity wins on
|
||||
// Python. Keeping the caller's would authorize and bill the call as a
|
||||
// different principal, and only when the deployment carries `rust: true`.
|
||||
let mut call = request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({"maxTokens": 16}),
|
||||
);
|
||||
call.extra_headers = Some(Map::from_iter([(
|
||||
"Authorization".to_string(),
|
||||
json!("Bearer caller-supplied"),
|
||||
)]));
|
||||
let prepared = prepare_chat_completions_call(call).expect("prepares");
|
||||
let headers = wire_headers(&prepared);
|
||||
let authorizations: Vec<_> = headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("authorization"))
|
||||
.map(|(_, value)| value.as_str())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
authorizations,
|
||||
vec!["Bearer sk-test"],
|
||||
"the deployment token must be the only authorization on the wire"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
|
||||
// The opposite precedence, and deliberate: Anthropic's own transform
|
||||
// honours a forwarded OAuth bearer, so the Bedrock fix above must not be
|
||||
// generalized into a rule that the configured key always wins.
|
||||
//
|
||||
// An OAuth bearer is the whole of that exception. This forwarded a plain
|
||||
// `x-api-key` until round 17, which read as the same claim and was not:
|
||||
// Python overwrites a forwarded `x-api-key` with the deployment's.
|
||||
let mut call = request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
);
|
||||
call.extra_headers = Some(Map::from_iter([(
|
||||
"authorization".to_string(),
|
||||
json!("Bearer sk-ant-oat01-forwarded"),
|
||||
)]));
|
||||
let prepared = prepare_chat_completions_call(call).expect("prepares");
|
||||
let headers = wire_headers(&prepared);
|
||||
let keys: Vec<_> = headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
|
||||
.map(|(_, value)| value.as_str())
|
||||
.collect();
|
||||
assert!(keys.is_empty(), "got {:?}", headers);
|
||||
assert!(
|
||||
wire_headers(&prepared)
|
||||
.iter()
|
||||
.any(|(name, value)| name.eq_ignore_ascii_case("authorization")
|
||||
&& value == "Bearer sk-ant-oat01-forwarded")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
|
||||
// The configured bearer identity has its own account and quota boundary,
|
||||
// so a request carrying one must not be signed as whatever principal the
|
||||
// host's AWS credentials resolve to.
|
||||
let prepared = prepare_chat_completions_call(request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({"maxTokens": 16}),
|
||||
))
|
||||
.expect("prepares");
|
||||
assert!(matches!(
|
||||
&prepared.environment.auth,
|
||||
AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret }
|
||||
if secret.expose() == "sk-test"
|
||||
));
|
||||
assert!(
|
||||
wire_headers(&prepared)
|
||||
.iter()
|
||||
.any(|(name, value)| name.eq_ignore_ascii_case("authorization")
|
||||
&& value == "Bearer sk-test"),
|
||||
"prepare did not carry the bearer token"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
use litellm_host::observation::ObservationSender;
|
||||
use std::convert::Infallible;
|
||||
|
||||
use litellm_host::{
|
||||
|
|
@ -7,10 +6,7 @@ use litellm_host::{
|
|||
};
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
|
||||
|
||||
use super::{
|
||||
ChatCompletionsRoute, Error,
|
||||
types::{ChatCompletionsCall, ChatCompletionsRequest},
|
||||
};
|
||||
use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
|
||||
|
||||
pub struct ChatCompletions;
|
||||
|
||||
|
|
@ -37,44 +33,12 @@ impl ChatCompletionsRoute {
|
|||
call,
|
||||
observers,
|
||||
move |call, _, interceptors, observers| async move {
|
||||
self.run_call(call, cache_options, &interceptors, observers.as_ref())
|
||||
self.run(call, cache_options, &interceptors, observers.as_ref())
|
||||
.await
|
||||
.map(CallOutput::Complete)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.route", skip_all, fields(
|
||||
route = "chat_completions",
|
||||
model = %call.model,
|
||||
provider,
|
||||
resolved_model,
|
||||
stream = false,
|
||||
outcome
|
||||
))]
|
||||
pub(super) async fn run_call(
|
||||
&self,
|
||||
call: ChatCompletionsCall,
|
||||
cache_options: Option<litellm_cache_response::CachePolicy>,
|
||||
interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<ChatCompletionsResponse, Error> {
|
||||
litellm_inference::diagnostic::unary(async {
|
||||
let request = ChatCompletionsRequest {
|
||||
model: &call.model,
|
||||
messages: call.messages,
|
||||
optional_params: call.optional_params,
|
||||
api_key: call.api_key.as_deref(),
|
||||
api_base: call.api_base.as_deref(),
|
||||
custom_llm_provider: call.custom_llm_provider.as_deref(),
|
||||
extra_headers: call.extra_headers,
|
||||
timeout: call.timeout,
|
||||
};
|
||||
self.run(request, cache_options, interceptors, observers)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl litellm_inference::caching::Cachable for ChatCompletions {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
use litellm_secrets::source::Secrets;
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_auth::SecretValue;
|
||||
use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
|
||||
use litellm_llms_types::formats::chat_completions::ChatMessage;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
@ -49,31 +46,11 @@ impl From<ChatCompletionsRequest<'_>> for ChatCompletionsCall {
|
|||
}
|
||||
}
|
||||
|
||||
pub struct ResolvedChatCompletionsRequest<'a> {
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub config: &'static dyn BaseConfig,
|
||||
pub struct ResolvedChatCompletionsRequest {
|
||||
pub messages: Vec<ChatMessage>,
|
||||
pub optional_params: Map<String, Value>,
|
||||
pub api_key: Option<&'a str>,
|
||||
pub api_base: Option<&'a str>,
|
||||
pub api_key: Option<String>,
|
||||
pub api_base: Option<String>,
|
||||
pub extra_headers: Option<Map<String, Value>>,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
pub struct ProviderChatCompletionsRequest {
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub config: &'static dyn BaseConfig,
|
||||
pub url: String,
|
||||
pub body: Value,
|
||||
/// The route's parameters before the provider transformation, reported to the host
|
||||
/// beside the wire request.
|
||||
pub optional_params: Map<String, Value>,
|
||||
/// The forwarded and default headers plus how the call authenticates; the credential
|
||||
/// itself is applied when the request is sent.
|
||||
pub environment: ValidatedEnvironment,
|
||||
pub secrets: Secrets,
|
||||
pub timeout: Option<Duration>,
|
||||
pub api_key: Option<SecretValue>,
|
||||
}
|
||||
|
|
|
|||
439
litellm-rust/crates/inference-chat/tests/request.rs
Normal file
439
litellm-rust/crates/inference-chat/tests/request.rs
Normal file
|
|
@ -0,0 +1,439 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_inference_chat::{Error, types::ChatCompletionsRequest};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Map, Value, json};
|
||||
use wiremock::ResponseTemplate;
|
||||
|
||||
mod support;
|
||||
use support::*;
|
||||
|
||||
const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
|
||||
|
||||
fn request<'a>(
|
||||
model: &'a str,
|
||||
provider: Option<&'a str>,
|
||||
messages: Value,
|
||||
optional_params: Value,
|
||||
) -> ChatCompletionsRequest<'a> {
|
||||
let Value::Object(optional_params) = optional_params else {
|
||||
panic!("params must be an object");
|
||||
};
|
||||
ChatCompletionsRequest {
|
||||
model,
|
||||
messages,
|
||||
optional_params,
|
||||
api_key: Some("sk-test"),
|
||||
api_base: None,
|
||||
custom_llm_provider: provider,
|
||||
extra_headers: None,
|
||||
timeout: Some(Duration::from_secs(10)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn complete(
|
||||
request: ChatCompletionsRequest<'_>,
|
||||
) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletionsResponse, Error> {
|
||||
chat_completions_route().execute(request, &(), None).await
|
||||
}
|
||||
|
||||
fn object(value: Value) -> Map<String, Value> {
|
||||
let Value::Object(map) = value else {
|
||||
panic!("expected an object");
|
||||
};
|
||||
map
|
||||
}
|
||||
|
||||
fn anthropic_response() -> ResponseTemplate {
|
||||
ResponseTemplate::new(200).set_body_raw(ANTHROPIC_MESSAGE, "application/json")
|
||||
}
|
||||
|
||||
fn bedrock_response() -> ResponseTemplate {
|
||||
json_response(json!({
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}
|
||||
}))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn resolves_the_provider_from_the_model_prefix() {
|
||||
let upstream = upstream([anthropic_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
..request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({"max_tokens": 16}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
assert_eq!(sent.url.path(), "/v1/messages");
|
||||
assert_eq!(sent.json()["model"], "claude-sonnet-4-5");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn strips_an_explicit_provider_prefix_from_the_model() {
|
||||
let upstream = upstream([anthropic_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
..request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
assert_eq!(
|
||||
only_request(&upstream).await.json()["model"],
|
||||
"claude-sonnet-4-5"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn adds_the_auth_and_default_headers() {
|
||||
let upstream = upstream([anthropic_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
..request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
|
||||
assert_eq!(sent.header("anthropic-version"), Some("2023-06-01"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
|
||||
let upstream = upstream([anthropic_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
extra_headers: Some(object(json!({"X-Api-Key": "sk-caller"}))),
|
||||
..request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
assert_eq!(
|
||||
only_request(&upstream).await.header_values("x-api-key"),
|
||||
["sk-test"]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
|
||||
let upstream = upstream([anthropic_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
extra_headers: Some(object(json!({
|
||||
"Authorization": "Bearer sk-ant-oat01-token",
|
||||
"X-Api-Key": "sk-caller"
|
||||
}))),
|
||||
..request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
assert_eq!(sent.header_values("x-api-key"), ["sk-caller"]);
|
||||
assert_eq!(
|
||||
sent.header_values("authorization"),
|
||||
["Bearer sk-ant-oat01-token"]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
|
||||
let upstream = upstream([anthropic_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
extra_headers: Some(object(json!({
|
||||
"Authorization": "Bearer unrelated",
|
||||
"X-Api-Key": "sk-caller"
|
||||
}))),
|
||||
..request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
|
||||
assert_eq!(sent.header_values("authorization"), ["Bearer unrelated"]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn rejects_empty_messages_before_resolving_credentials() {
|
||||
let error = complete(ChatCompletionsRequest {
|
||||
api_key: None,
|
||||
..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
|
||||
})
|
||||
.await
|
||||
.expect_err("empty messages are rejected");
|
||||
assert!(matches!(error, Error::InvalidRequest(_)));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn rejects_an_unknown_provider() {
|
||||
assert_eq!(
|
||||
complete(request(
|
||||
"openai/gpt-4o",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
))
|
||||
.await
|
||||
.unwrap_err(),
|
||||
Error::InvalidProvider("openai".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn rejects_a_model_with_no_resolvable_provider() {
|
||||
assert!(matches!(
|
||||
complete(request(
|
||||
"claude-sonnet-4-5",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
))
|
||||
.await,
|
||||
Err(Error::InvalidProvider(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty(json!([]), Some("chat completions requires at least one message"))]
|
||||
#[case::malformed(json!("not a list"), None)]
|
||||
#[tokio::test]
|
||||
async fn rejects_an_empty_or_malformed_message_list(
|
||||
#[case] messages: Value,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let error = complete(request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
None,
|
||||
messages,
|
||||
json!({}),
|
||||
))
|
||||
.await
|
||||
.unwrap_err();
|
||||
if let Some(expected) = expected {
|
||||
assert_eq!(error, Error::InvalidRequest(expected.to_string().into()));
|
||||
} else {
|
||||
assert!(matches!(error, Error::InvalidRequest(_)));
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn rejects_non_string_extra_headers() {
|
||||
assert_eq!(
|
||||
complete(ChatCompletionsRequest {
|
||||
extra_headers: Some(Map::from_iter([("x-trace".to_string(), json!(7))])),
|
||||
..request(
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.unwrap_err(),
|
||||
Error::Headers(litellm_http::request::HeaderError {
|
||||
context: "chat completions",
|
||||
name: "x-trace".to_string(),
|
||||
actual: "number",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
|
||||
let upstream = upstream([bedrock_response()]).await;
|
||||
let base = upstream.uri();
|
||||
let mut call = request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({
|
||||
"maxTokens": 16,
|
||||
"aws_access_key_id": "AKIDEXAMPLE",
|
||||
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
|
||||
}),
|
||||
);
|
||||
call.api_key = None;
|
||||
call.api_base = Some(&base);
|
||||
call.extra_headers = Some(Map::from_iter([(
|
||||
"x-request-id".to_string(),
|
||||
json!("abc-123"),
|
||||
)]));
|
||||
complete(call).await.expect("call succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
let authorization = sent.header("authorization").expect("signed request");
|
||||
assert!(authorization.starts_with("AWS4-HMAC-SHA256"));
|
||||
assert!(!authorization.contains("x-request-id"));
|
||||
assert_eq!(sent.header("x-request-id"), Some("abc-123"));
|
||||
assert_eq!(sent.url.path(), "/model/anthropic.claude-v2/converse");
|
||||
let body: Value = serde_json::from_slice(&sent.body).unwrap();
|
||||
assert_eq!(body["inferenceConfig"], json!({"maxTokens": 16}));
|
||||
assert_eq!(
|
||||
body["messages"],
|
||||
json!([{"role": "user", "content": [{"text": "hi"}]}])
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::authorization("Authorization")]
|
||||
#[case::amz_date("x-amz-date")]
|
||||
#[case::security_token("x-amz-security-token")]
|
||||
#[case::date("Date")]
|
||||
#[tokio::test]
|
||||
async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
|
||||
let upstream = wiremock::MockServer::start().await;
|
||||
let base = upstream.uri();
|
||||
let call = ChatCompletionsRequest {
|
||||
api_key: None,
|
||||
api_base: Some(&base),
|
||||
extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
|
||||
..request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({
|
||||
"maxTokens": 16,
|
||||
"aws_access_key_id": "AKIDEXAMPLE",
|
||||
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
|
||||
}),
|
||||
)
|
||||
};
|
||||
let error = complete(call)
|
||||
.await
|
||||
.expect_err("conflicting signing headers fail");
|
||||
assert!(matches!(error, Error::Unsupported(_)), "{error:?}");
|
||||
assert!(received(&upstream).await.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
|
||||
let upstream = upstream([bedrock_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
extra_headers: Some(Map::from_iter([(
|
||||
"Authorization".to_string(),
|
||||
json!("Bearer caller-supplied"),
|
||||
)])),
|
||||
..request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({"maxTokens": 16}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
assert_eq!(
|
||||
only_request(&upstream).await.header_values("authorization"),
|
||||
["Bearer sk-test"]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
|
||||
let upstream = upstream([anthropic_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
extra_headers: Some(Map::from_iter([(
|
||||
"authorization".to_string(),
|
||||
json!("Bearer sk-ant-oat01-forwarded"),
|
||||
)])),
|
||||
..request(
|
||||
"claude-sonnet-4-5",
|
||||
Some("anthropic"),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
assert!(sent.header_values("x-api-key").is_empty());
|
||||
assert_eq!(
|
||||
sent.header_values("authorization"),
|
||||
["Bearer sk-ant-oat01-forwarded"]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
|
||||
let upstream = upstream([bedrock_response()]).await;
|
||||
let base = upstream.uri();
|
||||
complete(ChatCompletionsRequest {
|
||||
api_base: Some(&base),
|
||||
..request(
|
||||
"bedrock/us-east-1/anthropic.claude-v2",
|
||||
None,
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
json!({"maxTokens": 16}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("call succeeds");
|
||||
|
||||
assert_eq!(
|
||||
only_request(&upstream).await.header_values("authorization"),
|
||||
["Bearer sk-test"]
|
||||
);
|
||||
}
|
||||
|
|
@ -7,8 +7,8 @@
|
|||
- delegate authentication policy, beta selection, payload rewriting, and response interpretation to those adapters
|
||||
- keep provider policy out of request preparation and transport handlers
|
||||
- calling a concrete provider helper for every provider is still a policy dependency
|
||||
- Route types such as `MessagesCall`, prepared requests, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
|
||||
- Route types such as `MessagesCall`, provider calls, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
|
||||
- The route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks
|
||||
- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `src/lib.rs` explicitly sequences cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
|
||||
- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `handler::execute` explicitly sequences the provider call build in Python's handler order, cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
|
||||
- Preserve the order of validation, normalization, caller-requested parameter removal, and provider transformation when that order affects observable behavior
|
||||
- Test provider dispatch, auth precedence, header handling, transformations, and responses through behavior, not source structure
|
||||
|
|
|
|||
|
|
@ -82,17 +82,17 @@ mod tests {
|
|||
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
|
||||
|
||||
#[rstest]
|
||||
#[case::tencent("tencent")]
|
||||
#[case::anthropic("anthropic")]
|
||||
#[case::azure_ai("azure_ai")]
|
||||
#[case::bedrock("bedrock")]
|
||||
#[case::deepseek("deepseek")]
|
||||
#[case::edenai("edenai")]
|
||||
#[case::minimax("minimax")]
|
||||
#[case::vertex_ai("vertex_ai")]
|
||||
fn provider_keeps_its_python_name(#[case] name: &str) {
|
||||
let provider =
|
||||
messages_provider(name.parse::<LlmProviders>().unwrap(), "claude-sonnet-4-5").unwrap();
|
||||
#[case::tencent(LlmProviders::Tencent, "tencent")]
|
||||
#[case::anthropic(LlmProviders::Anthropic, "anthropic")]
|
||||
#[case::azure_ai(LlmProviders::AzureAi, "azure_ai")]
|
||||
#[case::bedrock(LlmProviders::Bedrock, "bedrock")]
|
||||
#[case::deepseek(LlmProviders::Deepseek, "deepseek")]
|
||||
#[case::edenai(LlmProviders::Edenai, "edenai")]
|
||||
#[case::minimax(LlmProviders::Minimax, "minimax")]
|
||||
#[case::vertex_ai(LlmProviders::VertexAi, "vertex_ai")]
|
||||
#[case::openrouter(LlmProviders::Openrouter, "openrouter")]
|
||||
fn provider_keeps_its_python_name(#[case] provider: LlmProviders, #[case] name: &str) {
|
||||
let provider = messages_provider(provider, "claude-sonnet-4-5").unwrap();
|
||||
assert_eq!(provider.as_str(), name);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,148 +1,271 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
|
||||
use litellm_host::interceptors::{Interceptors, ProviderIdentity, RequestContext, WireRequest};
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use futures_util::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream};
|
||||
use litellm_auth::{AuthServices, SecretValue};
|
||||
use litellm_core_utils::{
|
||||
dot_notation_indexing::delete_nested_value,
|
||||
get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
|
||||
};
|
||||
use litellm_host::interceptors::{
|
||||
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
|
||||
};
|
||||
use litellm_http::{request::with_default_headers, transport::Error as TransportError};
|
||||
use litellm_llms::base_llm::{
|
||||
auth::{Authenticated, resolve_auth},
|
||||
auth::{Authenticated, ValidatedEnvironment, resolve_auth},
|
||||
messages::{
|
||||
context::MessagesTransformContext,
|
||||
streaming::{self, ByteStream, StreamDecoder},
|
||||
transformation::BaseMessagesConfig,
|
||||
},
|
||||
};
|
||||
use litellm_llms_types::formats::messages::MessagesResponse;
|
||||
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
|
||||
use litellm_secrets::source::Secrets;
|
||||
use litellm_tracing::ByteChunk;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{
|
||||
Error, MessagesCallResponse, MessagesRoute, common_utils::truncate_error_body,
|
||||
prepare::ProviderMessagesRequest,
|
||||
Error, MessagesCall, MessagesCallResponse, MessagesRoute,
|
||||
common_utils::{MessagesProvider, string_headers, truncate_error_body},
|
||||
constants::MESSAGES_TIMEOUT_SECS,
|
||||
route::Messages,
|
||||
types::invalid_request,
|
||||
};
|
||||
use crate::constants::MESSAGES_TIMEOUT_SECS;
|
||||
use litellm_inference::{context::CallContext, outbound::outbound_request};
|
||||
use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request};
|
||||
|
||||
pub(super) struct ProviderCall {
|
||||
pub identity: ProviderIdentity,
|
||||
pub wire: WireRequest,
|
||||
pub(super) provider: super::common_utils::MessagesProvider,
|
||||
struct ProviderCall {
|
||||
identity: ProviderIdentity,
|
||||
wire: WireRequest,
|
||||
provider: MessagesProvider,
|
||||
signer: Option<litellm_auth_aws::SigV4Signer>,
|
||||
timeout: Option<Duration>,
|
||||
stream: bool,
|
||||
}
|
||||
|
||||
impl ProviderCall {
|
||||
pub fn cacheable(&self) -> bool {
|
||||
fn cacheable(&self) -> bool {
|
||||
self.signer.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
impl MessagesRoute {
|
||||
pub(super) async fn prepare_outbound(
|
||||
&self,
|
||||
request: ProviderMessagesRequest,
|
||||
context: &CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<ProviderCall, Error> {
|
||||
let ProviderMessagesRequest {
|
||||
provider,
|
||||
url,
|
||||
body,
|
||||
environment,
|
||||
timeout,
|
||||
api_key,
|
||||
} = request;
|
||||
let request_context = RequestContext {
|
||||
model: body.model.clone(),
|
||||
custom_llm_provider: provider.as_str().to_string(),
|
||||
optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
|
||||
secret_fields: Vec::new(),
|
||||
api_key,
|
||||
};
|
||||
let authenticated =
|
||||
resolve_auth(&self.auth, environment, &|key| std::env::var(key).ok()).await?;
|
||||
let identity = ProviderIdentity {
|
||||
model: request_context.model.clone(),
|
||||
provider: request_context.custom_llm_provider.clone(),
|
||||
};
|
||||
let wire = context
|
||||
.interceptors
|
||||
.before_provider_request(
|
||||
WireRequest {
|
||||
url,
|
||||
headers: authenticated.headers,
|
||||
body: provider
|
||||
.config()
|
||||
.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
|
||||
},
|
||||
request_context,
|
||||
)
|
||||
.await?;
|
||||
let stream = match wire.body.get("stream") {
|
||||
None | Some(Value::Null) => false,
|
||||
Some(Value::Bool(stream)) => *stream,
|
||||
Some(value) => {
|
||||
return Err(Error::InvalidRequest(
|
||||
litellm_llms::ErrorDetail::InvalidValue {
|
||||
field: "stream",
|
||||
expected: "a boolean",
|
||||
actual: value.clone(),
|
||||
},
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok(ProviderCall {
|
||||
identity,
|
||||
wire,
|
||||
provider,
|
||||
signer: authenticated.signer,
|
||||
timeout,
|
||||
stream,
|
||||
pub(super) async fn execute(
|
||||
route: &MessagesRoute,
|
||||
provider: MessagesProvider,
|
||||
model: String,
|
||||
call: MessagesCall,
|
||||
secrets: Secrets,
|
||||
context: &CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<MessagesCallResponse, Error> {
|
||||
let request = provider_call(
|
||||
&route.auth,
|
||||
provider,
|
||||
model,
|
||||
call,
|
||||
secrets.as_ref(),
|
||||
context,
|
||||
)
|
||||
.boxed()
|
||||
.await?;
|
||||
let cache = CallCache::<Messages>::from_wire(
|
||||
route.cache.as_ref().filter(|_| request.cacheable()),
|
||||
context.cache,
|
||||
&request.identity,
|
||||
&request.wire,
|
||||
);
|
||||
let identity = request.identity.clone();
|
||||
let (output, source) = match cache.lookup().await {
|
||||
Some(hit) => hit,
|
||||
None => (
|
||||
call_provider(&route.http, request, context).await?,
|
||||
ResultSource::Provider,
|
||||
),
|
||||
};
|
||||
let reported = match &output {
|
||||
MessagesCallResponse::Complete(response) => provider.config().reported_cost(response),
|
||||
MessagesCallResponse::Stream { .. } => None,
|
||||
};
|
||||
context
|
||||
.result_ready(ExecutionFacts {
|
||||
provider: identity,
|
||||
cost: Cost::settle(&source, reported),
|
||||
source: source.clone(),
|
||||
})
|
||||
}
|
||||
.await?;
|
||||
Ok(cache.finish(output, &source).await)
|
||||
}
|
||||
|
||||
pub(super) async fn call_provider(
|
||||
&self,
|
||||
request: ProviderCall,
|
||||
context: &CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<MessagesCallResponse, Error> {
|
||||
let ProviderCall {
|
||||
identity,
|
||||
wire,
|
||||
provider,
|
||||
signer,
|
||||
timeout,
|
||||
stream,
|
||||
} = request;
|
||||
let provider_name = provider.as_str();
|
||||
log_request_body(provider_name, stream, &wire.body);
|
||||
let response = send(
|
||||
&self.http,
|
||||
Authenticated {
|
||||
headers: wire.headers,
|
||||
signer,
|
||||
async fn provider_call(
|
||||
auth: &AuthServices,
|
||||
provider: MessagesProvider,
|
||||
model: String,
|
||||
call: MessagesCall,
|
||||
secrets: &(dyn Lookup + Send + Sync),
|
||||
context: &CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<ProviderCall, Error> {
|
||||
let MessagesCall {
|
||||
body,
|
||||
api_key,
|
||||
api_base,
|
||||
litellm_params,
|
||||
extra_headers,
|
||||
provider_specific_header,
|
||||
timeout,
|
||||
shaping,
|
||||
..
|
||||
} = call;
|
||||
let connection_params = litellm_router_types::LitellmParams {
|
||||
api_base: api_base.clone().or(litellm_params.api_base.clone()),
|
||||
..litellm_params
|
||||
};
|
||||
let config = provider.config();
|
||||
let env_lookup = |key: &str| secrets.get(key);
|
||||
|
||||
let sanitized = config.shape_request(
|
||||
MessagesRequest { model, ..body },
|
||||
shaping.settings.reasoning_auto_summary,
|
||||
)?;
|
||||
let trimmed =
|
||||
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
|
||||
|
||||
let scoped =
|
||||
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
|
||||
let forwarded = string_headers(Some(
|
||||
extra_headers.into_iter().flatten().chain(scoped).collect(),
|
||||
))?;
|
||||
let validated = config.validate_environment(
|
||||
forwarded,
|
||||
api_key.as_deref(),
|
||||
&trimmed.model,
|
||||
&trimmed.params,
|
||||
&connection_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
|
||||
let body = config.transform_anthropic_messages_request(
|
||||
trimmed,
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: config.request_headers(
|
||||
with_default_headers(validated.headers, config.default_headers()),
|
||||
&body,
|
||||
),
|
||||
auth: validated.auth,
|
||||
};
|
||||
|
||||
let url = config.get_complete_url(
|
||||
api_base.as_deref(),
|
||||
&body.model,
|
||||
&connection_params,
|
||||
body.params.stream == Some(true),
|
||||
&env_lookup,
|
||||
)?;
|
||||
|
||||
let request_context = RequestContext {
|
||||
model: body.model.clone(),
|
||||
custom_llm_provider: provider.as_str().to_string(),
|
||||
optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
|
||||
secret_fields: Vec::new(),
|
||||
api_key: api_key.map(SecretValue::new),
|
||||
};
|
||||
let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
|
||||
let identity = ProviderIdentity {
|
||||
model: request_context.model.clone(),
|
||||
provider: request_context.custom_llm_provider.clone(),
|
||||
};
|
||||
let wire = context
|
||||
.interceptors
|
||||
.before_provider_request(
|
||||
WireRequest {
|
||||
url,
|
||||
headers: authenticated.headers,
|
||||
body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
|
||||
},
|
||||
&wire.url,
|
||||
&wire.body,
|
||||
timeout,
|
||||
request_context,
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(provider_error(response).await);
|
||||
}
|
||||
let config = provider.config();
|
||||
if stream {
|
||||
return Ok(streaming_response(
|
||||
response,
|
||||
config.stream_decoder(),
|
||||
provider_name,
|
||||
let stream = match wire.body.get("stream") {
|
||||
None | Some(Value::Null) => false,
|
||||
Some(Value::Bool(stream)) => *stream,
|
||||
Some(value) => {
|
||||
return Err(Error::InvalidRequest(
|
||||
litellm_llms::ErrorDetail::InvalidValue {
|
||||
field: "stream",
|
||||
expected: "a boolean",
|
||||
actual: value.clone(),
|
||||
},
|
||||
));
|
||||
}
|
||||
let text = response.text().await.map_err(network)?;
|
||||
log_response_body(&text);
|
||||
context.response_received(&text).await?;
|
||||
decode_response(config, &identity.model, &text)
|
||||
.map(|message| MessagesCallResponse::Complete(Box::new(message)))
|
||||
};
|
||||
Ok(ProviderCall {
|
||||
identity,
|
||||
wire,
|
||||
provider,
|
||||
signer: authenticated.signer,
|
||||
timeout: timeout.or_else(|| shaping.settings.request_timeout()),
|
||||
stream,
|
||||
})
|
||||
}
|
||||
|
||||
fn without_additional_drop_params(
|
||||
request: MessagesRequest,
|
||||
paths: &[String],
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
if paths.is_empty() {
|
||||
return Ok(request);
|
||||
}
|
||||
let params = serde_json::to_value(request.params).map_err(invalid_request)?;
|
||||
let trimmed = paths
|
||||
.iter()
|
||||
.fold(params, |params, path| delete_nested_value(params, path));
|
||||
Ok(MessagesRequest {
|
||||
params: serde_json::from_value(trimmed).map_err(invalid_request)?,
|
||||
..request
|
||||
})
|
||||
}
|
||||
|
||||
async fn call_provider(
|
||||
http: &litellm_http::Client,
|
||||
request: ProviderCall,
|
||||
context: &CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<MessagesCallResponse, Error> {
|
||||
let ProviderCall {
|
||||
identity,
|
||||
wire,
|
||||
provider,
|
||||
signer,
|
||||
timeout,
|
||||
stream,
|
||||
} = request;
|
||||
let provider_name = provider.as_str();
|
||||
log_request_body(provider_name, stream, &wire.body);
|
||||
let response = send(
|
||||
http,
|
||||
Authenticated {
|
||||
headers: wire.headers,
|
||||
signer,
|
||||
},
|
||||
&wire.url,
|
||||
&wire.body,
|
||||
timeout,
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(provider_error(response).await);
|
||||
}
|
||||
let config = provider.config();
|
||||
if stream {
|
||||
return Ok(streaming_response(
|
||||
response,
|
||||
config.stream_decoder(),
|
||||
provider_name,
|
||||
));
|
||||
}
|
||||
let text = response.text().await.map_err(network)?;
|
||||
log_response_body(&text);
|
||||
context.response_received(&text).await?;
|
||||
decode_response(config, &identity.model, &text)
|
||||
.map(|message| MessagesCallResponse::Complete(Box::new(message)))
|
||||
}
|
||||
|
||||
fn serialize_failure(err: serde_json::Error) -> Error {
|
||||
|
|
@ -271,10 +394,482 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use litellm_llms_types::headers::ProviderSpecificHeaders;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Map, Value, json};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
|
||||
|
||||
use super::*;
|
||||
use crate::{MessagesSettings, MessagesShaping, common_utils::messages_provider};
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
|
||||
#[fixture]
|
||||
fn shaping() -> MessagesShaping {
|
||||
MessagesShaping::default()
|
||||
}
|
||||
|
||||
fn body(value: Value) -> MessagesRequest {
|
||||
serde_json::from_value(value).unwrap()
|
||||
}
|
||||
|
||||
async fn provider_call_for(call: MessagesCall) -> Result<ProviderCall, Error> {
|
||||
provider_call_with_secrets(call, &|_: &str| None).await
|
||||
}
|
||||
|
||||
async fn provider_call_with_secrets(
|
||||
call: MessagesCall,
|
||||
secrets: &(dyn Lookup + Send + Sync),
|
||||
) -> Result<ProviderCall, Error> {
|
||||
let provider = resolve_llm_provider(
|
||||
&call.body.model,
|
||||
call.custom_llm_provider.as_deref(),
|
||||
"messages",
|
||||
)?;
|
||||
let config = messages_provider(provider.provider, provider.model)
|
||||
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?;
|
||||
let model = provider.model.to_owned();
|
||||
let interceptors = ();
|
||||
let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default());
|
||||
provider_call(
|
||||
&AuthServices::default(),
|
||||
config,
|
||||
model,
|
||||
call,
|
||||
secrets,
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn wire_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
|
||||
provider_call_for(MessagesCall {
|
||||
body: body(fields),
|
||||
api_key: Some("sk-test".into()),
|
||||
api_base: Some("https://anthropic.test".into()),
|
||||
custom_llm_provider: Some("anthropic".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping,
|
||||
})
|
||||
.await
|
||||
.map(|call| call.wire.body)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::api_key(
|
||||
&[("ANTHROPIC_API_KEY", "sk-secret")],
|
||||
&[("x-api-key", "sk-secret")],
|
||||
"https://api.anthropic.com/v1/messages"
|
||||
)]
|
||||
#[case::auth_token(
|
||||
&[("ANTHROPIC_AUTH_TOKEN", "token")],
|
||||
&[("authorization", "Bearer token")],
|
||||
"https://api.anthropic.com/v1/messages"
|
||||
)]
|
||||
#[case::api_base(
|
||||
&[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
|
||||
&[("x-api-key", "sk-secret")],
|
||||
"https://gateway.test/v1/messages"
|
||||
)]
|
||||
#[case::sdk_base_url(
|
||||
&[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
|
||||
&[("x-api-key", "sk-secret")],
|
||||
"https://sdk.test/v1/messages"
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn credentials_and_base_come_from_the_resolved_secrets(
|
||||
shaping: MessagesShaping,
|
||||
#[case] secrets: &[(&str, &str)],
|
||||
#[case] expected_auth: &[(&str, &str)],
|
||||
#[case] expected_url: &str,
|
||||
) {
|
||||
let lookup = |name: &str| {
|
||||
secrets
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
};
|
||||
let call = provider_call_with_secrets(
|
||||
MessagesCall {
|
||||
body: body(
|
||||
json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
|
||||
),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
custom_llm_provider: Some("anthropic".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping,
|
||||
},
|
||||
&lookup,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let auth: Vec<(&str, &str)> = call
|
||||
.wire
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
|
||||
.map(|(name, value)| (name.as_str(), value.as_str()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
(auth.as_slice(), call.wire.url.as_str()),
|
||||
(expected_auth, expected_url)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn missing_credentials_are_reported_before_the_request_is_transformed(
|
||||
shaping: MessagesShaping,
|
||||
) {
|
||||
let result = provider_call_for(MessagesCall {
|
||||
body: body(
|
||||
json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}),
|
||||
),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
custom_llm_provider: Some("anthropic".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping,
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(Error::Auth(litellm_auth::Error::MissingApiKey { .. }))
|
||||
),
|
||||
"{:?}",
|
||||
result.err()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::nothing_configured(None, None, None)]
|
||||
#[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
|
||||
#[case::call_timeout_wins(
|
||||
Some(Duration::from_secs(30)),
|
||||
Some(1800.0),
|
||||
Some(Duration::from_secs(30))
|
||||
)]
|
||||
#[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
|
||||
#[case::unusable_setting_ignored(None, Some(-1.0), None)]
|
||||
#[case::zero_setting_ignored(None, Some(0.0), None)]
|
||||
#[tokio::test]
|
||||
async fn timeout_falls_back_to_the_configured_request_timeout(
|
||||
shaping: MessagesShaping,
|
||||
#[case] call_timeout: Option<Duration>,
|
||||
#[case] request_timeout: Option<f64>,
|
||||
#[case] expected: Option<Duration>,
|
||||
) {
|
||||
let call = provider_call_for(MessagesCall {
|
||||
body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
|
||||
api_key: Some("sk-test".into()),
|
||||
api_base: Some("https://anthropic.test".into()),
|
||||
custom_llm_provider: Some("anthropic".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: call_timeout,
|
||||
shaping: MessagesShaping {
|
||||
settings: MessagesSettings { request_timeout, ..shaping.settings },
|
||||
..shaping
|
||||
},
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(call.timeout, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::top_level_and_nested_paths(
|
||||
json!({
|
||||
"max_tokens": 1024,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048},
|
||||
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
|
||||
"metadata": {"user_id": "u1"},
|
||||
"tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
|
||||
}),
|
||||
&["thinking", "context_management", "tools[*].input_examples"],
|
||||
json!({
|
||||
"max_tokens": 1024,
|
||||
"metadata": {"user_id": "u1"},
|
||||
"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
|
||||
}),
|
||||
)]
|
||||
#[case::no_paths(
|
||||
json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
|
||||
&[],
|
||||
json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
|
||||
)]
|
||||
#[case::model_and_messages_are_never_dropped(
|
||||
json!({"max_tokens": 16}),
|
||||
&["model", "messages", "messages[0].content"],
|
||||
json!({"max_tokens": 16}),
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn wire_body_drops_configured_paths(
|
||||
shaping: MessagesShaping,
|
||||
#[case] fields: Value,
|
||||
#[case] additional_drop_params: &[&str],
|
||||
#[case] expected_fields: Value,
|
||||
) {
|
||||
let with_messages = |fields: Value| -> Value {
|
||||
let Value::Object(fields) = fields else {
|
||||
unreachable!()
|
||||
};
|
||||
Value::Object(
|
||||
[
|
||||
("model".to_string(), json!("claude-test")),
|
||||
(
|
||||
"messages".to_string(),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.chain(fields)
|
||||
.collect(),
|
||||
)
|
||||
};
|
||||
let shaping = MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
additional_drop_params: additional_drop_params
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect(),
|
||||
..shaping.settings
|
||||
},
|
||||
..shaping
|
||||
};
|
||||
assert_eq!(
|
||||
wire_body(with_messages(fields), shaping).await,
|
||||
Ok(with_messages(expected_fields))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::model_prefix_picks_the_provider(
|
||||
"azure_ai/claude-test",
|
||||
None,
|
||||
&[("x-priority", "extra"), ("x-scoped", "azure_ai")]
|
||||
)]
|
||||
#[case::explicit_provider(
|
||||
"claude-test",
|
||||
Some("anthropic"),
|
||||
&[("x-priority", "scoped"), ("x-scoped", "anthropic")]
|
||||
)]
|
||||
#[case::provider_prefix_on_an_anthropic_model(
|
||||
"anthropic/claude-test",
|
||||
None,
|
||||
&[("x-priority", "scoped"), ("x-scoped", "anthropic")]
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn provider_specific_headers_follow_the_resolved_provider(
|
||||
shaping: MessagesShaping,
|
||||
#[case] model: &str,
|
||||
#[case] custom_llm_provider: Option<&str>,
|
||||
#[case] expected: &[(&str, &str)],
|
||||
) {
|
||||
let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
|
||||
{"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
|
||||
{"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
|
||||
]))
|
||||
.unwrap();
|
||||
let call = provider_call_for(MessagesCall {
|
||||
body: body(
|
||||
json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
|
||||
),
|
||||
api_key: Some("sk-test".into()),
|
||||
api_base: Some("https://resource.services.ai.azure.com".into()),
|
||||
custom_llm_provider: custom_llm_provider.map(Into::into),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
|
||||
provider_specific_header: Some(configured),
|
||||
timeout: None,
|
||||
shaping,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let caller_headers: Vec<(&str, &str)> = call
|
||||
.wire
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
|
||||
.map(|(name, value)| (name.as_str(), value.as_str()))
|
||||
.collect();
|
||||
assert_eq!(caller_headers, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn wire_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
|
||||
assert_eq!(
|
||||
wire_body(
|
||||
json!({
|
||||
"model": "anthropic/claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16
|
||||
}),
|
||||
shaping,
|
||||
)
|
||||
.await,
|
||||
Ok(json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
|
||||
let shaping = MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
reasoning_auto_summary: true,
|
||||
additional_drop_params: vec!["thinking.display".to_string()],
|
||||
..shaping.settings
|
||||
},
|
||||
..shaping
|
||||
};
|
||||
assert_eq!(
|
||||
wire_body(
|
||||
json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 4096,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048}
|
||||
}),
|
||||
shaping,
|
||||
)
|
||||
.await,
|
||||
Ok(json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 4096,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048}
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(
|
||||
shaping: MessagesShaping,
|
||||
) {
|
||||
let shaping = MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
additional_drop_params: vec!["metadata.user_id".to_string()],
|
||||
..shaping.settings
|
||||
},
|
||||
..shaping
|
||||
};
|
||||
assert!(matches!(
|
||||
wire_body(
|
||||
json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16,
|
||||
"metadata": {"user_id": 123}
|
||||
}),
|
||||
shaping,
|
||||
)
|
||||
.await,
|
||||
Err(Error::InvalidRequest(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn wire_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
|
||||
assert_eq!(
|
||||
wire_body(
|
||||
json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16,
|
||||
"metadata": {"user_id": 123}
|
||||
}),
|
||||
shaping,
|
||||
)
|
||||
.await,
|
||||
Err(Error::InvalidRequest(
|
||||
litellm_llms::ErrorDetail::InvalidValue {
|
||||
field: "metadata.user_id",
|
||||
expected: "a string",
|
||||
actual: json!(123),
|
||||
}
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::mantle(
|
||||
"bedrock_mantle/claude-test",
|
||||
"https://bedrock-mantle.us-west-2.api.aws",
|
||||
"/anthropic/v1/messages",
|
||||
"bedrock"
|
||||
)]
|
||||
#[case::platform(
|
||||
"bedrock/claude_platform/claude-test",
|
||||
"https://aws-external-anthropic.us-west-2.api.aws",
|
||||
"/v1/messages",
|
||||
"aws-external-anthropic"
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn explicit_endpoint_region_is_used_by_the_signer(
|
||||
shaping: MessagesShaping,
|
||||
#[case] model: &str,
|
||||
#[case] base: &str,
|
||||
#[case] path: &str,
|
||||
#[case] service: &str,
|
||||
) {
|
||||
let call = provider_call_with_secrets(MessagesCall {
|
||||
body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
|
||||
api_key: None,
|
||||
api_base: Some(base.into()),
|
||||
custom_llm_provider: None,
|
||||
litellm_params: crate::litellm_params(Map::from_iter([
|
||||
("model".into(), json!(model)),
|
||||
("aws_access_key_id".into(), json!("AKIDEXAMPLE")),
|
||||
("aws_secret_access_key".into(), json!("secret")),
|
||||
]))
|
||||
.unwrap(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping,
|
||||
}, &|name: &str| match name {
|
||||
"AWS_REGION_NAME" => Some("us-east-1".into()),
|
||||
"ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
|
||||
_ => None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(call.wire.url, format!("{base}{path}"));
|
||||
let signed = outbound_request(
|
||||
Authenticated {
|
||||
headers: call.wire.headers,
|
||||
signer: call.signer,
|
||||
},
|
||||
call.wire.url,
|
||||
&call.wire.body,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let authorization = signed.header("authorization").unwrap_or_default();
|
||||
assert!(
|
||||
authorization.contains(&format!("/us-west-2/{service}/aws4_request")),
|
||||
"{authorization}"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::event(
|
||||
|
|
|
|||
|
|
@ -1,18 +1,19 @@
|
|||
mod common_utils;
|
||||
mod constants;
|
||||
mod handler;
|
||||
mod prepare;
|
||||
pub mod route;
|
||||
mod types;
|
||||
|
||||
use futures_util::FutureExt;
|
||||
use litellm_auth::AuthServices;
|
||||
use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource};
|
||||
|
||||
use litellm_inference::{caching::CallCache, context::CallContext};
|
||||
use litellm_host::interceptors::Interceptors;
|
||||
use litellm_inference::context::CallContext;
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::common_utils::messages_provider;
|
||||
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use types::{
|
||||
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
|
||||
|
|
@ -74,39 +75,23 @@ impl MessagesRoute {
|
|||
context: CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<MessagesCallResponse, Error> {
|
||||
litellm_inference::diagnostic::call(async {
|
||||
let prepared = prepare::prepare(call, self.secrets.as_ref()).await?;
|
||||
let provider = resolve_llm_provider(
|
||||
&call.body.model,
|
||||
call.custom_llm_provider.as_deref(),
|
||||
"messages",
|
||||
)?;
|
||||
let config = messages_provider(provider.provider, provider.model).ok_or_else(|| {
|
||||
Error::InvalidProvider(<&str>::from(provider.provider).to_string())
|
||||
})?;
|
||||
litellm_inference::diagnostic::provider(
|
||||
&prepared.body.model,
|
||||
prepared.provider.as_str(),
|
||||
provider.model,
|
||||
<&'static str>::from(provider.provider),
|
||||
);
|
||||
let request = self.prepare_outbound(prepared, &context).boxed().await?;
|
||||
let cache = CallCache::<route::Messages>::from_wire(
|
||||
self.cache.as_ref().filter(|_| request.cacheable()),
|
||||
context.cache,
|
||||
&request.identity,
|
||||
&request.wire,
|
||||
);
|
||||
let identity = request.identity.clone();
|
||||
let config = request.provider.config();
|
||||
let (output, source) = match cache.lookup().await {
|
||||
Some(hit) => hit,
|
||||
None => (
|
||||
self.call_provider(request, &context).await?,
|
||||
ResultSource::Provider,
|
||||
),
|
||||
};
|
||||
let reported = match &output {
|
||||
MessagesCallResponse::Complete(response) => config.reported_cost(response),
|
||||
MessagesCallResponse::Stream { .. } => None,
|
||||
};
|
||||
context
|
||||
.result_ready(ExecutionFacts {
|
||||
provider: identity,
|
||||
cost: Cost::settle(&source, reported),
|
||||
source: source.clone(),
|
||||
})
|
||||
.await?;
|
||||
Ok(cache.finish(output, &source).await)
|
||||
let model = provider.model.to_owned();
|
||||
let secrets = self.secrets.resolve(config.config().secret_names()).await?;
|
||||
handler::execute(self, config, model, call, secrets, &context)
|
||||
.boxed()
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,554 +0,0 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_auth::SecretValue;
|
||||
use litellm_core_utils::{
|
||||
dot_notation_indexing::delete_nested_value,
|
||||
get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
|
||||
};
|
||||
use litellm_http::request::with_default_headers;
|
||||
use litellm_llms::base_llm::{
|
||||
auth::ValidatedEnvironment, messages::context::MessagesTransformContext,
|
||||
};
|
||||
use litellm_llms_types::formats::messages::MessagesRequest;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
|
||||
use super::{
|
||||
Error, MessagesCall,
|
||||
common_utils::{MessagesProvider, messages_provider, string_headers},
|
||||
types::invalid_request,
|
||||
};
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
|
||||
struct ResolvedProvider {
|
||||
model: String,
|
||||
provider: MessagesProvider,
|
||||
}
|
||||
|
||||
pub(super) struct ProviderMessagesRequest {
|
||||
pub(super) provider: MessagesProvider,
|
||||
pub(super) url: String,
|
||||
pub(super) body: MessagesRequest,
|
||||
pub(super) environment: ValidatedEnvironment,
|
||||
pub(super) timeout: Option<Duration>,
|
||||
/// The caller's own credential, reported to the host beside the wire request.
|
||||
pub(super) api_key: Option<SecretValue>,
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
|
||||
pub(super) async fn prepare(
|
||||
call: MessagesCall,
|
||||
secrets: &dyn SecretSource,
|
||||
) -> Result<ProviderMessagesRequest, Error> {
|
||||
let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
|
||||
let secrets = secrets
|
||||
.resolve(resolved.provider.config().secret_names())
|
||||
.await?;
|
||||
prepare_provider_request(call, resolved, secrets.as_ref())
|
||||
}
|
||||
|
||||
fn resolve_provider(
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Result<ResolvedProvider, Error> {
|
||||
let resolved = resolve_llm_provider(model, custom_llm_provider, "messages")?;
|
||||
let provider = messages_provider(resolved.provider, resolved.model)
|
||||
.ok_or_else(|| Error::InvalidProvider(<&str>::from(resolved.provider).to_string()))?;
|
||||
Ok(ResolvedProvider {
|
||||
model: resolved.model.to_string(),
|
||||
provider,
|
||||
})
|
||||
}
|
||||
|
||||
fn prepare_provider_request(
|
||||
call: MessagesCall,
|
||||
resolved: ResolvedProvider,
|
||||
secrets: &dyn Lookup,
|
||||
) -> Result<ProviderMessagesRequest, Error> {
|
||||
let ResolvedProvider { model, provider } = resolved;
|
||||
let MessagesCall {
|
||||
body,
|
||||
api_key,
|
||||
api_base,
|
||||
litellm_params,
|
||||
extra_headers,
|
||||
provider_specific_header,
|
||||
timeout,
|
||||
shaping,
|
||||
..
|
||||
} = call;
|
||||
let connection_params = litellm_router_types::LitellmParams {
|
||||
api_base: api_base.clone().or(litellm_params.api_base.clone()),
|
||||
..litellm_params
|
||||
};
|
||||
let config = provider.config();
|
||||
let env_lookup = |key: &str| secrets.get(key);
|
||||
|
||||
let sanitized = config.shape_request(
|
||||
MessagesRequest { model, ..body },
|
||||
shaping.settings.reasoning_auto_summary,
|
||||
)?;
|
||||
let trimmed =
|
||||
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
|
||||
let transformed = config.transform_anthropic_messages_request(
|
||||
trimmed,
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
|
||||
)?;
|
||||
|
||||
let scoped =
|
||||
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
|
||||
let forwarded = string_headers(Some(
|
||||
extra_headers.into_iter().flatten().chain(scoped).collect(),
|
||||
))?;
|
||||
let forwarded = config.request_headers(
|
||||
with_default_headers(forwarded, config.default_headers()),
|
||||
&transformed,
|
||||
);
|
||||
let environment = config.validate_environment(
|
||||
forwarded,
|
||||
api_key.as_deref(),
|
||||
&transformed.model,
|
||||
&connection_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
|
||||
let url = config.get_complete_url(
|
||||
api_base.as_deref(),
|
||||
&transformed.model,
|
||||
&connection_params,
|
||||
transformed.params.stream == Some(true),
|
||||
&env_lookup,
|
||||
)?;
|
||||
|
||||
Ok(ProviderMessagesRequest {
|
||||
provider,
|
||||
url,
|
||||
body: transformed,
|
||||
environment,
|
||||
timeout: timeout.or_else(|| shaping.settings.request_timeout()),
|
||||
api_key: api_key.map(SecretValue::new),
|
||||
})
|
||||
}
|
||||
|
||||
fn without_additional_drop_params(
|
||||
request: MessagesRequest,
|
||||
paths: &[String],
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
if paths.is_empty() {
|
||||
return Ok(request);
|
||||
}
|
||||
let params = serde_json::to_value(request.params).map_err(invalid_request)?;
|
||||
let trimmed = paths
|
||||
.iter()
|
||||
.fold(params, |params, path| delete_nested_value(params, path));
|
||||
Ok(MessagesRequest {
|
||||
params: serde_json::from_value(trimmed).map_err(invalid_request)?,
|
||||
..request
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_llms::base_llm::auth::resolve_auth;
|
||||
use litellm_llms_types::headers::ProviderSpecificHeaders;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use super::*;
|
||||
use crate::{MessagesSettings, MessagesShaping};
|
||||
|
||||
#[fixture]
|
||||
fn shaping() -> MessagesShaping {
|
||||
MessagesShaping::default()
|
||||
}
|
||||
|
||||
fn body(value: Value) -> MessagesRequest {
|
||||
serde_json::from_value(value).unwrap()
|
||||
}
|
||||
|
||||
fn prepare(call: MessagesCall) -> Result<ProviderMessagesRequest, Error> {
|
||||
prepare_with_secrets(call, &|_: &str| None)
|
||||
}
|
||||
|
||||
fn prepare_with_secrets(
|
||||
call: MessagesCall,
|
||||
secrets: &dyn Lookup,
|
||||
) -> Result<ProviderMessagesRequest, Error> {
|
||||
let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
|
||||
prepare_provider_request(call, resolved, secrets)
|
||||
}
|
||||
|
||||
/// The headers as they go on the wire, credential applied.
|
||||
fn wire_headers(prepared: &ProviderMessagesRequest) -> Vec<(String, String)> {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(resolve_auth(
|
||||
&litellm_auth::AuthServices::default(),
|
||||
prepared.environment.clone(),
|
||||
&|_| None,
|
||||
))
|
||||
.unwrap()
|
||||
.headers
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::api_key(
|
||||
&[("ANTHROPIC_API_KEY", "sk-secret")],
|
||||
&[("x-api-key", "sk-secret")],
|
||||
"https://api.anthropic.com/v1/messages"
|
||||
)]
|
||||
#[case::auth_token(
|
||||
&[("ANTHROPIC_AUTH_TOKEN", "token")],
|
||||
&[("authorization", "Bearer token")],
|
||||
"https://api.anthropic.com/v1/messages"
|
||||
)]
|
||||
#[case::api_base(
|
||||
&[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
|
||||
&[("x-api-key", "sk-secret")],
|
||||
"https://gateway.test/v1/messages"
|
||||
)]
|
||||
#[case::sdk_base_url(
|
||||
&[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
|
||||
&[("x-api-key", "sk-secret")],
|
||||
"https://sdk.test/v1/messages"
|
||||
)]
|
||||
fn credentials_and_base_come_from_the_resolved_secrets(
|
||||
shaping: MessagesShaping,
|
||||
#[case] secrets: &[(&str, &str)],
|
||||
#[case] expected_auth: &[(&str, &str)],
|
||||
#[case] expected_url: &str,
|
||||
) {
|
||||
let lookup = |name: &str| {
|
||||
secrets
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
};
|
||||
let prepared = prepare_with_secrets(
|
||||
MessagesCall {
|
||||
body: body(
|
||||
json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
|
||||
),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
custom_llm_provider: Some("anthropic".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping,
|
||||
},
|
||||
&lookup,
|
||||
)
|
||||
.unwrap();
|
||||
let headers = wire_headers(&prepared);
|
||||
let auth: Vec<(&str, &str)> = headers
|
||||
.iter()
|
||||
.filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
|
||||
.map(|(name, value)| (name.as_str(), value.as_str()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
(auth.as_slice(), prepared.url.as_str()),
|
||||
(expected_auth, expected_url)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::nothing_configured(None, None, None)]
|
||||
#[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
|
||||
#[case::call_timeout_wins(
|
||||
Some(Duration::from_secs(30)),
|
||||
Some(1800.0),
|
||||
Some(Duration::from_secs(30))
|
||||
)]
|
||||
#[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
|
||||
#[case::unusable_setting_ignored(None, Some(-1.0), None)]
|
||||
#[case::zero_setting_ignored(None, Some(0.0), None)]
|
||||
fn timeout_falls_back_to_the_configured_request_timeout(
|
||||
shaping: MessagesShaping,
|
||||
#[case] call_timeout: Option<Duration>,
|
||||
#[case] request_timeout: Option<f64>,
|
||||
#[case] expected: Option<Duration>,
|
||||
) {
|
||||
let prepared = prepare(MessagesCall {
|
||||
body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
|
||||
api_key: Some("sk-test".into()),
|
||||
api_base: Some("https://anthropic.test".into()),
|
||||
custom_llm_provider: Some("anthropic".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: call_timeout,
|
||||
shaping: MessagesShaping {
|
||||
settings: MessagesSettings { request_timeout, ..shaping.settings },
|
||||
..shaping
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(prepared.timeout, expected);
|
||||
}
|
||||
|
||||
fn prepared_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
|
||||
prepare(MessagesCall {
|
||||
body: body(fields),
|
||||
api_key: Some("sk-test".into()),
|
||||
api_base: Some("https://anthropic.test".into()),
|
||||
custom_llm_provider: Some("anthropic".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping,
|
||||
})
|
||||
.map(|prepared| serde_json::to_value(prepared.body).unwrap())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::top_level_and_nested_paths(
|
||||
json!({
|
||||
"max_tokens": 1024,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048},
|
||||
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
|
||||
"metadata": {"user_id": "u1"},
|
||||
"tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
|
||||
}),
|
||||
&["thinking", "context_management", "tools[*].input_examples"],
|
||||
json!({
|
||||
"max_tokens": 1024,
|
||||
"metadata": {"user_id": "u1"},
|
||||
"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
|
||||
}),
|
||||
)]
|
||||
#[case::no_paths(
|
||||
json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
|
||||
&[],
|
||||
json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
|
||||
)]
|
||||
#[case::model_and_messages_are_never_dropped(
|
||||
json!({"max_tokens": 16}),
|
||||
&["model", "messages", "messages[0].content"],
|
||||
json!({"max_tokens": 16}),
|
||||
)]
|
||||
fn prepared_body_drops_configured_paths(
|
||||
shaping: MessagesShaping,
|
||||
#[case] fields: Value,
|
||||
#[case] additional_drop_params: &[&str],
|
||||
#[case] expected_fields: Value,
|
||||
) {
|
||||
let with_messages = |fields: Value| -> Value {
|
||||
let Value::Object(fields) = fields else {
|
||||
unreachable!()
|
||||
};
|
||||
Value::Object(
|
||||
[
|
||||
("model".to_string(), json!("claude-test")),
|
||||
(
|
||||
"messages".to_string(),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.chain(fields)
|
||||
.collect(),
|
||||
)
|
||||
};
|
||||
let shaping = MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
additional_drop_params: additional_drop_params
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect(),
|
||||
..shaping.settings
|
||||
},
|
||||
..shaping
|
||||
};
|
||||
assert_eq!(
|
||||
prepared_body(with_messages(fields), shaping),
|
||||
Ok(with_messages(expected_fields))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::model_prefix_picks_the_provider(
|
||||
"azure_ai/claude-test",
|
||||
None,
|
||||
&[("x-priority", "extra"), ("x-scoped", "azure_ai")]
|
||||
)]
|
||||
#[case::explicit_provider(
|
||||
"claude-test",
|
||||
Some("anthropic"),
|
||||
&[("x-priority", "scoped"), ("x-scoped", "anthropic")]
|
||||
)]
|
||||
#[case::provider_prefix_on_an_anthropic_model(
|
||||
"anthropic/claude-test",
|
||||
None,
|
||||
&[("x-priority", "scoped"), ("x-scoped", "anthropic")]
|
||||
)]
|
||||
fn provider_specific_headers_follow_the_resolved_provider(
|
||||
shaping: MessagesShaping,
|
||||
#[case] model: &str,
|
||||
#[case] custom_llm_provider: Option<&str>,
|
||||
#[case] expected: &[(&str, &str)],
|
||||
) {
|
||||
let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
|
||||
{"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
|
||||
{"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
|
||||
]))
|
||||
.unwrap();
|
||||
let prepared = prepare(MessagesCall {
|
||||
body: body(
|
||||
json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
|
||||
),
|
||||
api_key: Some("sk-test".into()),
|
||||
api_base: Some("https://resource.services.ai.azure.com".into()),
|
||||
custom_llm_provider: custom_llm_provider.map(Into::into),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
|
||||
provider_specific_header: Some(configured),
|
||||
timeout: None,
|
||||
shaping,
|
||||
})
|
||||
.unwrap();
|
||||
let caller_headers: Vec<(&str, &str)> = prepared
|
||||
.environment
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
|
||||
.map(|(name, value)| (name.as_str(), value.as_str()))
|
||||
.collect();
|
||||
assert_eq!(caller_headers, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn prepared_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
|
||||
assert_eq!(
|
||||
prepared_body(
|
||||
json!({
|
||||
"model": "anthropic/claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16
|
||||
}),
|
||||
shaping,
|
||||
),
|
||||
Ok(json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
|
||||
let shaping = MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
reasoning_auto_summary: true,
|
||||
additional_drop_params: vec!["thinking.display".to_string()],
|
||||
..shaping.settings
|
||||
},
|
||||
..shaping
|
||||
};
|
||||
assert_eq!(
|
||||
prepared_body(
|
||||
json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 4096,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048}
|
||||
}),
|
||||
shaping,
|
||||
),
|
||||
Ok(json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 4096,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048}
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) {
|
||||
let shaping = MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
additional_drop_params: vec!["metadata.user_id".to_string()],
|
||||
..shaping.settings
|
||||
},
|
||||
..shaping
|
||||
};
|
||||
assert!(matches!(
|
||||
prepared_body(
|
||||
json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16,
|
||||
"metadata": {"user_id": 123}
|
||||
}),
|
||||
shaping,
|
||||
),
|
||||
Err(Error::InvalidRequest(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn prepared_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
|
||||
assert_eq!(
|
||||
prepared_body(
|
||||
json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16,
|
||||
"metadata": {"user_id": 123}
|
||||
}),
|
||||
shaping,
|
||||
),
|
||||
Err(Error::InvalidRequest(
|
||||
litellm_llms::ErrorDetail::InvalidValue {
|
||||
field: "metadata.user_id",
|
||||
expected: "a string",
|
||||
actual: json!(123),
|
||||
}
|
||||
))
|
||||
);
|
||||
}
|
||||
#[rstest]
|
||||
#[case::mantle(
|
||||
"bedrock_mantle/claude-test",
|
||||
"https://bedrock-mantle.us-west-2.api.aws",
|
||||
"/anthropic/v1/messages",
|
||||
"bedrock"
|
||||
)]
|
||||
#[case::platform(
|
||||
"bedrock/claude_platform/claude-test",
|
||||
"https://aws-external-anthropic.us-west-2.api.aws",
|
||||
"/v1/messages",
|
||||
"aws-external-anthropic"
|
||||
)]
|
||||
fn explicit_endpoint_region_is_used_by_the_signer(
|
||||
shaping: MessagesShaping,
|
||||
#[case] model: &str,
|
||||
#[case] base: &str,
|
||||
#[case] path: &str,
|
||||
#[case] service: &str,
|
||||
) {
|
||||
let prepared = prepare_with_secrets(MessagesCall {
|
||||
body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
|
||||
api_key: None,
|
||||
api_base: Some(base.into()),
|
||||
custom_llm_provider: None,
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping,
|
||||
}, &|name: &str| match name {
|
||||
"AWS_REGION_NAME" => Some("us-east-1".into()),
|
||||
"ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
|
||||
_ => None,
|
||||
}).unwrap();
|
||||
assert_eq!(prepared.url, format!("{base}{path}"));
|
||||
assert!(
|
||||
matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -44,8 +44,8 @@ pub fn consumed_optional_param_names(
|
|||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Result<Vec<&'static str>, Error> {
|
||||
let (model, config) = resolve_provider_config(model, custom_llm_provider)?;
|
||||
let provider_fields = config.get_supported_ocr_params(&model);
|
||||
let (provider, config) = resolve_provider_config(model, custom_llm_provider)?;
|
||||
let provider_fields = config.get_supported_ocr_params(provider.model);
|
||||
let auth_fields: &[&str] = match config {
|
||||
OcrConfigKind::AwsTextract | OcrConfigKind::AwsTextractAnalyze => AWS_AUTH_OPTION_FIELDS,
|
||||
OcrConfigKind::AzureAi
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
use litellm_host::observation::ObservationSender;
|
||||
use std::sync::Arc;
|
||||
|
||||
use litellm_host::interceptors::Interceptors;
|
||||
use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
|
||||
use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
|
||||
|
||||
use super::{
|
||||
handler::perform_ocr_request,
|
||||
types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest},
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct OcrRoute {
|
||||
client: OcrClient,
|
||||
}
|
||||
|
||||
impl OcrRoute {
|
||||
pub fn new(client: OcrClient) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
|
||||
pub async fn execute(
|
||||
&self,
|
||||
request: LiteLLMOcrRequest,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<ObservationSender>,
|
||||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
litellm_host::lifecycle::observe_unary(
|
||||
observers.clone(),
|
||||
self.run(request, interceptors, observers.as_ref()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.route", skip_all, fields(
|
||||
route = "ocr",
|
||||
model = %request.model,
|
||||
resolved_model = %request.model,
|
||||
provider = <&str>::from(request.config.provider()),
|
||||
stream = false,
|
||||
outcome
|
||||
))]
|
||||
pub(super) async fn run(
|
||||
&self,
|
||||
request: LiteLLMOcrRequest,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
litellm_inference::diagnostic::unary(async {
|
||||
let caller_document = matches!(&request.document, OcrDocumentInput::Document(_));
|
||||
let prepared = prepare_request_document(request).await?;
|
||||
let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
|
||||
Box::pin(perform_ocr_request(
|
||||
&self.client,
|
||||
prepared,
|
||||
interceptors,
|
||||
caller_document,
|
||||
observers,
|
||||
));
|
||||
execute.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
|
||||
async fn prepare_request_document(
|
||||
request: LiteLLMOcrRequest<OcrDocumentInput>,
|
||||
) -> Result<ResolvedOcrRequest, Error> {
|
||||
if let OcrDocumentInput::Document(_) = &request.document {
|
||||
return request.map_document(super::document::prepare_document);
|
||||
}
|
||||
let logger = litellm_tracing::Logger::current();
|
||||
let span = tracing::Span::current();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
logger.scope(|| span.in_scope(|| request.map_document(super::document::prepare_document)))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::DocumentTask(Arc::new(error)))?
|
||||
}
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
use std::{collections::BTreeMap as Map, io::Read, path::Path};
|
||||
use std::{collections::BTreeMap as Map, io::Read, path::Path, sync::Arc};
|
||||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use litellm_llms::base_llm::ocr::{error::Error, transformation::OCR_INLINE_MAX_BYTES};
|
||||
use litellm_llms_types::formats::ocr::OcrDocument;
|
||||
|
||||
use crate::types::OcrDocumentInput;
|
||||
use crate::types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest};
|
||||
|
||||
pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
|
||||
match input {
|
||||
|
|
@ -24,6 +24,21 @@ pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_document(
|
||||
request: LiteLLMOcrRequest<OcrDocumentInput>,
|
||||
) -> Result<ResolvedOcrRequest, Error> {
|
||||
if let OcrDocumentInput::Document(_) = &request.document {
|
||||
return request.map_document(prepare_document);
|
||||
}
|
||||
let logger = litellm_tracing::Logger::current();
|
||||
let span = tracing::Span::current();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
logger.scope(|| span.in_scope(|| request.map_document(prepare_document)))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::DocumentTask(Arc::new(error)))?
|
||||
}
|
||||
|
||||
pub fn read_path_document(path: &Path, mime_type: Option<&str>) -> Result<OcrDocument, Error> {
|
||||
let mut bytes = Vec::new();
|
||||
std::fs::File::open(path)
|
||||
|
|
|
|||
|
|
@ -4,29 +4,45 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender};
|
|||
use litellm_llms::base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient},
|
||||
transformation::PreparedOcrRequest,
|
||||
transformation::{OcrConnection, PreparedOcrRequest},
|
||||
};
|
||||
use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
|
||||
use litellm_secrets::source::Secrets;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind};
|
||||
use crate::types::ResolvedOcrRequest;
|
||||
use super::{arguments::is_secret_param, provider_config::OcrConfigKind};
|
||||
use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
|
||||
|
||||
pub(crate) async fn perform_ocr_request(
|
||||
pub(crate) async fn execute(
|
||||
client: &OcrClient,
|
||||
request: ResolvedOcrRequest,
|
||||
config: OcrConfigKind,
|
||||
call: ResolvedOcrRequest,
|
||||
secrets: Secrets,
|
||||
host: &impl Interceptors<Error>,
|
||||
caller_document: bool,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
request.response_format()?;
|
||||
let config = request.config;
|
||||
let secrets = client
|
||||
.secret_source()
|
||||
.resolve(&config.secret_names())
|
||||
.await
|
||||
.map_err(|error| Error::Secret(std::sync::Arc::new(error)))?;
|
||||
let request = prepare_request(request, caller_document, client, secrets);
|
||||
call.response_format()?;
|
||||
let LiteLLMOcrRequest {
|
||||
model,
|
||||
document,
|
||||
credentials,
|
||||
transport,
|
||||
optional_params,
|
||||
input_sources,
|
||||
azure_ad_token_provider,
|
||||
..
|
||||
} = call;
|
||||
let resolved = config.resolve_credentials(credentials, &secrets);
|
||||
let request = PreparedOcrRequest {
|
||||
model,
|
||||
document,
|
||||
connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
|
||||
caller_document,
|
||||
optional_params,
|
||||
input_sources,
|
||||
azure_ad_token_provider,
|
||||
};
|
||||
let interceptors = OcrCallHooks::new(host, &request, config, observers);
|
||||
config.ocr(client, &request, &interceptors).await
|
||||
}
|
||||
|
|
@ -86,3 +102,257 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
|
|||
Box::pin(self.interceptors.after_provider_response(raw))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
|
||||
use litellm_llms_types::formats::ocr::OcrResponseFormat;
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
|
||||
|
||||
use super::*;
|
||||
use crate::{
|
||||
OcrRoute,
|
||||
types::LiteLLMOcrRequest,
|
||||
wire::{OcrWireRequest, decode_request},
|
||||
};
|
||||
|
||||
fn client() -> OcrClient {
|
||||
OcrClient::for_test(
|
||||
litellm_http::Client::plain_for_test(),
|
||||
litellm_http::Client::no_redirect_for_test(),
|
||||
)
|
||||
.with_secrets(litellm_inference_testing::no_secrets())
|
||||
}
|
||||
|
||||
fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
|
||||
decode_request(OcrWireRequest {
|
||||
model: model.into(),
|
||||
document,
|
||||
api_key: Some(litellm_auth::SecretValue::new("test-key")),
|
||||
api_base: Some(base.into()),
|
||||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
optional_params: options.as_object().unwrap().clone(),
|
||||
input_sources: Default::default(),
|
||||
timeout_seconds: Some(2.0),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn run_request(request: LiteLLMOcrRequest) -> Result<LiteLLMOcrResponse, Error> {
|
||||
OcrRoute::new(client()).execute(request, &(), None).await
|
||||
}
|
||||
|
||||
async fn upstream(request_count: u64, response: Value) -> MockServer {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(any())
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(response))
|
||||
.up_to_n_times(request_count)
|
||||
.mount(&server)
|
||||
.await;
|
||||
server
|
||||
}
|
||||
|
||||
fn image(url: &str) -> Value {
|
||||
json!({"type": "image_url", "image_url": url})
|
||||
}
|
||||
|
||||
fn body(request: &wiremock::Request) -> Value {
|
||||
serde_json::from_slice(&request.body).unwrap()
|
||||
}
|
||||
|
||||
fn assert_mistral_request(request: &wiremock::Request) {
|
||||
assert_eq!(
|
||||
request
|
||||
.headers
|
||||
.get("authorization")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap(),
|
||||
"Bearer test-key"
|
||||
);
|
||||
assert_eq!(
|
||||
request
|
||||
.headers
|
||||
.get("content-type")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap(),
|
||||
"application/json"
|
||||
);
|
||||
assert_eq!(
|
||||
body(request),
|
||||
json!({
|
||||
"model": "mistral-ocr-maas",
|
||||
"document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
|
||||
"pages": [0, 2],
|
||||
"include_image_base64": true,
|
||||
"unknown": "preserved"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
|
||||
let upstream = upstream(
|
||||
1,
|
||||
json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
|
||||
)
|
||||
.await;
|
||||
let request = request(
|
||||
"cohere/parse",
|
||||
&upstream.uri(),
|
||||
image("https://example.com/original.png"),
|
||||
json!({
|
||||
"output_format": "markdown", "timeout": 30,
|
||||
"extra_body": {
|
||||
"output_format": {"future": true},
|
||||
"document": {"type": "image_url", "image_url": "https://example.com/a.png",
|
||||
"provider_options": {"nested": [false, 0, null]}}
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
run_request(request).await.unwrap();
|
||||
|
||||
let [request] = upstream
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_into()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
body(&request),
|
||||
json!({
|
||||
"model": "parse", "output_format": {"future": true},
|
||||
"document": {"type": "image_url", "image_url": "https://example.com/a.png",
|
||||
"provider_options": {"nested": [false, 0, null]}}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn explicit_null_options_use_defaults_before_http() {
|
||||
let upstream = upstream(
|
||||
1,
|
||||
json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
|
||||
)
|
||||
.await;
|
||||
let request = request(
|
||||
"cohere/parse",
|
||||
&upstream.uri(),
|
||||
image("https://example.com/a.png"),
|
||||
json!({"output_format": null, "req_format": null}),
|
||||
);
|
||||
assert_eq!(
|
||||
request.response_format().unwrap(),
|
||||
OcrResponseFormat::Litellm
|
||||
);
|
||||
|
||||
run_request(request).await.unwrap();
|
||||
|
||||
let [request] = upstream
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_into()
|
||||
.unwrap();
|
||||
let body = body(&request);
|
||||
assert_eq!(body["output_format"], "markdown");
|
||||
assert!(body.get("req_format").is_none());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
|
||||
let upstream = upstream(
|
||||
2,
|
||||
json!({
|
||||
"pages": [{"index": 0, "markdown": "hello"}],
|
||||
"extra": "preserved"
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let options = json!({
|
||||
"pages": [0, 2],
|
||||
"include_image_base64": true,
|
||||
"vertex_project": "project-1",
|
||||
"vertex_location": "us-central1",
|
||||
"unknown": "preserved"
|
||||
});
|
||||
let document =
|
||||
json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
|
||||
let direct_call = request(
|
||||
"mistral/mistral-ocr-maas",
|
||||
&upstream.uri(),
|
||||
document.clone(),
|
||||
options.clone(),
|
||||
);
|
||||
let vertex_call = request(
|
||||
"vertex_ai/mistral-ocr-maas",
|
||||
&upstream.uri(),
|
||||
document,
|
||||
options,
|
||||
);
|
||||
assert_eq!(direct_call.transport.timeout, Some(Duration::from_secs(2)));
|
||||
assert_eq!(vertex_call.transport.timeout, Some(Duration::from_secs(2)));
|
||||
let direct_response = run_request(direct_call).await.unwrap().into_json();
|
||||
let vertex_response = run_request(vertex_call).await.unwrap().into_json();
|
||||
|
||||
let [direct_http, vertex_http] = upstream
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_into()
|
||||
.unwrap();
|
||||
assert_eq!(direct_http.url.path(), "/v1/ocr");
|
||||
assert_eq!(
|
||||
vertex_http.url.path(),
|
||||
"/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
|
||||
);
|
||||
assert_mistral_request(&direct_http);
|
||||
assert_mistral_request(&vertex_http);
|
||||
assert_eq!(direct_response, vertex_response);
|
||||
assert_eq!(direct_response["model"], "mistral-ocr-maas");
|
||||
assert_eq!(direct_response["object"], "ocr");
|
||||
assert_eq!(direct_response["extra"], "preserved");
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct KnownParams {
|
||||
pages: Option<Vec<i64>>,
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn parsed_provider_params_separates_known_and_extra_params() {
|
||||
let arguments: CallArguments = serde_json::from_value(json!({
|
||||
"pages": [0, 2],
|
||||
"future_ocr_option": true,
|
||||
"extra_body": {"provider_option": "value"}
|
||||
}))
|
||||
.unwrap();
|
||||
let known: KnownParams = parse_options(&arguments).unwrap();
|
||||
assert_eq!(known.pages, Some(vec![0, 2]));
|
||||
assert_eq!(arguments["future_ocr_option"], true);
|
||||
assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
|
||||
assert_eq!(
|
||||
arguments
|
||||
.iter()
|
||||
.filter(|(name, _)| name.as_str() != "pages")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
|
||||
json!({
|
||||
"pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,82 @@
|
|||
pub mod arguments;
|
||||
mod client;
|
||||
pub use client::OcrRoute;
|
||||
pub mod document;
|
||||
pub(crate) mod handler;
|
||||
pub(crate) mod prepare;
|
||||
pub mod provider_config;
|
||||
pub mod route;
|
||||
pub mod types;
|
||||
pub mod wire;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use litellm_host::interceptors::Interceptors;
|
||||
use litellm_host::observation::ObservationSender;
|
||||
use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
|
||||
use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
|
||||
|
||||
use crate::{
|
||||
handler::execute,
|
||||
types::{LiteLLMOcrRequest, OcrDocumentInput},
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct OcrRoute {
|
||||
client: OcrClient,
|
||||
}
|
||||
|
||||
impl OcrRoute {
|
||||
pub fn new(client: OcrClient) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
|
||||
pub async fn execute(
|
||||
&self,
|
||||
request: LiteLLMOcrRequest,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<ObservationSender>,
|
||||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
litellm_host::lifecycle::observe_unary(
|
||||
observers.clone(),
|
||||
self.run(request, interceptors, observers.as_ref()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.route", skip_all, fields(
|
||||
route = "ocr",
|
||||
model = %call.model,
|
||||
resolved_model = %call.model,
|
||||
provider = <&str>::from(call.config.provider()),
|
||||
stream = false,
|
||||
outcome
|
||||
))]
|
||||
async fn run(
|
||||
&self,
|
||||
call: LiteLLMOcrRequest,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
litellm_inference::diagnostic::unary(async {
|
||||
let caller_document = matches!(&call.document, OcrDocumentInput::Document(_));
|
||||
let call = document::resolve_document(call).await?;
|
||||
let config = call.config;
|
||||
let secrets = self
|
||||
.client
|
||||
.secret_source()
|
||||
.resolve(&config.secret_names())
|
||||
.await
|
||||
.map_err(|error| Error::Secret(Arc::new(error)))?;
|
||||
let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
|
||||
Box::pin(execute(
|
||||
&self.client,
|
||||
config,
|
||||
call,
|
||||
secrets,
|
||||
interceptors,
|
||||
caller_document,
|
||||
observers,
|
||||
));
|
||||
execute.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,309 +0,0 @@
|
|||
use litellm_auth::{InputSource, SecretValue, Sourced};
|
||||
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
handler::OcrClient,
|
||||
transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
|
||||
};
|
||||
use litellm_secrets::source::Secrets;
|
||||
|
||||
use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
|
||||
|
||||
pub(crate) fn prepare_request(
|
||||
request: ResolvedOcrRequest,
|
||||
caller_document: bool,
|
||||
client: &OcrClient,
|
||||
secrets: Secrets,
|
||||
) -> PreparedOcrRequest {
|
||||
let credentials = request.credentials.clone();
|
||||
let (preferred_api_key_env, api_base_env) = match request.config.provider() {
|
||||
LlmProviders::Mistral => (
|
||||
Some("MISTRAL_AZURE_API_KEY"),
|
||||
Some("MISTRAL_AZURE_API_BASE"),
|
||||
),
|
||||
LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
|
||||
_ => (None, None),
|
||||
};
|
||||
let secret = |name: &str| secrets.truthy(name);
|
||||
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
|
||||
credentials.api_key.clone().or_else(|| {
|
||||
preferred_api_key_env
|
||||
.into_iter()
|
||||
.chain(request.config.get_api_key_env_var())
|
||||
.find_map(secret)
|
||||
.map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
|
||||
})
|
||||
});
|
||||
let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
|
||||
credentials.api_base.clone().or_else(|| {
|
||||
api_base_env
|
||||
.and_then(secret)
|
||||
.map(|value| Sourced::new(value, InputSource::Environment))
|
||||
})
|
||||
});
|
||||
let resolved = request
|
||||
.config
|
||||
.resolve_connection_params(OcrCredentialInputs {
|
||||
dynamic_api_key,
|
||||
dynamic_api_base,
|
||||
..credentials
|
||||
});
|
||||
let LiteLLMOcrRequest {
|
||||
model,
|
||||
document,
|
||||
transport,
|
||||
optional_params,
|
||||
input_sources,
|
||||
azure_ad_token_provider,
|
||||
..
|
||||
} = request;
|
||||
PreparedOcrRequest {
|
||||
model,
|
||||
document,
|
||||
connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
|
||||
caller_document,
|
||||
optional_params,
|
||||
input_sources,
|
||||
azure_ad_token_provider,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
|
||||
use litellm_host::interceptors::WireRequest;
|
||||
use litellm_llms::{
|
||||
base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient},
|
||||
transformation::BaseOcrConfig,
|
||||
},
|
||||
cohere::ocr::transformation::CohereParseConfig,
|
||||
mistral::ocr::transformation::MistralOcrConfig,
|
||||
vertex_ai::ocr::transformation::VertexAiOcrConfig,
|
||||
};
|
||||
use litellm_llms_types::formats::ocr::OcrResponseFormat;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::*;
|
||||
use crate::{
|
||||
document::prepare_document,
|
||||
types::LiteLLMOcrRequest,
|
||||
wire::{OcrWireRequest, decode_request},
|
||||
};
|
||||
|
||||
/// Stands in for a host with no interceptors registered.
|
||||
struct NoHooks;
|
||||
|
||||
impl CallHooks<Error> for NoHooks {
|
||||
fn before_provider_request(
|
||||
&self,
|
||||
wire: WireRequest,
|
||||
) -> BoxFuture<'_, Result<WireRequest, Error>> {
|
||||
Box::pin(async move { Ok(wire) })
|
||||
}
|
||||
|
||||
fn response_received<'a>(&'a self, _body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
fn client() -> OcrClient {
|
||||
OcrClient::for_test(
|
||||
litellm_http::Client::plain_for_test(),
|
||||
litellm_http::Client::no_redirect_for_test(),
|
||||
)
|
||||
}
|
||||
|
||||
fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
|
||||
decode_request(OcrWireRequest {
|
||||
model: model.into(),
|
||||
document,
|
||||
api_key: Some(litellm_auth::SecretValue::new("test-key")),
|
||||
api_base: Some(base.into()),
|
||||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
optional_params: options.as_object().unwrap().clone(),
|
||||
input_sources: Default::default(),
|
||||
timeout_seconds: Some(2.0),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn prepared(request: LiteLLMOcrRequest) -> PreparedOcrRequest {
|
||||
prepare_request(
|
||||
request.map_document(prepare_document).unwrap(),
|
||||
true,
|
||||
&client(),
|
||||
std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
|
||||
)
|
||||
}
|
||||
|
||||
fn image(url: &str) -> Value {
|
||||
json!({"type": "image_url", "image_url": url})
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
|
||||
let request = request(
|
||||
"cohere/parse",
|
||||
"https://example.com",
|
||||
image("https://example.com/original.png"),
|
||||
json!({
|
||||
"output_format": "markdown", "timeout": 30,
|
||||
"extra_body": {
|
||||
"output_format": {"future": true},
|
||||
"document": {"type": "image_url", "image_url": "https://example.com/a.png",
|
||||
"provider_options": {"nested": [false, 0, null]}}
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
let http = CohereParseConfig
|
||||
.prepare_request(&prepared(request), &client(), &NoHooks)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let body: Value = serde_json::from_slice(http.body()).unwrap();
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({
|
||||
"model": "parse", "output_format": {"future": true},
|
||||
"document": {"type": "image_url", "image_url": "https://example.com/a.png",
|
||||
"provider_options": {"nested": [false, 0, null]}}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn explicit_null_options_use_defaults_before_http() {
|
||||
let request = request(
|
||||
"cohere/parse",
|
||||
"https://example.com",
|
||||
image("https://example.com/a.png"),
|
||||
json!({"output_format": null, "req_format": null}),
|
||||
);
|
||||
assert_eq!(
|
||||
request.response_format().unwrap(),
|
||||
OcrResponseFormat::Litellm
|
||||
);
|
||||
|
||||
let http = CohereParseConfig
|
||||
.prepare_request(&prepared(request), &client(), &NoHooks)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let body: Value = serde_json::from_slice(http.body()).unwrap();
|
||||
assert_eq!(body["output_format"], "markdown");
|
||||
assert!(body.get("req_format").is_none());
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
|
||||
let options = json!({
|
||||
"pages": [0, 2],
|
||||
"include_image_base64": true,
|
||||
"vertex_project": "project-1",
|
||||
"vertex_location": "us-central1",
|
||||
"unknown": "preserved"
|
||||
});
|
||||
let document =
|
||||
json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
|
||||
let direct = prepared(request(
|
||||
"mistral/mistral-ocr-maas",
|
||||
"https://mistral.test",
|
||||
document.clone(),
|
||||
options.clone(),
|
||||
));
|
||||
let vertex = prepared(request(
|
||||
"vertex_ai/mistral-ocr-maas",
|
||||
"https://vertex.test",
|
||||
document,
|
||||
options,
|
||||
));
|
||||
|
||||
let direct_http = MistralOcrConfig
|
||||
.prepare_request(&direct, &client(), &NoHooks)
|
||||
.await
|
||||
.unwrap();
|
||||
let vertex_http = VertexAiOcrConfig
|
||||
.prepare_request(&vertex, &client(), &NoHooks)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(direct_http.url(), "https://mistral.test/v1/ocr");
|
||||
assert_eq!(
|
||||
vertex_http.url(),
|
||||
"https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
|
||||
);
|
||||
for http in [&direct_http, &vertex_http] {
|
||||
assert_eq!(http.header("authorization").unwrap(), "Bearer test-key");
|
||||
assert_eq!(http.header("content-type").unwrap(), "application/json");
|
||||
assert_eq!(http.timeout(), Some(Duration::from_secs(2)));
|
||||
let body: Value = serde_json::from_slice(http.body()).unwrap();
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({
|
||||
"model": "mistral-ocr-maas",
|
||||
"document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
|
||||
"pages": [0, 2],
|
||||
"include_image_base64": true,
|
||||
"unknown": "preserved"
|
||||
})
|
||||
);
|
||||
}
|
||||
let payload = serde_json::to_vec(
|
||||
&json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}),
|
||||
)
|
||||
.unwrap();
|
||||
let direct_response = MistralOcrConfig
|
||||
.transform_ocr_response(&direct.model, &payload, OcrResponseFormat::Litellm)
|
||||
.unwrap()
|
||||
.into_json();
|
||||
let vertex_response = VertexAiOcrConfig
|
||||
.transform_ocr_response(&vertex.model, &payload, OcrResponseFormat::Litellm)
|
||||
.unwrap()
|
||||
.into_json();
|
||||
assert_eq!(direct_response, vertex_response);
|
||||
assert_eq!(direct_response["model"], "mistral-ocr-maas");
|
||||
assert_eq!(direct_response["object"], "ocr");
|
||||
assert_eq!(direct_response["extra"], "preserved");
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct KnownParams {
|
||||
pages: Option<Vec<i64>>,
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
fn parsed_provider_params_separates_known_and_extra_params() {
|
||||
let arguments: CallArguments = serde_json::from_value(json!({
|
||||
"pages": [0, 2],
|
||||
"future_ocr_option": true,
|
||||
"extra_body": {"provider_option": "value"}
|
||||
}))
|
||||
.unwrap();
|
||||
let known: KnownParams = parse_options(&arguments).unwrap();
|
||||
assert_eq!(known.pages, Some(vec![0, 2]));
|
||||
assert_eq!(arguments["future_ocr_option"], true);
|
||||
assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
|
||||
assert_eq!(
|
||||
arguments
|
||||
.iter()
|
||||
.filter(|(name, _)| name.as_str() != "pages")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
|
||||
json!({
|
||||
"pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
|
||||
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
|
||||
use litellm_auth::{InputSource, SecretValue, Sourced};
|
||||
use litellm_core_utils::get_llm_provider_logic::{LlmProviders, get_custom_llm_provider};
|
||||
use litellm_inference::provider::{ResolvedProvider, resolve_llm_provider};
|
||||
use litellm_llms::{
|
||||
aws_textract::ocr::{
|
||||
analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation,
|
||||
|
|
@ -25,6 +26,7 @@ use litellm_llms::{
|
|||
},
|
||||
};
|
||||
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat};
|
||||
use litellm_secrets::source::Secrets;
|
||||
|
||||
macro_rules! with_config {
|
||||
($kind:expr, $config:ident => $body:expr) => {
|
||||
|
|
@ -115,7 +117,18 @@ impl OcrConfigKind {
|
|||
}
|
||||
|
||||
pub(crate) fn secret_names(self) -> Vec<&'static str> {
|
||||
with_config!(self, config => config.secret_names())
|
||||
let names = with_config!(self, config => config.secret_names());
|
||||
let fallback_names: &[&str] = match self.provider() {
|
||||
LlmProviders::Mistral => &["MISTRAL_AZURE_API_KEY", "MISTRAL_AZURE_API_BASE"],
|
||||
LlmProviders::AzureAi => &["AZURE_AI_API_BASE"],
|
||||
_ => &[],
|
||||
};
|
||||
let additional_names = fallback_names
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|name| !names.contains(name))
|
||||
.collect::<Vec<_>>();
|
||||
names.into_iter().chain(additional_names).collect()
|
||||
}
|
||||
|
||||
pub(crate) fn get_health_check_document(self) -> OcrDocument {
|
||||
|
|
@ -129,6 +142,43 @@ impl OcrConfigKind {
|
|||
with_config!(self, config => config.resolve_connection_params(inputs))
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_credentials(
|
||||
self,
|
||||
credentials: OcrCredentialInputs,
|
||||
secrets: &Secrets,
|
||||
) -> ResolvedOcrCredentials {
|
||||
let (preferred_api_key_env, api_base_env) = match self.provider() {
|
||||
LlmProviders::Mistral => (
|
||||
Some("MISTRAL_AZURE_API_KEY"),
|
||||
Some("MISTRAL_AZURE_API_BASE"),
|
||||
),
|
||||
LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
|
||||
_ => (None, None),
|
||||
};
|
||||
let secret = |name: &str| secrets.truthy(name);
|
||||
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
|
||||
credentials.api_key.clone().or_else(|| {
|
||||
preferred_api_key_env
|
||||
.into_iter()
|
||||
.chain(self.get_api_key_env_var())
|
||||
.find_map(secret)
|
||||
.map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
|
||||
})
|
||||
});
|
||||
let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
|
||||
credentials.api_base.clone().or_else(|| {
|
||||
api_base_env
|
||||
.and_then(secret)
|
||||
.map(|value| Sourced::new(value, InputSource::Environment))
|
||||
})
|
||||
});
|
||||
self.resolve_connection_params(OcrCredentialInputs {
|
||||
dynamic_api_key,
|
||||
dynamic_api_base,
|
||||
..credentials
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn ocr(
|
||||
self,
|
||||
client: &OcrClient,
|
||||
|
|
@ -164,7 +214,7 @@ pub fn passthrough_response(
|
|||
endpoint: &str,
|
||||
body: &[u8],
|
||||
) -> Result<Option<LiteLLMOcrResponse>, Error> {
|
||||
let (model, config) = resolve_provider_config(model, Some("azure_ai"))?;
|
||||
let (provider, config) = resolve_provider_config(model, Some("azure_ai"))?;
|
||||
let segments: Vec<&str> = endpoint
|
||||
.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
|
|
@ -173,7 +223,7 @@ pub fn passthrough_response(
|
|||
OcrConfigKind::AzureAi => segments == AZURE_AI_OCR_PATH,
|
||||
OcrConfigKind::AzureCohere => segments == AZURE_COHERE_PARSE_PATH,
|
||||
OcrConfigKind::AzureDocumentIntelligence => {
|
||||
segments == AzureDocumentIntelligenceOcrConfig::analyze_path(&model)?
|
||||
segments == AzureDocumentIntelligenceOcrConfig::analyze_path(provider.model)?
|
||||
}
|
||||
other => {
|
||||
let provider: &'static str = other.provider().into();
|
||||
|
|
@ -183,24 +233,28 @@ pub fn passthrough_response(
|
|||
if !is_ocr_endpoint {
|
||||
return Ok(None);
|
||||
}
|
||||
with_config!(config, config => config.transform_ocr_response(&model, body, OcrResponseFormat::Litellm))
|
||||
with_config!(config, config => config.transform_ocr_response(provider.model, body, OcrResponseFormat::Litellm))
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_provider_config(
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Result<(String, OcrConfigKind), Error> {
|
||||
let provider =
|
||||
get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider {
|
||||
model,
|
||||
custom_llm_provider: LlmProviders::Mistral.into(),
|
||||
});
|
||||
let llm_provider = provider
|
||||
.custom_llm_provider
|
||||
.parse::<LlmProviders>()
|
||||
.map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?;
|
||||
let config = match llm_provider {
|
||||
pub(crate) fn resolve_provider_config<'a>(
|
||||
model: &'a str,
|
||||
custom_llm_provider: Option<&'a str>,
|
||||
) -> Result<(ResolvedProvider<'a>, OcrConfigKind), Error> {
|
||||
let provider = match resolve_llm_provider(model, custom_llm_provider, "ocr") {
|
||||
Ok(provider) => provider,
|
||||
Err(_) if get_custom_llm_provider(model, custom_llm_provider).is_none() => {
|
||||
ResolvedProvider {
|
||||
model,
|
||||
provider: LlmProviders::Mistral,
|
||||
}
|
||||
}
|
||||
Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
|
||||
return Err(Error::InvalidProvider(provider));
|
||||
}
|
||||
Err(error) => return Err(Error::InvalidProvider(error.to_string())),
|
||||
};
|
||||
let config = match provider.provider {
|
||||
LlmProviders::AwsTextract => match TextractOperation::from_model(provider.model)? {
|
||||
TextractOperation::DetectDocumentText => OcrConfigKind::AwsTextract,
|
||||
TextractOperation::AnalyzeDocument => OcrConfigKind::AwsTextractAnalyze,
|
||||
|
|
@ -227,11 +281,11 @@ pub(crate) fn resolve_provider_config(
|
|||
LlmProviders::VertexAi => OcrConfigKind::VertexAi,
|
||||
_ => {
|
||||
return Err(Error::InvalidProvider(
|
||||
provider.custom_llm_provider.to_string(),
|
||||
<&'static str>::from(provider.provider).to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok((provider.model.to_string(), config))
|
||||
Ok((provider, config))
|
||||
}
|
||||
|
||||
fn is_document_intelligence_model(model: &str) -> bool {
|
||||
|
|
@ -243,7 +297,7 @@ fn is_document_intelligence_model(model: &str) -> bool {
|
|||
mod tests {
|
||||
use std::collections::HashSet;
|
||||
|
||||
use litellm_auth::{InputSource, Sourced};
|
||||
use litellm_auth::Sourced;
|
||||
use litellm_llms::{
|
||||
base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document,
|
||||
};
|
||||
|
|
@ -484,8 +538,8 @@ mod tests {
|
|||
#[case] expected_config: OcrConfigKind,
|
||||
) {
|
||||
let expected_model = qualified_model.split_once('/').unwrap().1;
|
||||
let (model, config) = resolve_provider_config(qualified_model, None).unwrap();
|
||||
assert_eq!(model, expected_model);
|
||||
let (provider, config) = resolve_provider_config(qualified_model, None).unwrap();
|
||||
assert_eq!(provider.model, expected_model);
|
||||
assert_eq!(config, expected_config);
|
||||
}
|
||||
|
||||
|
|
@ -528,7 +582,7 @@ mod tests {
|
|||
expected_config
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_provider_config(model, None).unwrap().0,
|
||||
resolve_provider_config(model, None).unwrap().0.model,
|
||||
model.split_once('/').unwrap().1
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -75,6 +75,8 @@ impl OcrConnectionInputs {
|
|||
|
||||
pub struct LiteLLMOcrRequest<D = OcrDocumentInput> {
|
||||
pub model: String,
|
||||
pub(crate) requested_model: String,
|
||||
pub(crate) custom_llm_provider: Option<String>,
|
||||
pub document: D,
|
||||
pub credentials: OcrCredentialInputs,
|
||||
pub transport: OcrTransportConfig,
|
||||
|
|
@ -91,7 +93,8 @@ impl LiteLLMOcrRequest {
|
|||
custom_llm_provider: Option<&str>,
|
||||
optional_params: CallArguments,
|
||||
) -> Result<Self, Error> {
|
||||
let (model, config) = resolve_provider_config(&model, custom_llm_provider)?;
|
||||
let (provider, config) = resolve_provider_config(&model, custom_llm_provider)?;
|
||||
let resolved_model = provider.model.to_owned();
|
||||
let default_transport = OcrTransportConfig::default();
|
||||
let max_response_bytes = optional_params
|
||||
.get("max_response_bytes")
|
||||
|
|
@ -116,7 +119,9 @@ impl LiteLLMOcrRequest {
|
|||
.collect();
|
||||
|
||||
Ok(Self {
|
||||
model,
|
||||
model: resolved_model,
|
||||
requested_model: model,
|
||||
custom_llm_provider: custom_llm_provider.map(str::to_owned),
|
||||
document: document.into(),
|
||||
credentials: OcrCredentialInputs::default(),
|
||||
transport,
|
||||
|
|
@ -135,6 +140,8 @@ impl<D> LiteLLMOcrRequest<D> {
|
|||
) -> Result<LiteLLMOcrRequest<T>, E> {
|
||||
Ok(LiteLLMOcrRequest {
|
||||
model: self.model,
|
||||
requested_model: self.requested_model,
|
||||
custom_llm_provider: self.custom_llm_provider,
|
||||
document: map(self.document)?,
|
||||
credentials: self.credentials,
|
||||
transport: self.transport,
|
||||
|
|
@ -148,6 +155,8 @@ impl<D> LiteLLMOcrRequest<D> {
|
|||
pub fn with_document<T>(self, document: T) -> LiteLLMOcrRequest<T> {
|
||||
LiteLLMOcrRequest {
|
||||
model: self.model,
|
||||
requested_model: self.requested_model,
|
||||
custom_llm_provider: self.custom_llm_provider,
|
||||
document,
|
||||
credentials: self.credentials,
|
||||
transport: self.transport,
|
||||
|
|
|
|||
|
|
@ -66,6 +66,19 @@ async fn interception_runs_once_and_observers_receive_ordered_success_events() {
|
|||
assert_eq!(received(&upstream).await.len(), 1);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::mutated_model("model-b")]
|
||||
#[tokio::test]
|
||||
async fn provider_request_uses_the_request_model_after_it_is_mutated(#[case] model: &str) {
|
||||
let upstream = upstream([pages_response()]).await;
|
||||
let mut request = ocr_request("mistral/model-a", &upstream.uri(), json!({}));
|
||||
request.model = model.into();
|
||||
|
||||
perform(request).await.unwrap();
|
||||
|
||||
assert_eq!(only_request(&upstream).await.json()["model"], json!(model));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn a_blocking_before_send_prevents_the_call_and_emits_one_failure() {
|
||||
|
|
|
|||
|
|
@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender;
|
|||
use std::time::Duration;
|
||||
|
||||
use futures_util::StreamExt;
|
||||
use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest};
|
||||
use litellm_llms::base_llm::auth::{Authenticated, resolve_auth};
|
||||
use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
|
||||
use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth};
|
||||
use litellm_secrets::source::Secrets;
|
||||
|
||||
use super::{
|
||||
Error,
|
||||
types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead},
|
||||
Error, ResponsesRoute,
|
||||
types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead},
|
||||
};
|
||||
|
||||
pub(super) async fn execute(
|
||||
http: &litellm_http::Client,
|
||||
auth: &litellm_auth::AuthServices,
|
||||
request: ProviderResponsesRequest,
|
||||
cache: Option<litellm_cache_response::ScopedCache>,
|
||||
route: &ResponsesRoute,
|
||||
ResponsesProvider { identity, config }: ResponsesProvider,
|
||||
call: ResponsesCall,
|
||||
secrets: Secrets,
|
||||
cache_options: Option<litellm_cache_response::CachePolicy>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<ResponsesOutput, Error> {
|
||||
let authenticated = resolve_auth(auth, request.environment, &|_| None).await?;
|
||||
let identity = litellm_host::interceptors::ProviderIdentity {
|
||||
model: request.context.model.clone(),
|
||||
provider: request.context.custom_llm_provider.clone(),
|
||||
let lookup = |name: &str| secrets.get(name);
|
||||
let environment = config.validate_environment(
|
||||
litellm_http::request::string_headers("responses", call.extra_headers)?,
|
||||
call.api_key.as_deref(),
|
||||
&lookup,
|
||||
)?;
|
||||
let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
|
||||
let context = RequestContext {
|
||||
model: identity.model.clone(),
|
||||
custom_llm_provider: identity.provider.clone(),
|
||||
optional_params: serde_json::Value::Object(call.optional_params.clone()),
|
||||
secret_fields: Vec::new(),
|
||||
api_key: match &environment.auth {
|
||||
AuthScheme::Credential { secret, .. } => Some(secret.clone()),
|
||||
_ => None,
|
||||
},
|
||||
};
|
||||
let body = config.transform_responses_api_request(
|
||||
&identity.model,
|
||||
call.input,
|
||||
call.optional_params,
|
||||
)?;
|
||||
let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?;
|
||||
let wire = interceptors
|
||||
.before_provider_request(
|
||||
WireRequest {
|
||||
url: request.url,
|
||||
url,
|
||||
headers: authenticated.headers,
|
||||
body: request.body,
|
||||
body,
|
||||
},
|
||||
request.context,
|
||||
context,
|
||||
)
|
||||
.await?;
|
||||
let cache = cache.filter(|_| authenticated.signer.is_none());
|
||||
let cache = route
|
||||
.cache
|
||||
.clone()
|
||||
.filter(|_| authenticated.signer.is_none());
|
||||
let cache_request = litellm_inference::caching::CacheRequest::from_wire(
|
||||
identity,
|
||||
cache.as_ref().map(|_| &wire),
|
||||
|
|
@ -59,9 +81,9 @@ pub(super) async fn execute(
|
|||
},
|
||||
wire.url,
|
||||
&wire.body,
|
||||
Some(request.timeout.unwrap_or(Duration::from_secs(600))),
|
||||
Some(call.timeout.unwrap_or(Duration::from_secs(600))),
|
||||
)?;
|
||||
let response = litellm_inference::outbound::send(outbound, http)
|
||||
let response = litellm_inference::outbound::send(outbound, &route.http)
|
||||
.await
|
||||
.map_err(network)?;
|
||||
let status = response.status().as_u16();
|
||||
|
|
@ -103,8 +125,7 @@ pub(super) async fn execute(
|
|||
.map_err(Error::post_call)?;
|
||||
let value = serde_json::from_str(&body)
|
||||
.map_err(|error| Error::InvalidResponse(error.to_string().into()))?;
|
||||
request
|
||||
.config
|
||||
config
|
||||
.transform_response_api_response(value)
|
||||
.map(ResponsesOutput::Complete)
|
||||
.map_err(Error::from)
|
||||
|
|
|
|||
|
|
@ -7,14 +7,30 @@ pub mod types;
|
|||
pub mod websocket;
|
||||
|
||||
mod handler;
|
||||
mod prepare;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use litellm_auth::AuthServices;
|
||||
use litellm_core_utils::get_llm_provider_logic::{
|
||||
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
|
||||
};
|
||||
use litellm_host::interceptors::Interceptors;
|
||||
use litellm_host::interceptors::ProviderIdentity;
|
||||
use litellm_llms::{
|
||||
base_llm::responses::transformation::BaseResponsesApiConfig,
|
||||
openai::responses::transformation::OpenAiResponsesApiConfig,
|
||||
openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
|
||||
};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use types::{ResponsesCall, ResponsesOutput};
|
||||
use types::{ResponsesCall, ResponsesOutput, ResponsesProvider};
|
||||
|
||||
fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> {
|
||||
match provider {
|
||||
LlmProviders::Openai => Some(&OpenAiResponsesApiConfig),
|
||||
LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ResponsesRoute {
|
||||
|
|
@ -78,34 +94,68 @@ impl ResponsesRoute {
|
|||
observers: Option<&ObservationSender>,
|
||||
) -> Result<ResponsesOutput, Error> {
|
||||
litellm_inference::diagnostic::call(async {
|
||||
self.run_provider(call, cache_options, interceptors, observers)
|
||||
.await
|
||||
let explicit = call
|
||||
.custom_llm_provider
|
||||
.as_deref()
|
||||
.filter(|provider| !provider.is_empty());
|
||||
let (provider_name, resolved_model) =
|
||||
match get_custom_llm_provider(&call.model, explicit) {
|
||||
Some(CustomLlmProvider {
|
||||
model,
|
||||
custom_llm_provider,
|
||||
}) => (custom_llm_provider, model),
|
||||
None => ("openai", call.model.as_str()),
|
||||
};
|
||||
if resolved_model.is_empty() {
|
||||
return Err(Error::InvalidProvider(call.model));
|
||||
}
|
||||
let provider = provider_name
|
||||
.parse::<LlmProviders>()
|
||||
.map_err(|_| Error::Unsupported("native HTTP responses provider"))?;
|
||||
let config = responses_provider(provider)
|
||||
.ok_or(Error::Unsupported("native HTTP responses provider"))?;
|
||||
let wire_model = match provider {
|
||||
LlmProviders::Openai if resolved_model.contains('/') => {
|
||||
return Err(Error::InvalidProvider(call.model));
|
||||
}
|
||||
LlmProviders::Openai => resolved_model.into(),
|
||||
LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => {
|
||||
format!("openrouter/{resolved_model}")
|
||||
}
|
||||
LlmProviders::Openrouter => resolved_model.into(),
|
||||
_ => return Err(Error::Unsupported("native HTTP responses provider")),
|
||||
};
|
||||
let provider = ResponsesProvider {
|
||||
identity: ProviderIdentity {
|
||||
model: wire_model,
|
||||
provider: provider_name.into(),
|
||||
},
|
||||
config,
|
||||
};
|
||||
litellm_inference::diagnostic::provider(
|
||||
&provider.identity.model,
|
||||
&provider.identity.provider,
|
||||
);
|
||||
let secrets = self
|
||||
.secrets
|
||||
.resolve(
|
||||
provider
|
||||
.config
|
||||
.secret_names(call.api_key.as_deref(), call.api_base.as_deref()),
|
||||
)
|
||||
.await?;
|
||||
let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
|
||||
Box::pin(handler::execute(
|
||||
self,
|
||||
provider,
|
||||
call,
|
||||
secrets,
|
||||
cache_options,
|
||||
interceptors,
|
||||
observers,
|
||||
));
|
||||
execute.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn run_provider(
|
||||
&self,
|
||||
call: ResponsesCall,
|
||||
cache_options: Option<litellm_cache_response::CachePolicy>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<ResponsesOutput, Error> {
|
||||
let request = prepare::prepare(call, self.secrets.as_ref()).await?;
|
||||
litellm_inference::diagnostic::provider(
|
||||
&request.context.model,
|
||||
&request.context.custom_llm_provider,
|
||||
);
|
||||
let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
|
||||
Box::pin(handler::execute(
|
||||
&self.http,
|
||||
&self.auth,
|
||||
request,
|
||||
self.cache.clone(),
|
||||
cache_options,
|
||||
interceptors,
|
||||
observers,
|
||||
));
|
||||
execute.await
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,162 +0,0 @@
|
|||
use litellm_core_utils::get_llm_provider_logic::{
|
||||
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
|
||||
};
|
||||
use litellm_host::interceptors::{ProviderIdentity, RequestContext};
|
||||
use litellm_llms::{
|
||||
base_llm::responses::transformation::BaseResponsesApiConfig,
|
||||
openai::responses::transformation::OpenAiResponsesApiConfig,
|
||||
openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
|
||||
};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{
|
||||
Error,
|
||||
types::{ProviderResponsesRequest, ResponsesCall},
|
||||
};
|
||||
|
||||
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
|
||||
pub(super) async fn prepare(
|
||||
call: ResponsesCall,
|
||||
secrets: &dyn SecretSource,
|
||||
) -> Result<ProviderResponsesRequest, Error> {
|
||||
let ResponsesProvider { identity, config } =
|
||||
resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
|
||||
let provider = identity.provider.as_str();
|
||||
let model = identity.model.as_str();
|
||||
let snapshot = secrets
|
||||
.resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
|
||||
.await?;
|
||||
let lookup = |name: &str| snapshot.get(name);
|
||||
let environment = config.validate_environment(
|
||||
litellm_http::request::string_headers("responses", call.extra_headers)?,
|
||||
call.api_key.as_deref(),
|
||||
&lookup,
|
||||
)?;
|
||||
let context = RequestContext {
|
||||
model: model.into(),
|
||||
custom_llm_provider: provider.into(),
|
||||
optional_params: Value::Object(call.optional_params.clone()),
|
||||
secret_fields: Vec::new(),
|
||||
api_key: match &environment.auth {
|
||||
litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => {
|
||||
Some(secret.clone())
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
};
|
||||
let body = config.transform_responses_api_request(model, call.input, call.optional_params)?;
|
||||
Ok(ProviderResponsesRequest {
|
||||
url: config.get_complete_url(call.api_base.as_deref(), &lookup),
|
||||
config,
|
||||
environment,
|
||||
body,
|
||||
context,
|
||||
timeout: call.timeout,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) struct ResponsesProvider {
|
||||
pub identity: ProviderIdentity,
|
||||
pub config: &'static dyn BaseResponsesApiConfig,
|
||||
}
|
||||
|
||||
/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
|
||||
/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id.
|
||||
pub(super) fn resolve_provider(
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Result<ResponsesProvider, Error> {
|
||||
let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
|
||||
let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
|
||||
Some(CustomLlmProvider {
|
||||
model,
|
||||
custom_llm_provider,
|
||||
}) => (custom_llm_provider, model),
|
||||
None => ("openai", model),
|
||||
};
|
||||
if resolved.is_empty() {
|
||||
return Err(Error::InvalidProvider(model.into()));
|
||||
}
|
||||
let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
|
||||
match provider.parse::<LlmProviders>() {
|
||||
Ok(LlmProviders::Openai) if resolved.contains('/') => {
|
||||
return Err(Error::InvalidProvider(model.into()));
|
||||
}
|
||||
Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
|
||||
Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
|
||||
&OPENROUTER_RESPONSES_CONFIG,
|
||||
format!("openrouter/{resolved}"),
|
||||
),
|
||||
Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
|
||||
_ => return Err(Error::Unsupported("native HTTP responses provider")),
|
||||
};
|
||||
Ok(ResponsesProvider {
|
||||
identity: ProviderIdentity {
|
||||
model: wire_model,
|
||||
provider: provider.into(),
|
||||
},
|
||||
config,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Pinned against Python's `get_llm_provider` on 2026-10-11.
|
||||
#[rstest]
|
||||
#[case::bare("gpt-5", None, "openai", "gpt-5")]
|
||||
#[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")]
|
||||
#[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")]
|
||||
#[case::openrouter_prefix(
|
||||
"openrouter/anthropic/claude",
|
||||
None,
|
||||
"openrouter",
|
||||
"anthropic/claude"
|
||||
)]
|
||||
#[case::openrouter_explicit(
|
||||
"openrouter/anthropic/claude",
|
||||
Some("openrouter"),
|
||||
"openrouter",
|
||||
"anthropic/claude"
|
||||
)]
|
||||
#[case::openrouter_explicit_unprefixed(
|
||||
"anthropic/claude",
|
||||
Some("openrouter"),
|
||||
"openrouter",
|
||||
"anthropic/claude"
|
||||
)]
|
||||
#[case::openrouter_native_id(
|
||||
"openrouter/auto",
|
||||
Some("openrouter"),
|
||||
"openrouter",
|
||||
"openrouter/auto"
|
||||
)]
|
||||
#[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")]
|
||||
fn providers_and_wire_models_match_python(
|
||||
#[case] model: &str,
|
||||
#[case] custom_llm_provider: Option<&str>,
|
||||
#[case] provider: &str,
|
||||
#[case] wire_model: &str,
|
||||
) {
|
||||
let resolved = resolve_provider(model, custom_llm_provider).unwrap();
|
||||
assert_eq!(resolved.identity.provider, provider);
|
||||
assert_eq!(resolved.identity.model, wire_model);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::other_prefix("other/model", None)]
|
||||
#[case::other_explicit("model", Some("other"))]
|
||||
#[case::openai_with_vendor_segment("openai/other/model", None)]
|
||||
#[case::empty_model("openai/", None)]
|
||||
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
|
||||
fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
|
||||
assert!(matches!(
|
||||
resolve_provider(model, custom_llm_provider),
|
||||
Err(Error::InvalidProvider(_) | Error::Unsupported(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
@ -2,9 +2,8 @@ use std::time::Duration;
|
|||
|
||||
use bytes::Bytes;
|
||||
use litellm_host::call::CallOutput;
|
||||
use litellm_llms::base_llm::{
|
||||
auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig,
|
||||
};
|
||||
use litellm_host::interceptors::ProviderIdentity;
|
||||
use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig;
|
||||
use litellm_llms_types::formats::responses::ResponsesApiResponse;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
@ -21,17 +20,13 @@ pub struct ResponsesCall {
|
|||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
pub(crate) struct ResponsesProvider {
|
||||
pub identity: ProviderIdentity,
|
||||
pub config: &'static dyn BaseResponsesApiConfig,
|
||||
}
|
||||
|
||||
pub struct ResponsesStreamHead {
|
||||
pub headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
pub type ResponsesOutput = CallOutput<ResponsesApiResponse, ResponsesStreamHead, Bytes, Error>;
|
||||
|
||||
pub(super) struct ProviderResponsesRequest {
|
||||
pub config: &'static dyn BaseResponsesApiConfig,
|
||||
pub environment: ValidatedEnvironment,
|
||||
pub url: String,
|
||||
pub body: Value,
|
||||
pub context: litellm_host::interceptors::RequestContext,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use std::sync::Arc;
|
|||
use futures_util::TryStreamExt;
|
||||
use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
|
||||
use litellm_inference_responses::{
|
||||
Error,
|
||||
route::Responses,
|
||||
types::{ResponsesCall, ResponsesOutput},
|
||||
};
|
||||
|
|
@ -32,6 +33,110 @@ fn call() -> ResponsesCall {
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bare_openai("gpt-5", None, "gpt-5")]
|
||||
#[case::openai_prefix("openai/gpt-5", None, "gpt-5")]
|
||||
#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")]
|
||||
#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")]
|
||||
#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")]
|
||||
#[case::explicit_openrouter_prefix(
|
||||
"openrouter/anthropic/claude",
|
||||
Some("openrouter"),
|
||||
"anthropic/claude"
|
||||
)]
|
||||
#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")]
|
||||
#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")]
|
||||
#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")]
|
||||
#[tokio::test]
|
||||
async fn provider_resolution_preserves_the_wire_model(
|
||||
#[case] model: &str,
|
||||
#[case] custom_llm_provider: Option<&str>,
|
||||
#[case] expected_wire_model: &str,
|
||||
) {
|
||||
let response = json!({
|
||||
"id": "response-1",
|
||||
"model": expected_wire_model,
|
||||
"output": [{"type": "message", "content": []}],
|
||||
"usage": {"total_tokens": 7}
|
||||
});
|
||||
let upstream = upstream([json_response(response)]).await;
|
||||
responses_route(no_secrets())
|
||||
.execute(
|
||||
ResponsesCall {
|
||||
model: model.into(),
|
||||
api_key: Some("test-key".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
custom_llm_provider: custom_llm_provider.map(str::to_owned),
|
||||
..call()
|
||||
},
|
||||
&(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
only_request(&upstream).await.json()["model"],
|
||||
expected_wire_model
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))]
|
||||
#[case::unsupported_responses_provider("test-model", Some("anthropic"))]
|
||||
#[tokio::test]
|
||||
async fn unsupported_providers_fail_before_transport(
|
||||
#[case] model: &str,
|
||||
#[case] custom_llm_provider: Option<&str>,
|
||||
) {
|
||||
let upstream = upstream([]).await;
|
||||
let result = responses_route(no_secrets())
|
||||
.execute(
|
||||
ResponsesCall {
|
||||
model: model.into(),
|
||||
api_base: Some(upstream.uri()),
|
||||
custom_llm_provider: custom_llm_provider.map(str::to_owned),
|
||||
..call()
|
||||
},
|
||||
&(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::Unsupported("native HTTP responses provider"))
|
||||
));
|
||||
assert!(received(&upstream).await.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::openai_with_vendor_segment("openai/other/model", None)]
|
||||
#[case::empty_model("openai/", None)]
|
||||
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
|
||||
#[tokio::test]
|
||||
async fn invalid_provider_models_fail_before_transport(
|
||||
#[case] model: &str,
|
||||
#[case] custom_llm_provider: Option<&str>,
|
||||
) {
|
||||
let upstream = upstream([]).await;
|
||||
let result = responses_route(no_secrets())
|
||||
.execute(
|
||||
ResponsesCall {
|
||||
model: model.into(),
|
||||
api_base: Some(upstream.uri()),
|
||||
custom_llm_provider: custom_llm_provider.map(str::to_owned),
|
||||
..call()
|
||||
},
|
||||
&(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(Error::InvalidProvider(_))));
|
||||
assert!(received(&upstream).await.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::direct(false)]
|
||||
#[case::hosted(true)]
|
||||
|
|
|
|||
|
|
@ -1,28 +1,57 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_http::{Client, request::truncate_error_body};
|
||||
use litellm_llms::base_llm::auth::resolve_auth;
|
||||
use litellm_http::{
|
||||
Client,
|
||||
request::{string_headers, truncate_error_body, with_default_headers},
|
||||
};
|
||||
use litellm_llms::base_llm::{
|
||||
audio_transcription::transformation::BaseAudioTranscriptionConfig,
|
||||
auth::{ValidatedEnvironment, resolve_auth},
|
||||
};
|
||||
use litellm_secrets::source::Secrets;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::Error;
|
||||
use crate::{
|
||||
constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::ProviderAudioTranscriptionRequest,
|
||||
};
|
||||
use crate::{constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::AudioTranscriptionRequest};
|
||||
use litellm_inference::provider::ResolvedProvider;
|
||||
|
||||
pub async fn execute_audio_transcription_provider_call(
|
||||
pub(super) async fn execute(
|
||||
http: &Client,
|
||||
auth: &litellm_auth::AuthServices,
|
||||
request: ProviderAudioTranscriptionRequest,
|
||||
config: &'static dyn BaseAudioTranscriptionConfig,
|
||||
provider: ResolvedProvider<'_>,
|
||||
call: AudioTranscriptionRequest<'_>,
|
||||
secrets: Secrets,
|
||||
) -> Result<Value, Error> {
|
||||
let env_lookup = |key: &str| request.secrets.get(key);
|
||||
let authenticated = resolve_auth(auth, request.environment.clone(), &env_lookup).await?;
|
||||
let env_lookup = |key: &str| secrets.get(key);
|
||||
let forwarded = string_headers("audio transcription", call.extra_headers)?;
|
||||
let validated = config.validate_environment(
|
||||
forwarded,
|
||||
provider.model,
|
||||
&call.optional_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: with_default_headers(validated.headers, config.default_headers()),
|
||||
auth: validated.auth,
|
||||
};
|
||||
let url = config.get_complete_url(
|
||||
call.api_base,
|
||||
provider.model,
|
||||
&call.optional_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
let filtered_params = config.map_transcription_params(&call.optional_params);
|
||||
let body = config
|
||||
.transform_audio_transcription_request(provider.model, call.audio, filtered_params)?
|
||||
.body;
|
||||
let authenticated = resolve_auth(auth, environment, &env_lookup).await?;
|
||||
let outbound = litellm_inference::outbound::outbound_request(
|
||||
authenticated,
|
||||
request.url.clone(),
|
||||
&request.body,
|
||||
url,
|
||||
&body,
|
||||
Some(
|
||||
request
|
||||
.timeout
|
||||
call.timeout
|
||||
.unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)),
|
||||
),
|
||||
)?;
|
||||
|
|
@ -47,8 +76,7 @@ pub async fn execute_audio_transcription_provider_call(
|
|||
error,
|
||||
))
|
||||
})?;
|
||||
Ok(request
|
||||
.config
|
||||
.transform_audio_transcription_response(&request.model, response_json)?
|
||||
Ok(config
|
||||
.transform_audio_transcription_response(provider.model, response_json)?
|
||||
.into_json())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,16 +2,29 @@ pub mod types;
|
|||
pub use litellm_inference::RouteError as Error;
|
||||
mod constants;
|
||||
mod handler;
|
||||
mod prepare;
|
||||
pub use handler::execute_audio_transcription_provider_call;
|
||||
|
||||
use litellm_auth::AuthServices;
|
||||
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
use litellm_llms::{
|
||||
base_llm::audio_transcription::transformation::BaseAudioTranscriptionConfig,
|
||||
bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
|
||||
};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
pub use prepare::prepare_audio_transcription_provider_call;
|
||||
use serde_json::Value;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::types::AudioTranscriptionRequest;
|
||||
|
||||
fn audio_transcription_provider(
|
||||
provider: LlmProviders,
|
||||
) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
|
||||
match provider {
|
||||
LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AudioTranscriptionRoute {
|
||||
http: litellm_http::Client,
|
||||
|
|
@ -32,6 +45,10 @@ impl AudioTranscriptionRoute {
|
|||
}
|
||||
}
|
||||
|
||||
pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
|
||||
self.run(request).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.route", skip_all, fields(
|
||||
route = "audio_transcription",
|
||||
model = request.model,
|
||||
|
|
@ -40,13 +57,23 @@ impl AudioTranscriptionRoute {
|
|||
stream = false,
|
||||
outcome
|
||||
))]
|
||||
pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
|
||||
async fn run(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
|
||||
litellm_inference::diagnostic::unary(async {
|
||||
let request =
|
||||
prepare_audio_transcription_provider_call(request, self.secrets.as_ref()).await?;
|
||||
litellm_inference::diagnostic::provider(&request.model, &request.custom_llm_provider);
|
||||
let provider = resolve_llm_provider(
|
||||
request.model,
|
||||
request.custom_llm_provider,
|
||||
"audio transcription",
|
||||
)?;
|
||||
let config = audio_transcription_provider(provider.provider).ok_or_else(|| {
|
||||
Error::InvalidProvider(<&str>::from(provider.provider).to_string())
|
||||
})?;
|
||||
litellm_inference::diagnostic::provider(
|
||||
provider.model,
|
||||
<&'static str>::from(provider.provider),
|
||||
);
|
||||
let secrets = self.secrets.resolve(&config.secret_names()).await?;
|
||||
let execute: futures_util::future::BoxFuture<'_, Result<Value, Error>> = Box::pin(
|
||||
execute_audio_transcription_provider_call(&self.http, &self.auth, request),
|
||||
handler::execute(&self.http, &self.auth, config, provider, request, secrets),
|
||||
);
|
||||
execute.await
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
|
||||
use litellm_http::request::string_headers;
|
||||
use litellm_http::request::with_default_headers;
|
||||
use litellm_llms::{
|
||||
base_llm::{
|
||||
audio_transcription::transformation::BaseAudioTranscriptionConfig,
|
||||
auth::ValidatedEnvironment,
|
||||
},
|
||||
bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
|
||||
};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
|
||||
use super::Error;
|
||||
use crate::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
|
||||
fn provider_config(provider: LlmProviders) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
|
||||
match provider {
|
||||
LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
|
||||
pub async fn prepare_audio_transcription_provider_call(
|
||||
request: AudioTranscriptionRequest<'_>,
|
||||
secrets: &dyn SecretSource,
|
||||
) -> Result<ProviderAudioTranscriptionRequest, Error> {
|
||||
let provider_info = resolve_llm_provider(
|
||||
request.model,
|
||||
request.custom_llm_provider,
|
||||
"audio transcription",
|
||||
)?;
|
||||
let model = provider_info.model.to_string();
|
||||
let config = provider_config(provider_info.provider)
|
||||
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?;
|
||||
let snapshot = secrets.resolve(&config.secret_names()).await?;
|
||||
let env_lookup = |key: &str| snapshot.get(key);
|
||||
let forwarded = string_headers("audio transcription", request.extra_headers)?;
|
||||
let validated =
|
||||
config.validate_environment(forwarded, &model, &request.optional_params, &env_lookup)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: with_default_headers(validated.headers, config.default_headers()),
|
||||
auth: validated.auth,
|
||||
};
|
||||
let url = config.get_complete_url(
|
||||
request.api_base,
|
||||
&model,
|
||||
&request.optional_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
let filtered_params = config.map_transcription_params(&request.optional_params);
|
||||
let transformed =
|
||||
config.transform_audio_transcription_request(&model, request.audio, filtered_params)?;
|
||||
Ok(ProviderAudioTranscriptionRequest {
|
||||
model,
|
||||
custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
|
||||
config,
|
||||
url,
|
||||
body: transformed.body,
|
||||
environment,
|
||||
secrets: snapshot,
|
||||
timeout: request.timeout,
|
||||
})
|
||||
}
|
||||
|
|
@ -1,9 +1,5 @@
|
|||
use litellm_secrets::source::Secrets;
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_llms::base_llm::{
|
||||
audio_transcription::transformation::BaseAudioTranscriptionConfig, auth::ValidatedEnvironment,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub struct AudioTranscriptionRequest<'a> {
|
||||
|
|
@ -16,37 +12,3 @@ pub struct AudioTranscriptionRequest<'a> {
|
|||
pub optional_params: Map<String, Value>,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ProviderAudioTranscriptionRequest {
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub config: &'static dyn BaseAudioTranscriptionConfig,
|
||||
pub url: String,
|
||||
pub body: Value,
|
||||
pub environment: ValidatedEnvironment,
|
||||
pub secrets: Secrets,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
impl ProviderAudioTranscriptionRequest {
|
||||
pub fn model(&self) -> &str {
|
||||
&self.model
|
||||
}
|
||||
|
||||
pub fn custom_llm_provider(&self) -> &str {
|
||||
&self.custom_llm_provider
|
||||
}
|
||||
|
||||
pub fn url(&self) -> &str {
|
||||
&self.url
|
||||
}
|
||||
|
||||
pub fn body(&self) -> &Value {
|
||||
&self.body
|
||||
}
|
||||
|
||||
pub fn with_body(self, body: Value) -> Self {
|
||||
Self { body, ..self }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
_litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -887,6 +888,7 @@ mod tests {
|
|||
headers(forwarded),
|
||||
api_key,
|
||||
"claude",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
&env(vars),
|
||||
)
|
||||
|
|
@ -1206,6 +1208,7 @@ mod tests {
|
|||
Vec::new(),
|
||||
None,
|
||||
"claude",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
&record,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use litellm_auth::SecretValue;
|
||||
use litellm_http::request::{has_bearer_auth, has_header};
|
||||
use litellm_llms_types::{
|
||||
formats::messages::MessagesRequest,
|
||||
formats::messages::{MessagesOptionalParams, MessagesRequest},
|
||||
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
|
|
@ -83,6 +83,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
_litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -237,6 +238,7 @@ mod tests {
|
|||
.collect(),
|
||||
api_key,
|
||||
"claude",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
&|_| None,
|
||||
)
|
||||
|
|
@ -622,6 +624,7 @@ mod tests {
|
|||
Vec::new(),
|
||||
None,
|
||||
"claude",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
&record,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
|
||||
use litellm_llms_types::formats::messages::{
|
||||
MessagesOptionalParams, MessagesRequest, MessagesResponse,
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -55,6 +57,7 @@ pub trait BaseMessagesConfig: Sync {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
model: &str,
|
||||
optional_params: &MessagesOptionalParams,
|
||||
litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error>;
|
||||
|
|
@ -110,6 +113,7 @@ mod tests {
|
|||
headers: Headers,
|
||||
_api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
_litellm_params: &LitellmParams,
|
||||
_env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
|
|||
use litellm_core_utils::settings::resolve_non_empty;
|
||||
use litellm_http::request::{header_value, with_header, without_headers};
|
||||
use litellm_llms_types::{
|
||||
formats::messages::MessagesRequest,
|
||||
formats::messages::{MessagesOptionalParams, MessagesRequest},
|
||||
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
|
|
@ -60,6 +60,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
_model: &str,
|
||||
optional_params: &MessagesOptionalParams,
|
||||
params: &LitellmParams,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -67,11 +68,8 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
|
|||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| {
|
||||
WORKSPACE_FIELDS.iter().find_map(|field| {
|
||||
params
|
||||
.extra
|
||||
.get(*field)?
|
||||
.as_str()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
workspace_value(optional_params.extra.get(*field))
|
||||
.or_else(|| workspace_value(params.extra.get(*field)))
|
||||
})
|
||||
});
|
||||
let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
|
||||
|
|
@ -182,6 +180,10 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
|
|||
}
|
||||
}
|
||||
|
||||
fn workspace_value(value: Option<&Value>) -> Option<&str> {
|
||||
value?.as_str().filter(|value| !value.trim().is_empty())
|
||||
}
|
||||
|
||||
fn platform_region(
|
||||
base: Option<&str>,
|
||||
params: &LitellmParams,
|
||||
|
|
@ -260,6 +262,7 @@ mod tests {
|
|||
vec![],
|
||||
Some("key"),
|
||||
"claude-test",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
&|_| None
|
||||
)
|
||||
|
|
@ -280,6 +283,7 @@ mod tests {
|
|||
],
|
||||
Some("explicit"),
|
||||
"claude-test",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
&|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()),
|
||||
)
|
||||
|
|
@ -345,7 +349,14 @@ mod tests {
|
|||
.get_complete_url(None, "claude-test", ¶ms, false, &env)
|
||||
.unwrap();
|
||||
let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
|
||||
.validate_environment(vec![], None, "claude-test", ¶ms, &env)
|
||||
.validate_environment(
|
||||
vec![],
|
||||
None,
|
||||
"claude-test",
|
||||
&MessagesOptionalParams::default(),
|
||||
¶ms,
|
||||
&env,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
url,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ use litellm_auth_aws::{
|
|||
resolve_bedrock_region,
|
||||
};
|
||||
use litellm_llms_types::formats::messages::{
|
||||
MessagesRequest,
|
||||
MessagesOptionalParams, MessagesRequest,
|
||||
streaming::{MessagesStreamEvent, MessagesStreamUsage},
|
||||
};
|
||||
use litellm_llms_types::providers::bedrock::{
|
||||
|
|
@ -146,6 +146,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -618,6 +619,7 @@ mod tests {
|
|||
Vec::new(),
|
||||
None,
|
||||
"anthropic.claude-3",
|
||||
&MessagesOptionalParams::default(),
|
||||
&litellm_params,
|
||||
&|_| None,
|
||||
)
|
||||
|
|
@ -658,6 +660,7 @@ mod tests {
|
|||
vec![("authorization".into(), "Bearer forwarded".into())],
|
||||
api_key,
|
||||
"anthropic.claude-3",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
&env,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use litellm_auth_aws::bedrock_model_id_and_region;
|
|||
use litellm_core_utils::settings::resolve_non_empty;
|
||||
use litellm_http::request::{with_header, without_headers};
|
||||
use litellm_llms_types::{
|
||||
formats::messages::MessagesRequest,
|
||||
formats::messages::{MessagesOptionalParams, MessagesRequest},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
|
|
@ -59,6 +59,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
params: &LitellmParams,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -317,7 +318,14 @@ mod tests {
|
|||
"https://private.test/anthropic/v1/messages"
|
||||
);
|
||||
let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
|
||||
.validate_environment(vec![], None, model, ¶ms, &|_| None)
|
||||
.validate_environment(
|
||||
vec![],
|
||||
None,
|
||||
model,
|
||||
&MessagesOptionalParams::default(),
|
||||
¶ms,
|
||||
&|_| None,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region)
|
||||
|
|
@ -347,7 +355,14 @@ mod tests {
|
|||
.get_complete_url(None, "claude-test", ¶ms, false, &|_| None)
|
||||
.unwrap();
|
||||
let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
|
||||
.validate_environment(vec![], None, "claude-test", ¶ms, &|_| None)
|
||||
.validate_environment(
|
||||
vec![],
|
||||
None,
|
||||
"claude-test",
|
||||
&MessagesOptionalParams::default(),
|
||||
¶ms,
|
||||
&|_| None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
url,
|
||||
|
|
@ -401,6 +416,7 @@ mod tests {
|
|||
],
|
||||
key,
|
||||
"claude-test",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
&|name| match name {
|
||||
"BEDROCK_MANTLE_API_KEY" => Some("mantle".into()),
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
_litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -298,6 +299,7 @@ mod tests {
|
|||
.collect(),
|
||||
api_key,
|
||||
"deepseek-v4-pro",
|
||||
&MessagesOptionalParams::default(),
|
||||
&LitellmParams::default(),
|
||||
env,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
_params: &LitellmParams,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ impl BaseMessagesConfig for MinimaxMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
_params: &LitellmParams,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -183,7 +184,14 @@ mod tests {
|
|||
.flatten()
|
||||
};
|
||||
let environment = MINIMAX_MESSAGES_CONFIG
|
||||
.validate_environment(vec![], key, "test-model", &Default::default(), &env)
|
||||
.validate_environment(
|
||||
vec![],
|
||||
key,
|
||||
"test-model",
|
||||
&MessagesOptionalParams::default(),
|
||||
&Default::default(),
|
||||
&env,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
environment.auth,
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
_params: &LitellmParams,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ impl BaseMessagesConfig for TencentAnthropicMessagesConfig {
|
|||
headers: Headers,
|
||||
api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
_params: &LitellmParams,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -229,7 +230,14 @@ mod tests {
|
|||
.flatten()
|
||||
};
|
||||
let environment = TENCENT_MESSAGES_CONFIG
|
||||
.validate_environment(vec![], key, "test-model", &Default::default(), &env)
|
||||
.validate_environment(
|
||||
vec![],
|
||||
key,
|
||||
"test-model",
|
||||
&MessagesOptionalParams::default(),
|
||||
&Default::default(),
|
||||
&env,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
environment.auth,
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
|
|||
headers: Headers,
|
||||
_api_key: Option<&str>,
|
||||
_model: &str,
|
||||
_optional_params: &MessagesOptionalParams,
|
||||
litellm_params: &LitellmParams,
|
||||
_env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
|
|
@ -439,6 +440,7 @@ mod tests {
|
|||
.collect(),
|
||||
Some("ignored-api-key"),
|
||||
"claude-sonnet-4-5",
|
||||
&MessagesOptionalParams::default(),
|
||||
¶ms(Some("proj"), None),
|
||||
&no_env,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3113,14 +3113,19 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
|
|||
return response_ids[-1] if response_ids else None
|
||||
|
||||
@staticmethod
|
||||
def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject:
|
||||
def create_logging_object(
|
||||
results: Sequence[Mapping[str, object]], litellm_model_name: str | None = None
|
||||
) -> LiteLLMRealtimeStreamLoggingObject:
|
||||
tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results)
|
||||
return LiteLLMRealtimeStreamLoggingObject(
|
||||
logging_object: Final = LiteLLMRealtimeStreamLoggingObject(
|
||||
id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results),
|
||||
usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results),
|
||||
results=cast(OpenAIRealtimeStreamList, list(results)), # cast-ok: raw WS frames, unvalidated field
|
||||
service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None,
|
||||
)
|
||||
if litellm_model_name is not None:
|
||||
logging_object.hidden_params = {"litellm_model_name": litellm_model_name}
|
||||
return logging_object
|
||||
|
||||
|
||||
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
|
|||
BaseGoogleGenAIGenerateContentConfig,
|
||||
)
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import CallTypes
|
||||
|
|
@ -194,12 +195,14 @@ class GenerateContentHelper:
|
|||
if litellm_logging_obj is None:
|
||||
raise ValueError("litellm_logging_obj is required, but got None")
|
||||
|
||||
vertex_location: Final = VertexBase.explicit_vertex_ai_location(dict(litellm_params))
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
model=model,
|
||||
optional_params=dict(generate_content_config_dict),
|
||||
litellm_params={
|
||||
"litellm_call_id": litellm_call_id,
|
||||
**({"vertex_location": vertex_location} if vertex_location else {}),
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -142,14 +142,12 @@ def get_base_model_from_litellm_call_metadata(
|
|||
if metadata is None:
|
||||
return None
|
||||
model_info: Final = metadata.get("model_info")
|
||||
if model_info:
|
||||
model_info_mapping: Final = cast( # cast-ok: model metadata is caller-provided and preserves its mapping shape
|
||||
Mapping[str, object], model_info
|
||||
)
|
||||
return cast( # cast-ok: model metadata values are caller-provided
|
||||
str | None, model_info_mapping.get("base_model")
|
||||
)
|
||||
return None
|
||||
if not model_info:
|
||||
return None
|
||||
base_model: Final = cast( # cast-ok: model metadata is caller-provided and preserves its mapping shape
|
||||
Mapping[str, object], model_info
|
||||
).get("base_model")
|
||||
return base_model if isinstance(base_model, str) else None
|
||||
|
||||
|
||||
_get_base_model_from_litellm_call_metadata = get_base_model_from_litellm_call_metadata
|
||||
|
|
|
|||
|
|
@ -989,6 +989,16 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return model_id
|
||||
return None
|
||||
|
||||
def _routed_deployment_model(self) -> str | None:
|
||||
litellm_params: Final = self._model_call_details_view.get("litellm_params")
|
||||
metadata: Final = (
|
||||
StandardLoggingPayloadSetup.merge_litellm_metadata(_LITELLM_PARAMS.validate_python(litellm_params))
|
||||
if isinstance(litellm_params, dict)
|
||||
else {}
|
||||
)
|
||||
deployment: Final = metadata.get("deployment")
|
||||
return deployment if isinstance(deployment, str) else None
|
||||
|
||||
def get_deployment_model_for_cost(self) -> str | None:
|
||||
"""The provider-qualified model to price against.
|
||||
|
||||
|
|
@ -2256,6 +2266,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
and litellm_params.get(CallTypes.agenerate_content.value, False) is not True
|
||||
and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True
|
||||
and litellm_params.get(CallTypes.arealtime.value, False) is not True
|
||||
and litellm_params.get(CallTypes.aresponses_websocket.value, False) is not True
|
||||
)
|
||||
|
||||
_is_sync_litellm_request = is_sync_litellm_request
|
||||
|
|
@ -2511,7 +2522,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
|
||||
logging_result = ResponsesWebSocketTokenUsageProcessor.create_logging_object(
|
||||
results=result # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
|
||||
results=result, # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
|
||||
litellm_model_name=self._routed_deployment_model(),
|
||||
)
|
||||
|
||||
elif self.relays_llm_passthrough and isinstance(result, Response):
|
||||
|
|
|
|||
|
|
@ -375,7 +375,7 @@ def get_azure_ad_token(
|
|||
"Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
|
||||
)
|
||||
try:
|
||||
azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
|
||||
azure_ad_token_provider = _cached_azure_ad_token_refresh_provider(scope)
|
||||
except ValueError:
|
||||
verbose_logger.debug("Azure AD Token Provider could not be used.")
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -7,17 +7,21 @@ from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPC
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
|
||||
def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str:
|
||||
"""Azure serves WebRTC only on the GA v1 surface, which rejects any api-version query param."""
|
||||
base: Final = api_base.rstrip("/").removesuffix("/v1").removesuffix("/openai")
|
||||
return f"{base}/openai/v1/realtime/{endpoint}"
|
||||
|
||||
|
||||
class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
|
||||
def get_api_base(self, api_base: str | None, **kwargs) -> str:
|
||||
def get_api_base(self, api_base: str | None, **kwargs: object) -> str:
|
||||
return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or ""
|
||||
|
||||
def get_api_key(self, api_key: str | None, **kwargs) -> str:
|
||||
def get_api_key(self, api_key: str | None, **kwargs: object) -> str:
|
||||
return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or ""
|
||||
|
||||
def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
base: Final = self.get_api_base(api_base).rstrip("/")
|
||||
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
|
||||
return f"{base}/openai/realtime/client_secrets?api-version={version}"
|
||||
return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets")
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
|
|
@ -32,16 +36,14 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
|
|||
}
|
||||
|
||||
def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
base: Final = self.get_api_base(api_base).rstrip("/")
|
||||
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
|
||||
return f"{base}/openai/realtime/calls?api-version={version}"
|
||||
return _azure_realtime_v1_url(self.get_api_base(api_base), "calls")
|
||||
|
||||
def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
base: Final = self.get_api_base(api_base).rstrip("/")
|
||||
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
|
||||
return f"{base}/openai/realtime/transcription_sessions?api-version={version}"
|
||||
|
||||
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict:
|
||||
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]:
|
||||
return {
|
||||
"api-key": ephemeral_key,
|
||||
"Authorization": f"Bearer {ephemeral_key}",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6911,6 +6911,7 @@ class BaseLLMHTTPHandler:
|
|||
custom_llm_provider: str | None = None,
|
||||
first_message: str | None = None,
|
||||
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
|
||||
turn_admission: Callable[[], Awaitable[None]] | None = None,
|
||||
frame_guard: "ResponsesWsFrameGuard | None" = None,
|
||||
**kwargs: object,
|
||||
) -> Exception | None:
|
||||
|
|
@ -6973,12 +6974,12 @@ class BaseLLMHTTPHandler:
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
first_message=first_message,
|
||||
quota_callbacks=_ws_quota_callbacks,
|
||||
turn_admission=turn_admission,
|
||||
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
|
||||
frame_guard=frame_guard,
|
||||
**kwargs,
|
||||
)
|
||||
await handler.run()
|
||||
return None
|
||||
return await handler.run()
|
||||
|
||||
import websockets
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
|
@ -7069,6 +7070,7 @@ class BaseLLMHTTPHandler:
|
|||
authorized_model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
request_defaults=request_defaults,
|
||||
turn_admission=turn_admission,
|
||||
frame_guard=frame_guard,
|
||||
)
|
||||
return await streaming.bidirectional_forward()
|
||||
|
|
|
|||
|
|
@ -47779,6 +47779,34 @@
|
|||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"tsubasa/tsubasa-fast": {
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 2.0000000000000002e-07,
|
||||
"output_cost_per_token": 1e-06,
|
||||
"litellm_provider": "tsubasa",
|
||||
"mode": "chat",
|
||||
"supports_system_messages": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": false,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"tsubasa/tsubasa-pro": {
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 16384,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"output_cost_per_token": 4e-06,
|
||||
"litellm_provider": "tsubasa",
|
||||
"mode": "chat",
|
||||
"supports_system_messages": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": false,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"together_ai/BAAI/bge-base-en-v1.5": {
|
||||
"input_cost_per_token": 8e-09,
|
||||
"litellm_provider": "together_ai",
|
||||
|
|
@ -76083,32 +76111,22 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"tsubasa/tsubasa-fast": {
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 2.0000000000000002e-07,
|
||||
"output_cost_per_token": 1e-06,
|
||||
"litellm_provider": "tsubasa",
|
||||
"mode": "chat",
|
||||
"supports_system_messages": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": false,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"tsubasa/tsubasa-pro": {
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 16384,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"output_cost_per_token": 4e-06,
|
||||
"litellm_provider": "tsubasa",
|
||||
"mode": "chat",
|
||||
"supports_system_messages": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": false,
|
||||
"supports_reasoning": false
|
||||
"meta/muse-image-1.0": {
|
||||
"litellm_provider": "meta",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.01,
|
||||
"source": "https://dev.meta.ai/docs/pricing-rate-limits",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"image"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,11 @@ import asyncio
|
|||
import traceback
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Protocol, cast
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.batches.batch_utils import batch_cost_is_final
|
||||
|
|
@ -17,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
get_key_object,
|
||||
|
|
@ -100,6 +104,37 @@ def _proxy_stamped_used_client_oauth_token(
|
|||
return stamped if isinstance(stamped, bool) else None
|
||||
|
||||
|
||||
def _str_keyed_fields(value: object) -> Mapping[str, object]:
|
||||
return value if is_str_keyed_mapping(value) else {}
|
||||
|
||||
|
||||
def _is_model_group(value: object) -> bool:
|
||||
return isinstance(value, str) and bool(value)
|
||||
|
||||
|
||||
_TAG_LIST: Final = TypeAdapter(list[str])
|
||||
|
||||
|
||||
def _is_tag_list(value: object) -> bool:
|
||||
try:
|
||||
return bool(_TAG_LIST.validate_python(value, strict=True))
|
||||
except ValidationError:
|
||||
return False
|
||||
|
||||
|
||||
_ROUTED_METADATA_FIELD_CHECKS: Final = MappingProxyType({"model_group": _is_model_group, "tags": _is_tag_list})
|
||||
|
||||
|
||||
def _routed_metadata_fallback(request_data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
caller_metadata: Final = _str_keyed_fields(request_data.get("metadata"))
|
||||
routed_metadata: Final = _str_keyed_fields(request_data.get("litellm_metadata"))
|
||||
return {
|
||||
field: routed_metadata[field]
|
||||
for field, is_valid in _ROUTED_METADATA_FIELD_CHECKS.items()
|
||||
if not caller_metadata.get(field) and is_valid(routed_metadata.get(field))
|
||||
}
|
||||
|
||||
|
||||
def _failure_snapshot_with_guardrails_recorded_after_it(
|
||||
failure_snapshot: object, recorded_guardrail_information: object
|
||||
) -> Mapping[str, object] | None:
|
||||
|
|
@ -220,10 +255,11 @@ class _ProxyDBLogger(CustomLogger):
|
|||
metadata=_metadata, original_exception=original_exception
|
||||
)
|
||||
|
||||
_metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_data, request_route)
|
||||
request_fields: Final = _str_keyed_fields(request_data)
|
||||
_metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route)
|
||||
|
||||
existing_metadata: Final[dict] = request_data.get("metadata", None) or {}
|
||||
existing_metadata.update(_metadata)
|
||||
existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata})
|
||||
|
||||
litellm_metadata_bucket: Final = request_data.get("litellm_metadata")
|
||||
if (
|
||||
|
|
@ -471,11 +507,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
await _release_budget_reservation(budget_reservation=budget_reservation)
|
||||
# Non-model call types (health checks, afile_delete) have no model or standard_logging_object.
|
||||
# Use .get() for "stream" to avoid KeyError on health checks.
|
||||
# WS session wrappers (_aresponses_websocket, _arealtime) also reach here with
|
||||
# result=None; their per-turn costs are tracked on the inner aresponses/realtime calls.
|
||||
if sl_object is None and (
|
||||
not kwargs.get("model") or kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime")
|
||||
):
|
||||
if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") == "_arealtime"):
|
||||
verbose_proxy_logger.warning(
|
||||
"Cost tracking - skipping, no standard_logging_object for call_type=%s",
|
||||
kwargs.get("call_type", "unknown"),
|
||||
|
|
|
|||
|
|
@ -245,5 +245,4 @@ class GeminiPassthroughLoggingHandler:
|
|||
logging_obj.model = litellm_model_response.model or model
|
||||
logging_obj.model_call_details["model"] = logging_obj.model
|
||||
logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
|
||||
logging_obj.model_call_details["response_cost"] = response_cost
|
||||
return kwargs
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from pydantic import ConfigDict, TypeAdapter
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
|
||||
InteractionsUsageObjectTransformation,
|
||||
|
|
@ -225,7 +226,7 @@ class VertexPassthroughLoggingHandler:
|
|||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route),
|
||||
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider(url_route, logging_obj),
|
||||
vertex_location=vertex_location,
|
||||
)
|
||||
|
||||
|
|
@ -549,7 +550,7 @@ class VertexPassthroughLoggingHandler:
|
|||
response_json=response_json,
|
||||
)
|
||||
|
||||
custom_llm_provider: Final = VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route)
|
||||
custom_llm_provider: Final = VertexPassthroughLoggingHandler._get_custom_llm_provider(url_route, logging_obj)
|
||||
|
||||
litellm_embedding_response.model = model
|
||||
logging_obj.model = model
|
||||
|
|
@ -621,7 +622,9 @@ class VertexPassthroughLoggingHandler:
|
|||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=litellm_logging_obj,
|
||||
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route),
|
||||
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider(
|
||||
url_route, litellm_logging_obj
|
||||
),
|
||||
vertex_location=vertex_location,
|
||||
)
|
||||
|
||||
|
|
@ -669,9 +672,16 @@ class VertexPassthroughLoggingHandler:
|
|||
all_openai_chunks: Final = [parsed_chunk for parsed_chunk in parsed_chunks if parsed_chunk is not None]
|
||||
|
||||
complete_streaming_response: Final = litellm.stream_chunk_builder(chunks=all_openai_chunks)
|
||||
VertexPassthroughLoggingHandler._drop_location_blind_stream_builder_cost(complete_streaming_response)
|
||||
|
||||
return complete_streaming_response
|
||||
|
||||
@staticmethod
|
||||
def _drop_location_blind_stream_builder_cost(response: ModelResponse | TextCompletionResponse | None) -> None:
|
||||
hidden_params: Final = get_hidden_params(response)
|
||||
if hidden_params is not None:
|
||||
hidden_params.pop("response_cost", None)
|
||||
|
||||
@staticmethod
|
||||
def extract_model_from_url(url: str) -> str:
|
||||
pattern: Final = r"/models/([^:]+)"
|
||||
|
|
@ -724,9 +734,15 @@ class VertexPassthroughLoggingHandler:
|
|||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_custom_llm_provider_from_url(url: str) -> str:
|
||||
parsed_url: Final = urlparse(url)
|
||||
if parsed_url.hostname and parsed_url.hostname.endswith("generativelanguage.googleapis.com"):
|
||||
def _get_custom_llm_provider(url: str, logging_obj: LiteLLMLoggingObj) -> str:
|
||||
hostname: Final = urlparse(url).hostname
|
||||
if logging_obj.model_call_details.get("custom_llm_provider") == litellm.LlmProviders.GEMINI.value or (
|
||||
hostname is not None
|
||||
and (
|
||||
hostname == "generativelanguage.googleapis.com"
|
||||
or hostname.endswith(".generativelanguage.googleapis.com")
|
||||
)
|
||||
):
|
||||
return litellm.LlmProviders.GEMINI.value
|
||||
return litellm.LlmProviders.VERTEX_AI.value
|
||||
|
||||
|
|
|
|||
|
|
@ -161,6 +161,12 @@ pass_through_endpoint_logging: Final = PassThroughEndpointLogging()
|
|||
_METADATA_KEYS: Final = frozenset(("litellm_metadata", "metadata"))
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_KEPT_OUT_OF_LITELLM_PARAMS: Final = _METADATA_KEYS | frozenset(wire_names(ProxyRequestState))
|
||||
_PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",))
|
||||
|
||||
|
||||
def _is_mergeable_client_metadata(key: str, value: object) -> bool:
|
||||
return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
|
||||
|
||||
|
||||
# Global registry to track registered pass-through routes and prevent memory leaks
|
||||
_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
|
||||
|
|
@ -633,8 +639,13 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
litellm_keys_in_body: Final = MappingProxyType(
|
||||
{k: _parsed_body.pop(k) for k in types_utils.all_litellm_params if k in _parsed_body}
|
||||
)
|
||||
kept_out_of_litellm_params: Final = (
|
||||
_KEPT_OUT_OF_LITELLM_PARAMS
|
||||
if _key_or_team_allows_client_pricing_override(user_api_key_dict)
|
||||
else _KEPT_OUT_OF_LITELLM_PARAMS | _PRICING_MODEL_OVERRIDE_PARAMS
|
||||
)
|
||||
litellm_params_in_body: Final = MappingProxyType(
|
||||
{k: v for k, v in litellm_keys_in_body.items() if k not in _KEPT_OUT_OF_LITELLM_PARAMS}
|
||||
{k: v for k, v in litellm_keys_in_body.items() if k not in kept_out_of_litellm_params}
|
||||
)
|
||||
|
||||
_metadata = dict(
|
||||
|
|
@ -645,7 +656,13 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
metadata: Final = litellm_keys_in_body.get("metadata")
|
||||
for client_metadata in (litellm_metadata, metadata):
|
||||
if isinstance(client_metadata, dict):
|
||||
_metadata.update({k: v for k, v in client_metadata.items() if not k.startswith("user_api_key_")})
|
||||
_metadata.update(
|
||||
{
|
||||
k: v
|
||||
for k, v in cast(Mapping[str, object], client_metadata).items() # cast-ok: json
|
||||
if _is_mergeable_client_metadata(k, v)
|
||||
}
|
||||
)
|
||||
|
||||
_metadata = _apply_key_team_project_controls(user_api_key_dict=user_api_key_dict, metadata=_metadata)
|
||||
_metadata = _update_metadata_with_tags_in_header(
|
||||
|
|
|
|||
|
|
@ -129,10 +129,11 @@ class PassThroughStreamingHandler:
|
|||
usage: Final = getattr(partial_response, "usage", None)
|
||||
if not isinstance(usage, Usage):
|
||||
return
|
||||
response_cost: Final = kwargs.get("response_cost")
|
||||
logger_cost: Final = litellm_logging_obj.response_cost_calculator(result=partial_response)
|
||||
handler_cost: Final = kwargs.get("response_cost")
|
||||
fallback_cost: Final = float(handler_cost) if isinstance(handler_cost, (int, float)) else 0.0
|
||||
litellm_logging_obj.record_partial_usage_for_failure(
|
||||
usage=usage,
|
||||
response_cost=float(response_cost) if isinstance(response_cost, (int, float)) else 0.0,
|
||||
usage=usage, response_cost=fallback_cost if logger_cost is None else logger_cost
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -945,6 +945,7 @@ from litellm.proxy.video_endpoints.endpoints import router as video_router
|
|||
from litellm.repositories.base_repository import SupportsModelDump
|
||||
from litellm.repositories.credentials_repository import CredentialsRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.router import (
|
||||
AssistantsTypedDict,
|
||||
Deployment,
|
||||
|
|
@ -1239,6 +1240,19 @@ async def _flush_spend_logs_queue_on_shutdown() -> None:
|
|||
verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e)
|
||||
|
||||
|
||||
async def _flush_spend_on_shutdown() -> None:
|
||||
await flush_spend_counters_on_shutdown()
|
||||
await _flush_spend_logs_queue_on_shutdown()
|
||||
|
||||
|
||||
async def _flush_spend_around_logging_drain_on_shutdown(
|
||||
flush_spend: Callable[[], Awaitable[None]] = _flush_spend_on_shutdown,
|
||||
) -> None:
|
||||
await flush_spend()
|
||||
await drain_logging_tasks()
|
||||
await flush_spend()
|
||||
|
||||
|
||||
async def _drain_audit_tasks_on_shutdown() -> None:
|
||||
try:
|
||||
from litellm.proxy.management_helpers.audit_logs import drain_audit_tasks
|
||||
|
|
@ -1843,9 +1857,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.error("Error stopping in-flight scheduled jobs: %s", e)
|
||||
|
||||
await flush_spend_counters_on_shutdown()
|
||||
|
||||
await _flush_spend_logs_queue_on_shutdown()
|
||||
await _flush_spend_around_logging_drain_on_shutdown()
|
||||
|
||||
await proxy_config.stop_config_sync_subscriber()
|
||||
|
||||
|
|
|
|||
|
|
@ -78,7 +78,8 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
|
|||
request_tags_from_request_data,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
|
||||
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard
|
||||
from litellm.proxy.utils import handle_exception_on_proxy
|
||||
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -1861,12 +1862,14 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
|
|||
|
||||
|
||||
def _responses_ws_failure_frame(failure: Exception) -> str:
|
||||
raw_status: Final = getattr(failure, "status_code", None)
|
||||
status: Final = raw_status if isinstance(raw_status, int) and not isinstance(raw_status, bool) else 500
|
||||
proxy_exception: Final = handle_exception_on_proxy(failure)
|
||||
status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500
|
||||
error_type: Final = (
|
||||
"rate_limit_exceeded" if status == 429 else "invalid_request_error" if 400 <= status < 500 else "server_error"
|
||||
)
|
||||
return json.dumps({"type": "error", "status": status, "error": {"type": error_type, "message": str(failure)}})
|
||||
return json.dumps(
|
||||
{"type": "error", "status": status, "error": {"type": error_type, "message": proxy_exception.message}}
|
||||
)
|
||||
|
||||
|
||||
async def _enforce_responses_ws_first_frame_model_auth(
|
||||
|
|
@ -1934,7 +1937,10 @@ class _PreCallGuardrailWalk(Protocol):
|
|||
|
||||
|
||||
def _responses_ws_frame_guard(
|
||||
admitted: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, proxy_logging_obj: _PreCallGuardrailWalk
|
||||
admitted: Mapping[str, object],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
proxy_logging_obj: _PreCallGuardrailWalk,
|
||||
record_refused_turn: Callable[[tuple[Mapping[str, object], ...], Exception], None],
|
||||
) -> ResponsesWsFrameGuard:
|
||||
"""Socket admission skips guardrails (the first frame's ``input`` is not part of the admitted request), so every
|
||||
``response.create`` frame gets the pre_call guardrail walk the HTTP ``/v1/responses`` path runs on its body."""
|
||||
|
|
@ -1993,10 +1999,12 @@ def _responses_ws_frame_guard(
|
|||
if isinstance(nested, dict)
|
||||
else {**kept, **guarded_fields}
|
||||
)
|
||||
guardrail_information: Final = _recorded_guardrail_information(guarded_data.get(metadata_key))
|
||||
return GuardedResponsesWsFrame(
|
||||
frame=json.dumps(rewritten),
|
||||
guardrail_information=_recorded_guardrail_information(guarded_data.get(metadata_key)),
|
||||
guardrail_information=guardrail_information,
|
||||
pii_tokens=_recorded_pii_tokens(guarded_data.get("metadata")),
|
||||
bill_refusal=partial(record_refused_turn, guardrail_information),
|
||||
)
|
||||
|
||||
return guard
|
||||
|
|
@ -2042,6 +2050,33 @@ def _recorded_pii_tokens(metadata: object) -> Mapping[str, str]:
|
|||
_NO_PII_TOKENS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _RefusedTurns:
|
||||
"""Turns the deployment refused after their guardrails passed. The socket's close-time failure row bills their
|
||||
guardrail records, so its failure hooks, model group and call type cover those turns too, and the first refusal
|
||||
is that row's exception when the session limit cancels the relay before it returns one."""
|
||||
|
||||
first: Exception | None = None
|
||||
records: tuple[Mapping[str, object], ...] = ()
|
||||
|
||||
def add(self, records: tuple[Mapping[str, object], ...], refusal: Exception) -> None:
|
||||
self.records = (*self.records, *records)
|
||||
if self.first is None:
|
||||
self.first = refusal
|
||||
|
||||
def billed_on(self, request_data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
if not self.records:
|
||||
return request_data
|
||||
metadata_key: Final = get_metadata_variable_name_from_kwargs(request_data)
|
||||
bucket: Final = request_data.get(metadata_key)
|
||||
fields: Final = _WS_FRAME.validate_python(bucket) if isinstance(bucket, dict) else {}
|
||||
recorded: Final = (*_recorded_guardrail_information(fields), *self.records)
|
||||
return {
|
||||
**request_data,
|
||||
metadata_key: {**fields, _GUARDRAIL_INFORMATION_KEY: [dict(record) for record in recorded]},
|
||||
}
|
||||
|
||||
|
||||
def _recorded_guardrail_information(bucket: object) -> tuple[Mapping[str, object], ...]:
|
||||
recorded: Final = (
|
||||
_WS_FRAME.validate_python(bucket).get(_GUARDRAIL_INFORMATION_KEY) if isinstance(bucket, dict) else None
|
||||
|
|
@ -2145,39 +2180,58 @@ async def _responses_websocket_session(
|
|||
await websocket.close(code=1008, reason="Pre-call error")
|
||||
return
|
||||
|
||||
refused_turns: Final = _RefusedTurns()
|
||||
routed_data: Final = dict(
|
||||
data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
responses_ws_frame_guard=_responses_ws_frame_guard(data, user_api_key_dict, proxy_logging_obj),
|
||||
responses_ws_frame_guard=_responses_ws_frame_guard(
|
||||
data, user_api_key_dict, proxy_logging_obj, refused_turns.add
|
||||
),
|
||||
**_routing_hints_from_first_ws_frame(first_message),
|
||||
)
|
||||
# Phase 2: route to upstream provider
|
||||
try:
|
||||
llm_call: Final = await route_request(
|
||||
data=routed_data,
|
||||
route_type="_aresponses_websocket",
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
)
|
||||
failure: Final = await llm_call
|
||||
if isinstance(failure, Exception):
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=failure,
|
||||
request_data=routed_data,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Responses WebSocket error")
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.send_text(_responses_ws_failure_frame(e))
|
||||
|
||||
async def write_failure_row(exception: Exception) -> None:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=routed_data,
|
||||
original_exception=exception,
|
||||
request_data=dict(refused_turns.billed_on(routed_data)),
|
||||
)
|
||||
await websocket.close(code=1011, reason="Internal server error")
|
||||
|
||||
def bill_socket_failure(exception: Exception) -> asyncio.Task[object]:
|
||||
return spawn_logging_task(
|
||||
asyncio.get_running_loop(), write_failure_row(exception), task_name="Responses WebSocket failure row"
|
||||
)
|
||||
|
||||
async def relay_and_bill_failure() -> asyncio.Task[object] | None:
|
||||
try:
|
||||
llm_call: Final = await route_request(
|
||||
data=routed_data,
|
||||
route_type="_aresponses_websocket",
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
)
|
||||
outcome: Final = await llm_call
|
||||
except asyncio.CancelledError:
|
||||
if refused_turns.first is not None:
|
||||
_ = bill_socket_failure(refused_turns.first)
|
||||
raise
|
||||
except Exception as e:
|
||||
crash_row: Final = bill_socket_failure(e)
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.send_text(_responses_ws_failure_frame(e))
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.close(code=1011, reason="Internal server error")
|
||||
return crash_row
|
||||
return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None
|
||||
|
||||
# Phase 2: route to upstream provider
|
||||
try:
|
||||
failure_row: Final = await relay_and_bill_failure()
|
||||
finally:
|
||||
await proxy_logging_obj.arelease_responses_websocket_slot(user_api_key_dict)
|
||||
if failure_row is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await asyncio.shield(failure_row)
|
||||
|
||||
|
||||
@router.websocket("/v1/responses")
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import contextvars
|
||||
import json
|
||||
from collections.abc import Coroutine, Generator, Iterable, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
|
|
@ -73,7 +73,7 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
|
|||
from litellm.types.responses.main import *
|
||||
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, all_litellm_params
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
|
||||
from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client,
|
||||
|
|
@ -2609,6 +2609,7 @@ async def _aresponses_websocket(
|
|||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
timeout: float | None = None,
|
||||
turn_admission: Callable[[], Awaitable[None]] | None = None,
|
||||
responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None,
|
||||
**kwargs,
|
||||
) -> Exception | None:
|
||||
|
|
@ -2624,7 +2625,7 @@ async def _aresponses_websocket(
|
|||
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
|
||||
user: Final = kwargs.get("user", None)
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
litellm_params_dict: Final = {**get_litellm_params(**kwargs), CallTypes.aresponses_websocket.value: True}
|
||||
|
||||
(
|
||||
provider_model,
|
||||
|
|
@ -2702,6 +2703,7 @@ async def _aresponses_websocket(
|
|||
litellm_metadata=_build_litellm_metadata_for_ws(kwargs),
|
||||
custom_llm_provider=_custom_llm_provider,
|
||||
request_defaults=_build_responses_websocket_request_defaults(deployment_kwargs),
|
||||
turn_admission=turn_admission,
|
||||
frame_guard=responses_ws_frame_guard,
|
||||
**remaining_kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import copy
|
||||
import json
|
||||
import time
|
||||
|
|
@ -198,7 +199,7 @@ def _running_loop() -> asyncio.AbstractEventLoop | None:
|
|||
return None
|
||||
|
||||
|
||||
def _spawn_logging_task(
|
||||
def spawn_logging_task(
|
||||
running_loop: asyncio.AbstractEventLoop, coroutine: Coroutine[object, object, object], *, task_name: str
|
||||
) -> asyncio.Task[object]:
|
||||
task: Final = running_loop.create_task(coroutine)
|
||||
|
|
@ -208,6 +209,14 @@ def _spawn_logging_task(
|
|||
return task
|
||||
|
||||
|
||||
async def drain_logging_tasks(timeout: float = 5.0) -> None:
|
||||
if not _PENDING_LOGGING_TASKS:
|
||||
return
|
||||
_, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout)
|
||||
if pending:
|
||||
verbose_logger.warning("Timed out draining %d logging tasks on shutdown, dropping them", len(pending))
|
||||
|
||||
|
||||
_ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType(
|
||||
{
|
||||
"server_error": 500,
|
||||
|
|
@ -930,7 +939,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
running_loop: Final = _running_loop()
|
||||
if running_loop is not None:
|
||||
self._record_pending_logging_task(
|
||||
_spawn_logging_task(
|
||||
spawn_logging_task(
|
||||
running_loop,
|
||||
async_post_call_success_deployment_hook(
|
||||
request_data=request_payload,
|
||||
|
|
@ -966,7 +975,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
running_loop: Final = _running_loop()
|
||||
if running_loop is not None:
|
||||
self._record_pending_logging_task(
|
||||
_spawn_logging_task(
|
||||
spawn_logging_task(
|
||||
running_loop,
|
||||
self._run_failure_handlers_in_order(exception, traceback_exception, end_time),
|
||||
task_name="Responses stream failure logging",
|
||||
|
|
@ -1947,6 +1956,38 @@ async def _enforce_frame_project_quota(
|
|||
)
|
||||
|
||||
|
||||
async def _admit_frame_turn(turn_admission: Callable[[], Awaitable[None]] | None, raw_message: str) -> Exception | None:
|
||||
frame: Final = _json_object_or_none(raw_message)
|
||||
if turn_admission is None or frame is None or frame.get("type") != "response.create":
|
||||
return None
|
||||
try:
|
||||
await turn_admission()
|
||||
except RateLimitError as refusal:
|
||||
return refusal
|
||||
except Exception as e:
|
||||
verbose_logger.exception("Responses WS turn admission failed: %s", e)
|
||||
return e
|
||||
return None
|
||||
|
||||
|
||||
def _turn_refusal_frame(refusal: RateLimitError) -> str:
|
||||
return json.dumps(
|
||||
{"type": "error", "status": 429, "error": {"type": "rate_limit_exceeded", "message": str(refusal)}}
|
||||
)
|
||||
|
||||
|
||||
def _turn_admission_error_frame(failure: Exception) -> str:
|
||||
return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}})
|
||||
|
||||
|
||||
def _turn_rejection_frame(rejection: Exception) -> str:
|
||||
match rejection:
|
||||
case RateLimitError():
|
||||
return _turn_refusal_frame(rejection)
|
||||
case _:
|
||||
return _turn_admission_error_frame(rejection)
|
||||
|
||||
|
||||
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
|
||||
"response.created",
|
||||
"response.completed",
|
||||
|
|
@ -2019,16 +2060,30 @@ def _unmask_responses_event(response_str: str, pii_tokens: Mapping[str, str], un
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class GuardedResponsesWsFrame:
|
||||
"""A ``response.create`` frame after the proxy's pre_call guardrails ran on it: the frame to forward (guardrails
|
||||
may rewrite ``input`` and ``instructions``), the guardrail records that bill against this turn only, and the
|
||||
Presidio ``pii_tokens`` the walk masked with, which the handler needs to unmask the provider's reply."""
|
||||
may rewrite ``input`` and ``instructions``), the guardrail records that bill against this turn only, the
|
||||
Presidio ``pii_tokens`` the walk masked with, which the handler needs to unmask the provider's reply, and how
|
||||
to carry those guardrail records onto the connection's failure row when the deployment then refuses the turn."""
|
||||
|
||||
frame: str
|
||||
guardrail_information: tuple[Mapping[str, object], ...] = ()
|
||||
pii_tokens: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({}))
|
||||
bill_refusal: Callable[[Exception], None] | None = None
|
||||
|
||||
|
||||
ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]]
|
||||
|
||||
|
||||
async def _reject_guarded_turn(
|
||||
websocket: ResponsesClientWebSocket, guarded: GuardedResponsesWsFrame, rejection: Exception
|
||||
) -> None:
|
||||
"""Answers the refused turn and leaves its guardrail records to the connection's close-time failure row, which
|
||||
runs every failure hook once per socket, like the first refusal it records."""
|
||||
if guarded.bill_refusal is not None:
|
||||
guarded.bill_refusal(rejection)
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.send_text(_turn_rejection_frame(rejection))
|
||||
|
||||
|
||||
_RESPONSES_WS_TERMINAL_EVENT_TYPES: Final = _HISTORY_TERMINAL_EVENT_TYPES | _RESPONSES_WS_FAILURE_EVENT_TYPES
|
||||
|
||||
|
||||
|
|
@ -2212,6 +2267,7 @@ class ResponsesWebSocketStreaming:
|
|||
authorized_model: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
|
||||
turn_admission: Callable[[], Awaitable[None]] | None = None,
|
||||
frame_guard: ResponsesWsFrameGuard | None = None,
|
||||
):
|
||||
self.websocket = websocket
|
||||
|
|
@ -2233,6 +2289,7 @@ class ResponsesWebSocketStreaming:
|
|||
# response.create frame to prevent deployment-substitution attacks.
|
||||
self.authorized_model: str | None = authorized_model
|
||||
self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults
|
||||
self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission
|
||||
|
||||
def _start_turn(
|
||||
self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = ()
|
||||
|
|
@ -2284,9 +2341,11 @@ class ResponsesWebSocketStreaming:
|
|||
await self._send_error_event("invalid_request_error", str(e))
|
||||
return None
|
||||
|
||||
async def _forward_guarded_frame(self, message: str) -> None:
|
||||
async def _forward_guarded_frame(
|
||||
self, message: str, turn_admission: Callable[[], Awaitable[None]] | None = None
|
||||
) -> None:
|
||||
guarded: Final = await self._guard_frame(message)
|
||||
if guarded is None:
|
||||
if guarded is None or not await self._admit_or_reject_turn(guarded, message, turn_admission):
|
||||
return
|
||||
if guarded.pii_tokens:
|
||||
self._remember_pii_tokens(guarded.pii_tokens)
|
||||
|
|
@ -2680,10 +2739,27 @@ class ResponsesWebSocketStreaming:
|
|||
self.quota_callbacks, self.user_api_key_dict, self.authorized_model, message
|
||||
)
|
||||
except RateLimitError as e:
|
||||
await self._send_error_event("rate_limit_exceeded", str(e))
|
||||
await self._send_turn_refusal(e)
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _admit_or_reject_turn(
|
||||
self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: Callable[[], Awaitable[None]] | None
|
||||
) -> bool:
|
||||
rejection: Final = await _admit_frame_turn(turn_admission, message)
|
||||
if rejection is None:
|
||||
return True
|
||||
if self._failure is None:
|
||||
self._failure = rejection
|
||||
await _reject_guarded_turn(self.websocket, guarded, rejection)
|
||||
return False
|
||||
|
||||
async def _send_turn_refusal(self, refusal: RateLimitError) -> None:
|
||||
try:
|
||||
await self.websocket.send_text(_turn_refusal_frame(refusal))
|
||||
except Exception: # noqa: BLE001, S110 # client may already be gone
|
||||
pass
|
||||
|
||||
async def _send_error_event(self, error_type: str, message: str) -> None:
|
||||
try:
|
||||
await self.websocket.send_text(
|
||||
|
|
@ -2702,7 +2778,7 @@ class ResponsesWebSocketStreaming:
|
|||
message = await self.websocket.receive_text()
|
||||
if not await self._enforce_or_reject_frame(message):
|
||||
continue
|
||||
await self._forward_guarded_frame(message)
|
||||
await self._forward_guarded_frame(message, self.turn_admission)
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Responses WS client_to_backend ended: %s", e)
|
||||
|
|
@ -2779,6 +2855,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
custom_llm_provider: str | None = None,
|
||||
first_message: str | None = None,
|
||||
quota_callbacks: Sequence[ProjectQuotaCallback] | None = None,
|
||||
turn_admission: Callable[[], Awaitable[None]] | None = None,
|
||||
frame_guard: ResponsesWsFrameGuard | None = None,
|
||||
guardrail_callbacks: Sequence[PresidioGuardrailCallback] | None = None,
|
||||
**kwargs: object,
|
||||
|
|
@ -2802,6 +2879,8 @@ class ManagedResponsesWebSocketHandler:
|
|||
self._connection_provider = self._resolve_provider(model) or custom_llm_provider
|
||||
self.first_message = first_message
|
||||
self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else ()
|
||||
self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission
|
||||
self._failure: Exception | None = None
|
||||
# Carry through safe pass-through kwargs (e.g. extra_headers)
|
||||
self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS}
|
||||
# In-memory session history: response_id → full accumulated message list.
|
||||
|
|
@ -2830,12 +2909,11 @@ class ManagedResponsesWebSocketHandler:
|
|||
return None
|
||||
|
||||
async def _send_error(self, message: str, error_type: str = "server_error") -> None:
|
||||
try:
|
||||
await self.websocket.send_text(
|
||||
json.dumps({"type": "error", "error": {"type": error_type, "message": message}})
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
await self._send_frame(json.dumps({"type": "error", "error": {"type": error_type, "message": message}}))
|
||||
|
||||
async def _send_frame(self, frame: str) -> None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.websocket.send_text(frame)
|
||||
|
||||
def _get_history_messages(self, previous_response_id: str) -> list[dict[str, object]]:
|
||||
"""
|
||||
|
|
@ -3197,7 +3275,9 @@ class ManagedResponsesWebSocketHandler:
|
|||
# Core request handler
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _process_response_create(self, raw_message: str) -> None:
|
||||
async def _process_response_create(
|
||||
self, raw_message: str, turn_admission: Callable[[], Awaitable[None]] | None = None
|
||||
) -> None:
|
||||
"""
|
||||
Parse one ``response.create`` event, call ``litellm.aresponses(stream=True)``,
|
||||
and forward every streaming event to the client.
|
||||
|
|
@ -3238,12 +3318,17 @@ class ManagedResponsesWebSocketHandler:
|
|||
self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message
|
||||
)
|
||||
except RateLimitError as e:
|
||||
await self._send_error(str(e), error_type="rate_limit_exceeded")
|
||||
await self._send_frame(_turn_refusal_frame(e))
|
||||
return
|
||||
|
||||
guarded: Final = await self._guard_frame(raw_message)
|
||||
if guarded is None:
|
||||
return
|
||||
rejection: Final = await _admit_frame_turn(turn_admission, raw_message)
|
||||
if rejection is not None:
|
||||
if self._failure is None:
|
||||
self._failure = rejection
|
||||
await _reject_guarded_turn(self.websocket, guarded, rejection)
|
||||
return
|
||||
if guarded.pii_tokens:
|
||||
self._pii_tokens = {**self._pii_tokens, **guarded.pii_tokens}
|
||||
|
||||
|
|
@ -3287,7 +3372,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
# Main entry point
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def run(self) -> None:
|
||||
async def run(self) -> Exception | None:
|
||||
"""
|
||||
Main loop: accept ``response.create`` events sequentially and handle
|
||||
each one before waiting for the next message.
|
||||
|
|
@ -3303,8 +3388,9 @@ class ManagedResponsesWebSocketHandler:
|
|||
verbose_logger.debug("ManagedResponsesWS: client disconnected: %s", exc)
|
||||
break
|
||||
|
||||
await self._process_response_create(message)
|
||||
await self._process_response_create(message, self.turn_admission)
|
||||
|
||||
except Exception as exc:
|
||||
verbose_logger.exception("ManagedResponsesWS: unexpected error: %s", exc)
|
||||
await self._send_error(f"Internal server error: {exc}")
|
||||
return self._failure
|
||||
|
|
|
|||
|
|
@ -289,6 +289,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
|
|||
deployment_has_io_token_limits,
|
||||
refund_stale_reservation_before_retry,
|
||||
set_io_token_rate_limit_request_kwargs,
|
||||
without_io_token_limits,
|
||||
)
|
||||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
|
||||
ModelRateLimitingCheck,
|
||||
|
|
@ -5965,6 +5966,7 @@ class Router:
|
|||
)
|
||||
|
||||
custom_llm_provider: Final = provider_for_generic_call(data)
|
||||
generic_function_name: Final[str] = getattr(original_generic_function, "__name__", "")
|
||||
|
||||
response_kwargs: Final = {
|
||||
**data,
|
||||
|
|
@ -5972,6 +5974,11 @@ class Router:
|
|||
**kwargs,
|
||||
"model": model_name,
|
||||
**_with_router_resolved_session_model(kwargs.get("session"), model_name),
|
||||
**(
|
||||
{"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs)}
|
||||
if generic_function_name == "_aresponses_websocket"
|
||||
else {}
|
||||
),
|
||||
}
|
||||
# Only set custom_llm_provider if it's not None
|
||||
if custom_llm_provider is not None:
|
||||
|
|
@ -5981,7 +5988,7 @@ class Router:
|
|||
self,
|
||||
deployment,
|
||||
response_kwargs,
|
||||
surface_for_call(getattr(original_generic_function, "__name__", "")),
|
||||
surface_for_call(generic_function_name),
|
||||
)
|
||||
async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span):
|
||||
response = await original_generic_function(**compacted_input)
|
||||
|
|
@ -8936,8 +8943,7 @@ class Router:
|
|||
Track remaining tpm/rpm quota for model in model_list
|
||||
"""
|
||||
try:
|
||||
# WS session wrappers fire with result=None; per-turn costs tracked by inner calls.
|
||||
if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"):
|
||||
if kwargs.get("call_type") == "_arealtime":
|
||||
return
|
||||
if is_batch_retrieve_call_type(kwargs.get("call_type")):
|
||||
return
|
||||
|
|
@ -9605,6 +9611,71 @@ class Router:
|
|||
)
|
||||
raise e
|
||||
|
||||
def _deployment_as_configured_now(self, deployment: Mapping[str, object]) -> Mapping[str, object]:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
model_id: Final = (
|
||||
_MODEL_INFO_ADAPTER.validate_python(model_info).get("id") if isinstance(model_info, dict) else None
|
||||
)
|
||||
current: Final = self.get_deployment(model_id) if isinstance(model_id, str) else None
|
||||
return deployment if current is None else current.model_dump(exclude_none=True)
|
||||
|
||||
async def async_admit_deployment_turn(
|
||||
self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object]
|
||||
) -> None:
|
||||
deployment: Final = self._deployment_as_configured_now(opened_on)
|
||||
budget_limiter: Final = self.get_router_deployment_budget_limiter()
|
||||
if budget_limiter is not None:
|
||||
try:
|
||||
await budget_limiter.async_filter_deployments( # pyright: ignore[reportUnknownMemberType] # legacy signature takes untyped deployment dicts
|
||||
model=model_group,
|
||||
healthy_deployments=[dict(deployment)],
|
||||
messages=None,
|
||||
request_kwargs=dict(request_kwargs),
|
||||
)
|
||||
except ValueError as e:
|
||||
raise litellm.RateLimitError(
|
||||
message=str(e),
|
||||
llm_provider="",
|
||||
model=model_group,
|
||||
category=litellm.RateLimitErrorCategory.LITELLM_RATE_LIMIT,
|
||||
) from e
|
||||
await self._admit_turn_through_routing_strategy(model_group, deployment, request_kwargs)
|
||||
await self.async_routing_strategy_pre_call_checks( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict
|
||||
deployment=dict(without_io_token_limits(deployment)), parent_otel_span=None
|
||||
)
|
||||
|
||||
async def _admit_turn_through_routing_strategy(
|
||||
self, model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object]
|
||||
) -> None:
|
||||
routing_kwargs: Final = {key: value for key, value in request_kwargs.items() if key != "litellm_logging_obj"}
|
||||
strategy, selector = self._get_routing_context( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped request dict
|
||||
model=model_group, request_kwargs=routing_kwargs
|
||||
)
|
||||
if strategy == "simple-shuffle" or selector is None:
|
||||
return
|
||||
selected: Final = await self._select_deployment_async( # pyright: ignore[reportUnknownMemberType] # legacy signature takes untyped deployment dicts
|
||||
strategy=strategy,
|
||||
selector=selector,
|
||||
model=model_group,
|
||||
healthy_deployments=[dict(deployment)],
|
||||
messages=None,
|
||||
input=None,
|
||||
request_kwargs=routing_kwargs,
|
||||
)
|
||||
if selected is None:
|
||||
no_deployment: Final = await async_raise_no_deployment_exception(
|
||||
litellm_router_instance=self, model=model_group, parent_otel_span=None
|
||||
)
|
||||
raise litellm.RateLimitError(
|
||||
message=str(no_deployment),
|
||||
llm_provider="",
|
||||
model=model_group,
|
||||
category=litellm.RateLimitErrorCategory.LITELLM_RATE_LIMIT,
|
||||
) from no_deployment
|
||||
await self._async_override_selector_pre_call_check( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict
|
||||
strategy, selector, dict(deployment), None
|
||||
)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _deployment_slot(
|
||||
self, deployment: dict, kwargs: Mapping[str, object], parent_otel_span: Span | None
|
||||
|
|
|
|||
|
|
@ -505,8 +505,7 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""Original method now uses helper functions"""
|
||||
verbose_router_logger.debug("in RouterBudgetLimiting.async_log_success_event")
|
||||
# WS session wrappers fire with result=None; per-turn costs tracked by inner calls.
|
||||
if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"):
|
||||
if kwargs.get("call_type") == "_arealtime":
|
||||
return
|
||||
standard_logging_payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
|
||||
if standard_logging_payload is None:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
|
||||
from litellm.types.router import RouterCacheEnum, RouterErrors
|
||||
from litellm.utils import get_utc_datetime
|
||||
|
|
@ -96,6 +97,24 @@ def deployment_has_io_token_limits(deployment: dict) -> bool:
|
|||
return itpm is not None or otpm is not None
|
||||
|
||||
|
||||
_IO_TOKEN_LIMIT_KEYS: Final = frozenset({"itpm", "otpm"})
|
||||
_IO_TOKEN_LIMIT_SECTIONS: Final = frozenset({"litellm_params", "model_info"})
|
||||
|
||||
|
||||
def without_io_token_limits(deployment: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {
|
||||
key: _without_io_token_limit_keys(value) if key in _IO_TOKEN_LIMIT_SECTIONS else value
|
||||
for key, value in deployment.items()
|
||||
if key not in _IO_TOKEN_LIMIT_KEYS
|
||||
}
|
||||
|
||||
|
||||
def _without_io_token_limit_keys(section: object) -> object:
|
||||
if not is_str_keyed_mapping(section):
|
||||
return section
|
||||
return {key: value for key, value in section.items() if key not in _IO_TOKEN_LIMIT_KEYS}
|
||||
|
||||
|
||||
def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] | None:
|
||||
model_id: Final = deployment.get("model_info", {}).get("id")
|
||||
deployment_name: Final = deployment.get("litellm_params", {}).get("model")
|
||||
|
|
|
|||
|
|
@ -8306,8 +8306,8 @@ def get_base_model_from_metadata(model_call_details: Mapping[str, object] | None
|
|||
litellm_params: Final = cast(Mapping[str, object], model_call_details.get("litellm_params", {}))
|
||||
if litellm_params is not None:
|
||||
base_model: Final = litellm_params.get("base_model", None)
|
||||
if base_model is not None:
|
||||
return cast(str | None, base_model)
|
||||
if isinstance(base_model, str):
|
||||
return base_model
|
||||
metadata: Final = cast(Mapping[str, object], litellm_params.get("metadata") or {})
|
||||
base_model_metadata_getter: Final[_BaseModelFromMetadataGetter] = (
|
||||
litellm_utils.get_base_model_from_litellm_call_metadata
|
||||
|
|
|
|||
|
|
@ -47779,6 +47779,34 @@
|
|||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"tsubasa/tsubasa-fast": {
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 2.0000000000000002e-07,
|
||||
"output_cost_per_token": 1e-06,
|
||||
"litellm_provider": "tsubasa",
|
||||
"mode": "chat",
|
||||
"supports_system_messages": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": false,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"tsubasa/tsubasa-pro": {
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 16384,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"output_cost_per_token": 4e-06,
|
||||
"litellm_provider": "tsubasa",
|
||||
"mode": "chat",
|
||||
"supports_system_messages": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": false,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"together_ai/BAAI/bge-base-en-v1.5": {
|
||||
"input_cost_per_token": 8e-09,
|
||||
"litellm_provider": "together_ai",
|
||||
|
|
@ -76083,32 +76111,22 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"tsubasa/tsubasa-fast": {
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 2.0000000000000002e-07,
|
||||
"output_cost_per_token": 1e-06,
|
||||
"litellm_provider": "tsubasa",
|
||||
"mode": "chat",
|
||||
"supports_system_messages": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": false,
|
||||
"supports_reasoning": false
|
||||
},
|
||||
"tsubasa/tsubasa-pro": {
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 16384,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"output_cost_per_token": 4e-06,
|
||||
"litellm_provider": "tsubasa",
|
||||
"mode": "chat",
|
||||
"supports_system_messages": true,
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": false,
|
||||
"supports_reasoning": false
|
||||
"meta/muse-image-1.0": {
|
||||
"litellm_provider": "meta",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.01,
|
||||
"source": "https://dev.meta.ai/docs/pricing-rate-limits",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"image"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -134,6 +134,9 @@ ignored_function_names = [
|
|||
"_provider_model_id", # Tested through get_router_model_info in test_router_model_cost_isolation.py
|
||||
"_names_route_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py
|
||||
"_declared_base_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py
|
||||
"async_admit_deployment_turn", # Bound as the WebSocket turn_admission partial, tested through it in test_router_helper_utils.py
|
||||
"_deployment_as_configured_now", # Read by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py
|
||||
"_admit_turn_through_routing_strategy", # Run by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1544,6 +1544,73 @@ GEMINI_3_1_PRO_PASSTHROUGH_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY: Final = CostT
|
|||
),
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY: Final = CostTrackingTestCase(
|
||||
name="gemini-3.1-pro-passthrough-stream_generate_content_priced_via_gemini_key",
|
||||
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
|
||||
model="gemini/gemini-3.1-pro",
|
||||
endpoint="/gemini/v1beta/models/$MODEL:streamGenerateContent?alt=sse",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "52b6a80ff038 summarize the attached material in one line and name the city weather",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
"allowed_openai_params": [],
|
||||
},
|
||||
response=SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": "scripted answer 52b6a80ff038"}], "role": "model"},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"modelVersion": "gemini-3.1-pro",
|
||||
},
|
||||
{
|
||||
"candidates": [],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 1840,
|
||||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro",
|
||||
},
|
||||
),
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.008624,
|
||||
input_cost=0.00368,
|
||||
output_cost=0.004944,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
breakdown_persisted=False,
|
||||
cost_header=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# gemini/gemini-3.8-flash
|
||||
GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase(
|
||||
|
|
@ -3475,6 +3542,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GEMINI_GEMINI_3_8_FLASH_STREAM_FULL_USAGE,
|
||||
GEMINI_EMBEDDINGS_002,
|
||||
GEMINI_3_1_PRO_PASSTHROUGH_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY,
|
||||
GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY,
|
||||
GEMINI_GEMINI_3_8_FLASH_LITE_INPUT_BELOW_128K,
|
||||
GEMINI_GEMINI_3_8_FLASH_LITE_INPUT_ABOVE_128K,
|
||||
GEMINI_NANO_BANANA_2_1_INPUT_TEXT,
|
||||
|
|
|
|||
|
|
@ -1659,73 +1659,6 @@ GEMINI_3_1_PRO_STREAM_FULL_USAGE: Final = CostTrackingTestCase(
|
|||
),
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_VERTEX_KEY: Final = CostTrackingTestCase(
|
||||
name="gemini-3.1-pro-passthrough-stream_generate_content_priced_via_vertex_key",
|
||||
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
|
||||
model="gemini-3.1-pro",
|
||||
endpoint="/gemini/v1beta/models/$MODEL:streamGenerateContent?alt=sse",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "52b6a80ff038 summarize the attached material in one line and name the city weather",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
"allowed_openai_params": [],
|
||||
},
|
||||
response=SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": "scripted answer 52b6a80ff038"}], "role": "model"},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"modelVersion": "gemini-3.1-pro",
|
||||
},
|
||||
{
|
||||
"candidates": [],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 1840,
|
||||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro",
|
||||
},
|
||||
),
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0090552,
|
||||
input_cost=0.003864,
|
||||
output_cost=0.0051912,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
breakdown_persisted=False,
|
||||
cost_header=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# gemini-3.8-flash
|
||||
GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase(
|
||||
|
|
@ -3647,7 +3580,6 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GEMINI_3_8_FLASH_STREAM_FULL_USAGE,
|
||||
IMAGEN_NEXT_IMAGES_ONE,
|
||||
VERTEX_EMBEDDINGS_TEXT_006,
|
||||
GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_VERTEX_KEY,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_STREAM,
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,629 @@
|
|||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Callable, Iterator, Mapping
|
||||
from contextlib import asynccontextmanager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import websockets
|
||||
import yaml
|
||||
from integration._support.client import (
|
||||
JSON_OBJECT,
|
||||
Gateway,
|
||||
Scenario,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
held,
|
||||
list_value,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.responses_ws_peer import (
|
||||
PROVIDER_MODEL,
|
||||
USAGE,
|
||||
Peer,
|
||||
Script,
|
||||
completed,
|
||||
create,
|
||||
drain,
|
||||
events,
|
||||
responses_peer,
|
||||
turn,
|
||||
until_terminal,
|
||||
)
|
||||
from integration._support.tls import write_self_signed_cert
|
||||
from pydantic import JsonValue
|
||||
from redis import Redis
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 240)
|
||||
|
||||
PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
|
||||
INPUT_RATE: Final = 0.001
|
||||
OUTPUT_RATE: Final = 0.002
|
||||
TURN_SPEND: Final = USAGE["input_tokens"] * INPUT_RATE + USAGE["output_tokens"] * OUTPUT_RATE
|
||||
TURN_TOKENS: Final = USAGE["total_tokens"]
|
||||
MANAGED_PROVIDER: Final = "mistral"
|
||||
SPEND_ROWS_FOR_KEY: Final = (
|
||||
'SELECT request_id, spend, status, model_group, metadata, request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s'
|
||||
)
|
||||
TURN_CHECK_FAULT: Final = "turn_check_fault"
|
||||
FAULTING_TURN_CHECK: Final = f"""
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
|
||||
class FaultingTurnCheck(CustomLogger):
|
||||
async def async_pre_call_check(self, deployment, parent_otel_span):
|
||||
fault = (deployment.get("model_info") or {{}}).get("{TURN_CHECK_FAULT}")
|
||||
if fault:
|
||||
raise RuntimeError(fault)
|
||||
return None
|
||||
|
||||
|
||||
instance = FaultingTurnCheck()
|
||||
"""
|
||||
FrameReader = Callable[[], int]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Deployment:
|
||||
name: str
|
||||
identity: str
|
||||
forwarded_turns: FrameReader
|
||||
|
||||
|
||||
def _key_hash(key: str) -> str:
|
||||
return hashlib.sha256(key.encode()).hexdigest()
|
||||
|
||||
|
||||
def _redis() -> Redis:
|
||||
return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), decode_responses=True)
|
||||
|
||||
|
||||
def _config(directory: Path, router_settings: Mapping[str, JsonValue], callbacks: tuple[str, ...]) -> Path:
|
||||
base: Final = JSON_OBJECT.validate_python(yaml.safe_load(PROXY_CONFIG.read_text()))
|
||||
path: Final = directory / "turn-admission.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**base,
|
||||
"litellm_settings": {
|
||||
**object_value(base["litellm_settings"]),
|
||||
**({"callbacks": list(callbacks)} if callbacks else {}),
|
||||
},
|
||||
"router_settings": {
|
||||
**object_value(base["router_settings"]),
|
||||
"redis_host": os.environ["REDIS_HOST"],
|
||||
"redis_port": int(os.environ["REDIS_PORT"]),
|
||||
**router_settings,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _admitting_proxy(
|
||||
tmp_path_factory: pytest.TempPathFactory,
|
||||
cert: tuple[Path, Path],
|
||||
router_settings: Mapping[str, JsonValue],
|
||||
callback_sources: Mapping[str, str] = MappingProxyType({}),
|
||||
) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("responses-ws-turn-admission")
|
||||
for module, source in callback_sources.items():
|
||||
(directory / f"{module}.py").write_text(source)
|
||||
config: Final = _config(directory, router_settings, tuple(f"{module}.instance" for module in callback_sources))
|
||||
with gateway_from_environment() as base:
|
||||
with owned_proxy(base, directory, {"SSL_CERT_FILE": str(cert[0])}, config=config) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def cert(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]:
|
||||
return write_self_signed_cert(tmp_path_factory.mktemp("responses-ws-turn-admission-cert"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def limited(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]:
|
||||
with _admitting_proxy(tmp_path_factory, cert, {"optional_pre_call_checks": ["enforce_model_rate_limits"]}) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def faulting(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]:
|
||||
with _admitting_proxy(tmp_path_factory, cert, {}, {"faulting_turn_check": FAULTING_TURN_CHECK}) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def usage_routed(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]:
|
||||
with _admitting_proxy(tmp_path_factory, cert, {"routing_strategy": "usage-based-routing-v2"}) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
||||
def _native(scenario: Scenario, peer: Peer, **limits: JsonValue) -> Deployment:
|
||||
identity: Final = f"ws-admission-{uuid.uuid4().hex}"
|
||||
name: Final = scenario.model(
|
||||
model_info={"id": identity},
|
||||
model=f"openai/{PROVIDER_MODEL}",
|
||||
api_base=peer.url,
|
||||
input_cost_per_token=INPUT_RATE,
|
||||
output_cost_per_token=OUTPUT_RATE,
|
||||
**limits,
|
||||
)
|
||||
return Deployment(
|
||||
name, identity, lambda: sum(frame.get("type") == "response.create" for frame in drain(peer.frames))
|
||||
)
|
||||
|
||||
|
||||
def _upstream_chats(gateway: Gateway, provider_model: str) -> int:
|
||||
with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as upstream:
|
||||
observed: Final = JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"]
|
||||
assert isinstance(observed, list)
|
||||
return sum(object_value(object_value(entry)["body"]).get("model") == provider_model for entry in observed)
|
||||
|
||||
|
||||
def _managed(scenario: Scenario, **limits: JsonValue) -> Deployment:
|
||||
identity: Final = f"ws-admission-{uuid.uuid4().hex}"
|
||||
provider_model: Final = f"ws-managed-{uuid.uuid4().hex}"
|
||||
name: Final = scenario.model(
|
||||
model_info={"id": identity},
|
||||
model=f"{MANAGED_PROVIDER}/{provider_model}",
|
||||
input_cost_per_token=INPUT_RATE,
|
||||
output_cost_per_token=OUTPUT_RATE,
|
||||
**limits,
|
||||
)
|
||||
return Deployment(name, identity, lambda: _upstream_chats(scenario.gateway, provider_model))
|
||||
|
||||
|
||||
def _socket_url(gateway: Gateway, model: str) -> str:
|
||||
return f"{str(gateway.client.base_url).rstrip('/').replace('http://', 'ws://')}/v1/responses?model={model}"
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _socket(
|
||||
gateway: Gateway, key: str, model: str, headers: Mapping[str, str] | None = None
|
||||
) -> AsyncIterator[websockets.ClientConnection]:
|
||||
async with websockets.connect(
|
||||
_socket_url(gateway, model),
|
||||
additional_headers={"Authorization": f"Bearer {key}", **(headers or {})},
|
||||
open_timeout=10,
|
||||
) as connection:
|
||||
yield connection
|
||||
|
||||
|
||||
async def _say(connection: websockets.ClientConnection, model: str, text: str = "") -> tuple[dict[str, JsonValue], ...]:
|
||||
return await turn(connection, create(model, text or f"turn {uuid.uuid4().hex}"))
|
||||
|
||||
|
||||
def _assert_refused(received: tuple[dict[str, JsonValue], ...]) -> None:
|
||||
assert [event.get("type") for event in received] == ["error"], received
|
||||
assert received[0]["status"] == 429, received
|
||||
assert object_value(received[0]["error"])["type"] == "rate_limit_exceeded", received
|
||||
|
||||
|
||||
def _fresh_minute() -> str:
|
||||
eventually(lambda: datetime.now(UTC), lambda now: now.second < 30, seconds=65)
|
||||
return datetime.now(UTC).strftime("%H-%M")
|
||||
|
||||
|
||||
def _deployment_spend(deployment: Deployment) -> float:
|
||||
with _redis() as redis:
|
||||
return float(redis.get(f"deployment_spend:{deployment.identity}:1d") or 0)
|
||||
|
||||
|
||||
def _configured(gateway: Gateway, deployment: Deployment, section: str) -> dict[str, JsonValue]:
|
||||
entries: Final = list_value(gateway.get("/model/info", {"litellm_model_id": deployment.identity})["data"])
|
||||
return object_value(object_value(entries[0])[section])
|
||||
|
||||
|
||||
def _configured_tpm(gateway: Gateway, deployment: Deployment) -> JsonValue:
|
||||
return _configured(gateway, deployment, "litellm_params").get("tpm")
|
||||
|
||||
|
||||
def _tokens_counted(identity: str, minute: str) -> int:
|
||||
with _redis() as redis:
|
||||
return sum(int(redis.get(key) or 0) for key in redis.scan_iter(f"{identity}:*:tpm:{minute}"))
|
||||
|
||||
|
||||
def _settled_count(read: Callable[[], int]) -> int:
|
||||
reached: Final = eventually(read, lambda tokens: tokens > 0, seconds=15)
|
||||
return held(read, lambda tokens: tokens == reached, holding=3, seconds=20)
|
||||
|
||||
|
||||
async def _turns_against_tpm(
|
||||
gateway: Gateway, key: str, deployment: Deployment
|
||||
) -> tuple[tuple[dict[str, JsonValue], ...], tuple[dict[str, JsonValue], ...], tuple[dict[str, JsonValue], ...], int]:
|
||||
minute: Final = await asyncio.to_thread(_fresh_minute)
|
||||
tokens: Final = partial(_tokens_counted, deployment.identity, minute)
|
||||
async with _socket(gateway, key, deployment.name) as connection:
|
||||
first: Final = await _say(connection, deployment.name)
|
||||
counted_after_first: Final = await asyncio.to_thread(_settled_count, tokens)
|
||||
second: Final = await _say(connection, deployment.name)
|
||||
if second[-1].get("type") == "response.completed":
|
||||
await _until(tokens, lambda counted: counted > counted_after_first)
|
||||
return first, second, await _say(connection, deployment.name), counted_after_first
|
||||
|
||||
|
||||
async def _until(read: Callable[[], float], satisfied: Callable[[float], bool]) -> float:
|
||||
return await asyncio.to_thread(eventually, read, satisfied, 15)
|
||||
|
||||
|
||||
def _failure_rows(key: str) -> list[dict[str, JsonValue]]:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
|
||||
lambda found: any(row["status"] == "failure" for row in found),
|
||||
seconds=70,
|
||||
)
|
||||
return [row for row in rows if row["status"] == "failure"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("relay", ("native", "managed"))
|
||||
def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_logged_once_as_its_failure(
|
||||
limited: Gateway, cert: tuple[Path, Path], relay: str
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, limited.scenario() as scenario:
|
||||
budget: Final = {"max_budget": TURN_SPEND / 2, "budget_duration": "1d"}
|
||||
deployment: Final = _native(scenario, peer, **budget) if relay == "native" else _managed(scenario, **budget)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
tag: Final = f"ws-turn-{uuid.uuid4().hex}"
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(limited, key, deployment.name, {"x-litellm-tags": tag}) as connection:
|
||||
first: Final = await _say(connection, deployment.name)
|
||||
await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2)
|
||||
return (first, await _say(connection, deployment.name), await _say(connection, deployment.name))
|
||||
|
||||
first, second, third = asyncio.run(session())
|
||||
|
||||
completed(first)
|
||||
_assert_refused(second)
|
||||
_assert_refused(third)
|
||||
assert deployment.forwarded_turns() == 1
|
||||
failures: Final = _failure_rows(key)
|
||||
assert len(failures) == 1, failures
|
||||
assert failures[0]["model_group"] == deployment.name, failures
|
||||
assert tag in list_value(failures[0]["request_tags"]), failures
|
||||
assert float(failures[0]["spend"]) == 0.0, failures
|
||||
error: Final = object_value(object_value(failures[0]["metadata"])["error_information"])
|
||||
assert str(error["error_code"]) == "429", error
|
||||
|
||||
|
||||
def _set_turn_check_fault(gateway: Gateway, deployment: Deployment, fault: str) -> None:
|
||||
gateway.request(
|
||||
"PATCH",
|
||||
f"/model/{deployment.identity}/update",
|
||||
{"model_info": {"id": deployment.identity, TURN_CHECK_FAULT: fault}},
|
||||
).raise_for_status()
|
||||
eventually(
|
||||
lambda: _configured(gateway, deployment, "model_info").get(TURN_CHECK_FAULT) or "",
|
||||
lambda configured: configured == fault,
|
||||
seconds=15,
|
||||
)
|
||||
|
||||
|
||||
def _assert_turn_check_failed(received: tuple[dict[str, JsonValue], ...], fault: str) -> None:
|
||||
assert [event.get("type") for event in received] == ["error"], received
|
||||
assert received[0]["status"] == 500, received
|
||||
error: Final = object_value(received[0]["error"])
|
||||
assert error["type"] == "server_error", received
|
||||
assert fault in string_value(error["message"]), received
|
||||
|
||||
|
||||
def _first_turn_held_until(release: threading.Event) -> Script:
|
||||
def script(response_id: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
if response_id == "resp_peer_1":
|
||||
release.wait(timeout=60)
|
||||
return events(f"resp_{uuid.uuid4().hex}", "seven")
|
||||
|
||||
return script
|
||||
|
||||
|
||||
@pytest.mark.parametrize("relay", ("native", "managed"))
|
||||
def test_a_turn_whose_admission_check_fails_gets_a_500_frame_and_the_socket_serves_the_next_turn(
|
||||
faulting: Gateway, cert: tuple[Path, Path], relay: str
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, faulting.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer) if relay == "native" else _managed(scenario)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
fault: Final = f"turn check unavailable {uuid.uuid4().hex}"
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(faulting, key, deployment.name) as connection:
|
||||
first: Final = await _say(connection, deployment.name)
|
||||
await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, fault)
|
||||
second: Final = await _say(connection, deployment.name)
|
||||
third: Final = await _say(connection, deployment.name)
|
||||
await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, "")
|
||||
return first, second, third, await _say(connection, deployment.name)
|
||||
|
||||
first, second, third, fourth = asyncio.run(session())
|
||||
|
||||
completed(first)
|
||||
_assert_turn_check_failed(second, fault)
|
||||
_assert_turn_check_failed(third, fault)
|
||||
completed(fourth)
|
||||
assert deployment.forwarded_turns() == 2
|
||||
failures: Final = _failure_rows(key)
|
||||
assert len(failures) == 1, failures
|
||||
assert failures[0]["model_group"] == deployment.name, failures
|
||||
recorded: Final = object_value(object_value(failures[0]["metadata"])["error_information"])
|
||||
assert fault in string_value(recorded["error_message"]), recorded
|
||||
|
||||
|
||||
def test_a_turn_still_streaming_when_the_next_turns_check_fails_completes_and_is_billed(
|
||||
faulting: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
release: Final = threading.Event()
|
||||
with responses_peer(cert, _first_turn_held_until(release)) as peer, faulting.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
fault: Final = f"turn check unavailable {uuid.uuid4().hex}"
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(faulting, key, deployment.name) as connection:
|
||||
try:
|
||||
await connection.send(create(deployment.name, f"streaming {uuid.uuid4().hex}"))
|
||||
await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, fault)
|
||||
rejected: Final = await _say(connection, deployment.name)
|
||||
finally:
|
||||
release.set()
|
||||
return rejected, await until_terminal(connection, ())
|
||||
|
||||
rejected, streamed = asyncio.run(session())
|
||||
|
||||
_assert_turn_check_failed(rejected, fault)
|
||||
billed: Final = completed(streamed)
|
||||
assert deployment.forwarded_turns() == 1
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
|
||||
lambda found: any(row["request_id"] == billed["id"] for row in found),
|
||||
seconds=70,
|
||||
)
|
||||
row: Final = next(row for row in rows if row["request_id"] == billed["id"])
|
||||
assert row["status"] == "success", row
|
||||
assert float(row["spend"]) == pytest.approx(TURN_SPEND), row
|
||||
|
||||
|
||||
def test_a_socket_opened_after_websocket_turns_spent_the_budget_is_refused_and_so_is_http(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, limited.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer, max_budget=TURN_SPEND / 2, budget_duration="1d")
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
|
||||
async def spend_then_reopen() -> tuple[tuple[dict[str, JsonValue], ...], int | None]:
|
||||
async with _socket(limited, key, deployment.name) as spending:
|
||||
completed(await _say(spending, deployment.name))
|
||||
await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2)
|
||||
async with _socket(limited, key, deployment.name) as refused:
|
||||
received: Final = await _say(refused, deployment.name)
|
||||
await asyncio.wait_for(refused.wait_closed(), timeout=20)
|
||||
return received, refused.close_code
|
||||
|
||||
received, close_code = asyncio.run(spend_then_reopen())
|
||||
|
||||
_assert_refused(received)
|
||||
assert close_code == 1011, close_code
|
||||
assert peer.paths.qsize() == 1
|
||||
assert deployment.forwarded_turns() == 1
|
||||
http: Final = limited.request(
|
||||
"POST", "/v1/responses", {"model": deployment.name, "input": f"after {uuid.uuid4().hex}"}, key=key
|
||||
)
|
||||
assert http.status_code == 429, http.text
|
||||
assert peer.paths.qsize() == 1
|
||||
|
||||
|
||||
def test_a_socket_the_router_refuses_to_open_gets_the_status_and_message_http_returns(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, limited.scenario() as scenario:
|
||||
name: Final = scenario.model(
|
||||
model=f"openai/{PROVIDER_MODEL}",
|
||||
api_base=peer.url,
|
||||
tags=[f"granted-{uuid.uuid4().hex}"],
|
||||
model_info={"enable_tag_filtering": True},
|
||||
)
|
||||
key: Final = scenario.key(models=[name])
|
||||
foreign: Final = {"x-litellm-tags": f"foreign-{uuid.uuid4().hex}"}
|
||||
http: Final = limited.request(
|
||||
"POST", "/v1/responses", {"model": name, "input": "tag routing"}, key=key, headers=foreign
|
||||
)
|
||||
assert http.status_code != 200, http.text
|
||||
http_error: Final = object_value(JSON_OBJECT.validate_json(http.content)["error"])
|
||||
|
||||
async def refused_open() -> tuple[dict[str, JsonValue], ...]:
|
||||
async with _socket(limited, key, name, foreign) as connection:
|
||||
return await _say(connection, name)
|
||||
|
||||
received: Final = asyncio.run(refused_open())
|
||||
|
||||
assert [event.get("type") for event in received] == ["error"], received
|
||||
assert received[0]["status"] == http.status_code, (received, http.text)
|
||||
assert object_value(received[0]["error"])["message"] == string_value(http_error["message"]), (
|
||||
received,
|
||||
http.text,
|
||||
)
|
||||
assert peer.paths.qsize() == 0
|
||||
|
||||
|
||||
def test_each_turn_counts_as_one_request_toward_the_deployment_rpm(limited: Gateway, cert: tuple[Path, Path]) -> None:
|
||||
with responses_peer(cert) as peer, limited.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer, rpm=1)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
_fresh_minute()
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(limited, key, deployment.name) as connection:
|
||||
return (await _say(connection, deployment.name), await _say(connection, deployment.name))
|
||||
|
||||
first, second = asyncio.run(session())
|
||||
|
||||
completed(first)
|
||||
_assert_refused(second)
|
||||
assert deployment.forwarded_turns() == 1
|
||||
|
||||
|
||||
def test_a_deployment_tpm_refuses_a_turn_only_once_earlier_turns_used_it_up(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, limited.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer, tpm=TURN_TOKENS + 2)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
|
||||
first, second, third, counted_after_first = asyncio.run(_turns_against_tpm(limited, key, deployment))
|
||||
|
||||
completed(first)
|
||||
completed(second)
|
||||
_assert_refused(third)
|
||||
assert counted_after_first == TURN_TOKENS
|
||||
assert deployment.forwarded_turns() == 2
|
||||
|
||||
|
||||
def test_a_tpm_limit_lowered_while_the_socket_is_open_applies_to_its_next_turn(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, limited.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer, tpm=100 * TURN_TOKENS)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
minute: Final = _fresh_minute()
|
||||
|
||||
def lower_tpm() -> None:
|
||||
limited.request(
|
||||
"PATCH", f"/model/{deployment.identity}/update", {"litellm_params": {"tpm": TURN_TOKENS - 2}}
|
||||
).raise_for_status()
|
||||
eventually(lambda: _configured_tpm(limited, deployment), lambda tpm: tpm == TURN_TOKENS - 2, seconds=15)
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(limited, key, deployment.name) as connection:
|
||||
first: Final = await _say(connection, deployment.name)
|
||||
await _until(partial(_tokens_counted, deployment.identity, minute), lambda tokens: tokens > 0)
|
||||
await asyncio.to_thread(lower_tpm)
|
||||
return (first, await _say(connection, deployment.name))
|
||||
|
||||
first, second = asyncio.run(session())
|
||||
|
||||
completed(first)
|
||||
_assert_refused(second)
|
||||
assert deployment.forwarded_turns() == 1
|
||||
|
||||
|
||||
def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_turn(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, limited.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer, otpm=10_000)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
_fresh_minute()
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(limited, key, deployment.name) as connection:
|
||||
return tuple([await _say(connection, deployment.name) for _ in range(5)])
|
||||
|
||||
turns: Final = asyncio.run(session())
|
||||
|
||||
assert [completed(received)["status"] for received in turns] == ["completed"] * 5
|
||||
assert deployment.forwarded_turns() == 5
|
||||
|
||||
|
||||
def test_usage_based_routing_refuses_a_turn_only_once_earlier_turns_used_up_the_deployment_tpm(
|
||||
usage_routed: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, usage_routed.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer, tpm=TURN_TOKENS + 2)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
|
||||
first, second, third, counted_after_first = asyncio.run(_turns_against_tpm(usage_routed, key, deployment))
|
||||
|
||||
completed(first)
|
||||
completed(second)
|
||||
_assert_refused(third)
|
||||
assert counted_after_first == TURN_TOKENS
|
||||
assert deployment.forwarded_turns() == 2
|
||||
|
||||
|
||||
def _assert_http_chat_counted_once(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
identity: Final = f"ws-admission-{uuid.uuid4().hex}"
|
||||
name: Final = scenario.model(model_info={"id": identity}, tpm=1_000_000)
|
||||
key: Final = scenario.key(models=[name])
|
||||
minute: Final = _fresh_minute()
|
||||
answered: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": name, "messages": [{"role": "user", "content": f"http turn {uuid.uuid4().hex}"}]},
|
||||
key=key,
|
||||
)
|
||||
assert answered.status_code == 200, answered.text
|
||||
tokens: Final = object_value(JSON_OBJECT.validate_json(answered.content)["usage"])["total_tokens"]
|
||||
assert isinstance(tokens, int) and tokens > 0, answered.text
|
||||
|
||||
assert _settled_count(partial(_tokens_counted, identity, minute)) == tokens
|
||||
|
||||
|
||||
def test_an_http_call_counts_its_tokens_once_toward_the_deployment_tpm_under_model_rate_limits(
|
||||
limited: Gateway,
|
||||
) -> None:
|
||||
_assert_http_chat_counted_once(limited)
|
||||
|
||||
|
||||
def test_an_http_call_counts_its_tokens_once_toward_the_deployment_tpm_under_usage_based_routing(
|
||||
usage_routed: Gateway,
|
||||
) -> None:
|
||||
_assert_http_chat_counted_once(usage_routed)
|
||||
|
||||
|
||||
def test_usage_based_routing_serves_every_turn_while_the_deployment_has_headroom(
|
||||
usage_routed: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
with responses_peer(cert) as peer, usage_routed.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer, tpm=1_000_000, rpm=1_000)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(usage_routed, key, deployment.name) as connection:
|
||||
return tuple([await _say(connection, deployment.name) for _ in range(3)])
|
||||
|
||||
turns: Final = asyncio.run(session())
|
||||
|
||||
assert [completed(received)["status"] for received in turns] == ["completed"] * 3
|
||||
assert deployment.forwarded_turns() == 3
|
||||
|
||||
|
||||
def test_a_frame_over_the_project_input_quota_gets_a_429_error_frame_and_never_reaches_the_provider(
|
||||
limited: Gateway,
|
||||
) -> None:
|
||||
with limited.scenario() as scenario:
|
||||
deployment: Final = _managed(scenario)
|
||||
team: Final = scenario.team()
|
||||
key: Final = scenario.key(
|
||||
models=[deployment.name],
|
||||
team_id=team,
|
||||
project_id=scenario.project(team, model_itpm_limit={deployment.name: 100}),
|
||||
)
|
||||
|
||||
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
|
||||
async with _socket(limited, key, deployment.name) as connection:
|
||||
return (
|
||||
await _say(connection, deployment.name, "short"),
|
||||
await _say(connection, deployment.name, "long " * 400),
|
||||
await _say(connection, deployment.name, "long " * 400),
|
||||
)
|
||||
|
||||
first, second, third = asyncio.run(session())
|
||||
|
||||
completed(first)
|
||||
_assert_refused(third)
|
||||
answered: Final = sum(received[-1].get("type") == "response.completed" for received in (first, second, third))
|
||||
assert deployment.forwarded_turns() == answered
|
||||
|
|
@ -1,11 +1,13 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.client import Gateway, eventually, list_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
||||
@pytest.mark.covers("spend.failed_dispatch.failure_row_records_estimated_input_tokens")
|
||||
|
|
@ -48,3 +50,79 @@ def test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_
|
|||
assert row["status"] == "failure" and float(row["spend"]) == 0 and row["completion_tokens"] == 0, row
|
||||
assert row["prompt_tokens"] > 0, f"failure row lost the dispatched input tokens: {row}"
|
||||
assert row["total_tokens"] == row["prompt_tokens"], row
|
||||
|
||||
|
||||
def _chat_body(model: str, text: str) -> Mapping[str, JsonValue]:
|
||||
return {"model": model, "messages": [{"role": "user", "content": text}]}
|
||||
|
||||
|
||||
def _responses_body(model: str, text: str) -> Mapping[str, JsonValue]:
|
||||
return {"model": model, "input": text}
|
||||
|
||||
|
||||
def _messages_body(model: str, text: str) -> Mapping[str, JsonValue]:
|
||||
return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": text}]}
|
||||
|
||||
|
||||
FAILURE_ROUTES: Final = pytest.mark.parametrize(
|
||||
("route", "body"),
|
||||
(
|
||||
("/v1/chat/completions", _chat_body),
|
||||
("/v1/responses", _responses_body),
|
||||
("/v1/messages", _messages_body),
|
||||
),
|
||||
ids=("chat", "responses", "messages"),
|
||||
)
|
||||
|
||||
|
||||
def _provider_500_failure_row(
|
||||
gateway: Gateway,
|
||||
route: str,
|
||||
body: Callable[[str, str], Mapping[str, JsonValue]],
|
||||
headers: Mapping[str, str] | None = None,
|
||||
) -> tuple[str, dict[str, JsonValue]]:
|
||||
def provider(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
return Reply(
|
||||
status=500,
|
||||
body=b'{"error":{"message":"synthetic provider outage","type":"server_error","code":"500"}}',
|
||||
)
|
||||
|
||||
with wire_server(provider) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0)
|
||||
key: Final = scenario.key(models=[model])
|
||||
failed: Final = gateway.request(
|
||||
"POST", route, body(model, f"failed dispatch group {uuid.uuid4().hex}"), key=key, headers=headers
|
||||
)
|
||||
assert failed.status_code == 500, failed.text
|
||||
assert wire.drain()
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT status, model_group, request_tags FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
|
||||
(failed.headers["x-litellm-call-id"],),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert rows[0]["status"] == "failure", rows
|
||||
return model, rows[0]
|
||||
|
||||
|
||||
@FAILURE_ROUTES
|
||||
def test_a_provider_500_failure_row_names_the_model_group_the_caller_asked_for(
|
||||
gateway: Gateway, route: str, body: Callable[[str, str], Mapping[str, JsonValue]]
|
||||
) -> None:
|
||||
model, row = _provider_500_failure_row(gateway, route, body)
|
||||
|
||||
assert row["model_group"] == model, row
|
||||
|
||||
|
||||
@FAILURE_ROUTES
|
||||
def test_a_provider_500_failure_row_keeps_the_tags_the_caller_sent(
|
||||
gateway: Gateway, route: str, body: Callable[[str, str], Mapping[str, JsonValue]]
|
||||
) -> None:
|
||||
tag: Final = f"failed-dispatch-{uuid.uuid4().hex}"
|
||||
|
||||
_, row = _provider_500_failure_row(gateway, route, body, {"x-litellm-tags": tag})
|
||||
|
||||
assert tag in list_value(row["request_tags"]), row
|
||||
|
|
|
|||
|
|
@ -0,0 +1,583 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, eventually, gateway_from_environment
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.forward_proxy import tunnelling_forward_proxy
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.tls import server_context, write_self_signed_cert
|
||||
from integration._support.vertex import service_account_json
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
_PROJECT: Final = "caller-project"
|
||||
_CALLER_GOOGLE_TOKEN: Final = "caller-google-token"
|
||||
_GEMINI: Final = "gemini-3.5-flash-lite"
|
||||
_CLAUDE_ROUTE_MODEL: Final = "claude-haiku-4-5"
|
||||
_CLAUDE_SERVED_MODEL: Final = "claude-haiku-4-5-20251001"
|
||||
_OPENAI_MODEL: Final = "gpt-5.6"
|
||||
_PROMPT_TOKENS: Final = 1200
|
||||
_COMPLETION_TOKENS: Final = 300
|
||||
_CUSTOM_INPUT_RATE: Final = 1e-05
|
||||
_CUSTOM_OUTPUT_RATE: Final = 2e-05
|
||||
_CUSTOM_PRICED_GEMINI_ID: Final = "audit-gemini-custom-priced"
|
||||
_HOSTS: Final = (
|
||||
"aiplatform.googleapis.com",
|
||||
"aiplatform.us.rep.googleapis.com",
|
||||
"us-east5-aiplatform.googleapis.com",
|
||||
"generativelanguage.googleapis.com",
|
||||
"api.openai.com",
|
||||
)
|
||||
_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json")
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_CUT_MARKER: Final = "cut this stream after the usage frame"
|
||||
_GEMINI_USAGE: Final = {
|
||||
"promptTokenCount": _PROMPT_TOKENS,
|
||||
"candidatesTokenCount": _COMPLETION_TOKENS,
|
||||
"totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS,
|
||||
}
|
||||
_DEPLOYMENT_LOCATIONS: Final = MappingProxyType({"audit-vertex-global": "global", "audit-vertex-us": "us"})
|
||||
_GEMINI_ACTIONS: Final = ("generateContent", "streamGenerateContent?alt=sse")
|
||||
_CONFIGURED_GATEWAY_PATH: Final = "/google-egress"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rig:
|
||||
proxy: Gateway
|
||||
upstream: Wire
|
||||
|
||||
|
||||
def _gemini_reply(request: Request) -> Reply:
|
||||
body: Final = json.dumps(
|
||||
{
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": _GEMINI_USAGE,
|
||||
"modelVersion": _GEMINI,
|
||||
"responseId": uuid.uuid4().hex,
|
||||
}
|
||||
)
|
||||
if ":streamGenerateContent" in request.target:
|
||||
return Reply(chunks=(f"data: {body}\r\n\r\n".encode(),), content_type="text/event-stream")
|
||||
return Reply(body=body.encode())
|
||||
|
||||
|
||||
def _sse(event: str, payload: Mapping[str, JsonValue]) -> bytes:
|
||||
return f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode()
|
||||
|
||||
|
||||
def _claude_reply(request: Request) -> Reply:
|
||||
message_id: Final = f"msg_vrtx_{uuid.uuid4().hex}"
|
||||
if ":streamRawPredict" not in request.target:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"model": _CLAUDE_SERVED_MODEL,
|
||||
"id": message_id,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "text", "text": "scripted"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": _PROMPT_TOKENS, "output_tokens": _COMPLETION_TOKENS},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=(
|
||||
_sse(
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"model": _CLAUDE_SERVED_MODEL,
|
||||
"id": message_id,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": _PROMPT_TOKENS, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
),
|
||||
_sse(
|
||||
"content_block_start",
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
),
|
||||
_sse(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "scripted"}},
|
||||
),
|
||||
_sse("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
_sse(
|
||||
"message_delta",
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
|
||||
"usage": {"output_tokens": _COMPLETION_TOKENS},
|
||||
},
|
||||
),
|
||||
_sse("message_stop", {"type": "message_stop"}),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _openai_reply(request: Request) -> Reply:
|
||||
identity: Final = f"chatcmpl-{uuid.uuid4().hex}"
|
||||
chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": _OPENAI_MODEL}
|
||||
frames: Final = (
|
||||
{**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}}]},
|
||||
{**chunk, "choices": [{"index": 0, "delta": {"content": "scripted"}}]},
|
||||
{**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{
|
||||
**chunk,
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": _PROMPT_TOKENS,
|
||||
"completion_tokens": _COMPLETION_TOKENS,
|
||||
"total_tokens": _PROMPT_TOKENS + _COMPLETION_TOKENS,
|
||||
},
|
||||
},
|
||||
)
|
||||
encoded: Final = tuple(f"data: {json.dumps(frame)}\n\n".encode() for frame in frames)
|
||||
if _CUT_MARKER.encode() in request.body:
|
||||
return Reply(chunks=(*encoded, b"data: [DONE]\n\n"), content_type="text/event-stream", abort_after=len(encoded))
|
||||
return Reply(chunks=(*encoded, b"data: [DONE]\n\n"), content_type="text/event-stream")
|
||||
|
||||
|
||||
def _upstream(request: Request) -> Reply:
|
||||
if request.target.startswith("/v1/chat/completions"):
|
||||
return _openai_reply(request)
|
||||
if "/publishers/anthropic/models/" in request.target:
|
||||
assert f"/v1/projects/{_PROJECT}/locations/" in request.target, request.target
|
||||
return _claude_reply(request)
|
||||
return _gemini_reply(request)
|
||||
|
||||
|
||||
def _port(url: str) -> int:
|
||||
port: Final = urlsplit(url).port
|
||||
assert port is not None, url
|
||||
return port
|
||||
|
||||
|
||||
def _cost_row(name: str) -> dict[str, JsonValue]:
|
||||
return _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[name])
|
||||
|
||||
|
||||
def _list_price(name: str) -> float:
|
||||
row: Final = _cost_row(name)
|
||||
input_rate: Final = row["input_cost_per_token"]
|
||||
output_rate: Final = row["output_cost_per_token"]
|
||||
assert isinstance(input_rate, float) and isinstance(output_rate, float), row
|
||||
return _PROMPT_TOKENS * input_rate + _COMPLETION_TOKENS * output_rate
|
||||
|
||||
|
||||
def _uplift(name: str) -> float:
|
||||
multiplier: Final = _cost_row(name).get("regional_endpoint_uplift_multiplier")
|
||||
assert isinstance(multiplier, int | float) and multiplier > 1, name
|
||||
return float(multiplier)
|
||||
|
||||
|
||||
def _config(directory: Path, token_url: str) -> Path:
|
||||
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
existing: Final = base.get("model_list") or []
|
||||
assert isinstance(existing, list), base
|
||||
deployments: Final = [
|
||||
{
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": f"vertex_ai/{_GEMINI}",
|
||||
"vertex_project": _PROJECT,
|
||||
"vertex_location": location,
|
||||
"vertex_credentials": service_account_json(_PROJECT, token_url),
|
||||
},
|
||||
}
|
||||
for name, location in _DEPLOYMENT_LOCATIONS.items()
|
||||
]
|
||||
custom_priced_gemini: Final = {
|
||||
"model_name": _CUSTOM_PRICED_GEMINI_ID,
|
||||
"litellm_params": {
|
||||
"model": f"gemini/{_GEMINI}",
|
||||
"api_key": "scripted-gemini-key",
|
||||
"input_cost_per_token": _CUSTOM_INPUT_RATE,
|
||||
"output_cost_per_token": _CUSTOM_OUTPUT_RATE,
|
||||
},
|
||||
"model_info": {"id": _CUSTOM_PRICED_GEMINI_ID},
|
||||
}
|
||||
path: Final = directory / "config.yaml"
|
||||
path.write_text(yaml.safe_dump({**base, "model_list": [*existing, *deployments, custom_priced_gemini]}))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("passthrough-stream-pricing")
|
||||
certificate: Final = write_self_signed_cert(directory, _HOSTS)
|
||||
with (
|
||||
wire_server(_upstream, tls=server_context(*certificate)) as upstream,
|
||||
tunnelling_forward_proxy(_port(upstream.url)) as tunnel,
|
||||
gateway_from_environment() as base,
|
||||
owned_proxy(
|
||||
base,
|
||||
directory,
|
||||
{
|
||||
"HTTPS_PROXY": tunnel.url,
|
||||
"NO_PROXY": "127.0.0.1,localhost",
|
||||
"AIOHTTP_TRUST_ENV": "True",
|
||||
"SSL_VERIFY": "False",
|
||||
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
|
||||
"OPENAI_API_KEY": "sk-scripted-openai",
|
||||
"GEMINI_API_KEY": "scripted-gemini-key",
|
||||
},
|
||||
config=_config(directory, base.upstream_url.rstrip("/")),
|
||||
workers=2,
|
||||
) as proxy,
|
||||
):
|
||||
yield _Rig(proxy, upstream)
|
||||
|
||||
|
||||
def _gateway_config(directory: Path, upstream_url: str) -> Path:
|
||||
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
general_settings: Final = base.get("general_settings")
|
||||
assert isinstance(general_settings, dict), base
|
||||
configured_endpoint: Final = {
|
||||
"path": _CONFIGURED_GATEWAY_PATH,
|
||||
"target": f"{upstream_url}/egress",
|
||||
"include_subpath": True,
|
||||
"auth": True,
|
||||
}
|
||||
path: Final = directory / "config.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{**base, "general_settings": {**general_settings, "pass_through_endpoints": [configured_endpoint]}}
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def gateway_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("passthrough-google-gateway")
|
||||
with (
|
||||
wire_server(_gemini_reply) as upstream,
|
||||
gateway_from_environment() as base,
|
||||
owned_proxy(
|
||||
base,
|
||||
directory,
|
||||
{
|
||||
"GEMINI_API_BASE": f"{upstream.url}/google",
|
||||
"GEMINI_API_KEY": "scripted-gemini-key",
|
||||
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
|
||||
},
|
||||
config=_gateway_config(directory, upstream.url),
|
||||
workers=2,
|
||||
) as proxy,
|
||||
):
|
||||
yield _Rig(proxy, upstream)
|
||||
|
||||
|
||||
def _spend_row(key: str) -> dict[str, JsonValue]:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT spend, custom_llm_provider FROM "LiteLLM_SpendLogs" WHERE api_key=%s',
|
||||
(sha256(key.encode()).hexdigest(),),
|
||||
),
|
||||
lambda found: len(found) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _spend_for_key(key: str) -> float:
|
||||
spend: Final = _spend_row(key)["spend"]
|
||||
assert isinstance(spend, float), key
|
||||
return spend
|
||||
|
||||
|
||||
def _provider_for_key(key: str) -> JsonValue:
|
||||
return _spend_row(key)["custom_llm_provider"]
|
||||
|
||||
|
||||
def _post_and_read(rig: _Rig, path: str, body: Mapping[str, JsonValue], headers: Mapping[str, str]) -> None:
|
||||
with rig.proxy.client.stream("POST", path, json=body, headers=dict(headers)) as response:
|
||||
payload: Final = response.read()
|
||||
assert response.status_code == 200, payload
|
||||
|
||||
|
||||
def _claude_path(location: str, action: str) -> str:
|
||||
return (
|
||||
f"/vertex_ai/v1/projects/{_PROJECT}/locations/{location}/publishers/anthropic/models/"
|
||||
f"{_CLAUDE_ROUTE_MODEL}:{action}"
|
||||
)
|
||||
|
||||
|
||||
def test_claude_on_vertex_passthrough_spend_carries_the_regional_uplift_whether_or_not_the_call_streams(
|
||||
rig: _Rig,
|
||||
) -> None:
|
||||
body: Final[Mapping[str, JsonValue]] = {
|
||||
"anthropic_version": "vertex-2023-10-16",
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
cells: Final = (
|
||||
("global", "rawPredict"),
|
||||
("global", "streamRawPredict"),
|
||||
("us-east5", "rawPredict"),
|
||||
("us-east5", "streamRawPredict"),
|
||||
)
|
||||
with rig.proxy.scenario() as scenario:
|
||||
keys: Final = {cell: scenario.key() for cell in cells}
|
||||
for (location, action), key in keys.items():
|
||||
_post_and_read(
|
||||
rig,
|
||||
_claude_path(location, action),
|
||||
{**body, "stream": action == "streamRawPredict"},
|
||||
{"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"},
|
||||
)
|
||||
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
|
||||
standard: Final = _list_price(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}")
|
||||
regional: Final = standard * _uplift(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}")
|
||||
assert spend == pytest.approx(
|
||||
{
|
||||
("global", "rawPredict"): standard,
|
||||
("global", "streamRawPredict"): standard,
|
||||
("us-east5", "rawPredict"): regional,
|
||||
("us-east5", "streamRawPredict"): regional,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_v1beta_generate_content_on_a_vertex_deployment_bills_the_deployment_location_in_both_modes(
|
||||
rig: _Rig,
|
||||
) -> None:
|
||||
body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
|
||||
actions: Final = ("generateContent", "streamGenerateContent?alt=sse")
|
||||
cells: Final = tuple((deployment, action) for deployment in _DEPLOYMENT_LOCATIONS for action in actions)
|
||||
with rig.proxy.scenario() as scenario:
|
||||
keys: Final = {cell: scenario.key() for cell in cells}
|
||||
for (deployment, action), key in keys.items():
|
||||
_post_and_read(rig, f"/v1beta/models/{deployment}:{action}", body, {"x-goog-api-key": key})
|
||||
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
|
||||
standard: Final = _list_price(f"vertex_ai/{_GEMINI}")
|
||||
regional: Final = standard * _uplift(f"vertex_ai/{_GEMINI}")
|
||||
assert spend == pytest.approx(
|
||||
{
|
||||
("audit-vertex-global", "generateContent"): standard,
|
||||
("audit-vertex-global", "streamGenerateContent?alt=sse"): standard,
|
||||
("audit-vertex-us", "generateContent"): regional,
|
||||
("audit-vertex-us", "streamGenerateContent?alt=sse"): regional,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_gemini_passthrough_bills_a_named_custom_priced_deployment_only_for_keys_that_allow_it_in_both_modes(
|
||||
rig: _Rig,
|
||||
) -> None:
|
||||
body: Final[Mapping[str, JsonValue]] = {
|
||||
"contents": [{"role": "user", "parts": [{"text": "hi"}]}],
|
||||
"metadata": {
|
||||
"model_info": {
|
||||
"id": _CUSTOM_PRICED_GEMINI_ID,
|
||||
"input_cost_per_token": _CUSTOM_INPUT_RATE,
|
||||
"output_cost_per_token": _CUSTOM_OUTPUT_RATE,
|
||||
}
|
||||
},
|
||||
}
|
||||
actions: Final = ("generateContent", "streamGenerateContent?alt=sse")
|
||||
with rig.proxy.scenario() as scenario:
|
||||
allowed: Final = {action: scenario.key(metadata={"allow_client_pricing_override": True}) for action in actions}
|
||||
plain: Final = {action: scenario.key() for action in actions}
|
||||
for action in actions:
|
||||
for key in (allowed[action], plain[action]):
|
||||
_post_and_read(rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key})
|
||||
spend: Final = {
|
||||
**{("allowed", action): _spend_for_key(key) for action, key in allowed.items()},
|
||||
**{("plain", action): _spend_for_key(key) for action, key in plain.items()},
|
||||
}
|
||||
custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE
|
||||
list_price: Final = _list_price(f"gemini/{_GEMINI}")
|
||||
assert spend == pytest.approx(
|
||||
{
|
||||
("allowed", "generateContent"): custom_price,
|
||||
("allowed", "streamGenerateContent?alt=sse"): custom_price,
|
||||
("plain", "generateContent"): list_price,
|
||||
("plain", "streamGenerateContent?alt=sse"): list_price,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_openai_passthrough_stream_cut_off_after_its_usage_bills_what_the_finished_stream_bills(rig: _Rig) -> None:
|
||||
def body(prompt: str) -> Mapping[str, JsonValue]:
|
||||
return {
|
||||
"model": _OPENAI_MODEL,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
|
||||
with rig.proxy.scenario() as scenario:
|
||||
finished_key: Final = scenario.key()
|
||||
cut_key: Final = scenario.key()
|
||||
_post_and_read(rig, "/openai/v1/chat/completions", body("hi"), {"Authorization": f"Bearer {finished_key}"})
|
||||
with rig.proxy.client.stream(
|
||||
"POST",
|
||||
"/openai/v1/chat/completions",
|
||||
json=dict(body(_CUT_MARKER)),
|
||||
headers={"Authorization": f"Bearer {cut_key}"},
|
||||
) as response:
|
||||
assert response.status_code == 200
|
||||
with pytest.raises(httpx.RemoteProtocolError):
|
||||
response.read()
|
||||
spend: Final = {"finished": _spend_for_key(finished_key), "cut off": _spend_for_key(cut_key)}
|
||||
list_price: Final = _list_price(_OPENAI_MODEL)
|
||||
assert spend == pytest.approx({"finished": list_price, "cut off": list_price})
|
||||
|
||||
|
||||
def _caller_headers(route: str, key: str) -> Mapping[str, str]:
|
||||
if route == "vertex_ai":
|
||||
return {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"}
|
||||
return {"x-goog-api-key": key}
|
||||
|
||||
|
||||
def test_a_body_base_model_reprices_passthrough_spend_only_for_keys_that_allow_client_pricing(rig: _Rig) -> None:
|
||||
routes: Final = MappingProxyType(
|
||||
{
|
||||
"vertex_ai": (
|
||||
f"/vertex_ai/v1/projects/{_PROJECT}/locations/global/publishers/google/models/{_GEMINI}",
|
||||
"gemini-2.5-flash-lite",
|
||||
),
|
||||
"gemini": (f"/gemini/v1beta/models/{_GEMINI}", "gemini/gemini-2.5-flash-lite"),
|
||||
}
|
||||
)
|
||||
cells: Final = tuple(
|
||||
(route, action, grant) for route in routes for action in _GEMINI_ACTIONS for grant in ("allowed", "plain")
|
||||
)
|
||||
markers: Final = {cell: uuid.uuid4().hex for cell in cells}
|
||||
with rig.proxy.scenario() as scenario:
|
||||
keys: Final = {
|
||||
cell: scenario.key(metadata={"allow_client_pricing_override": True})
|
||||
if cell[2] == "allowed"
|
||||
else scenario.key()
|
||||
for cell in cells
|
||||
}
|
||||
for (route, action, grant), key in keys.items():
|
||||
path, base_model = routes[route]
|
||||
_post_and_read(
|
||||
rig,
|
||||
f"{path}:{action}",
|
||||
{
|
||||
"contents": [{"role": "user", "parts": [{"text": markers[(route, action, grant)]}]}],
|
||||
"base_model": base_model,
|
||||
},
|
||||
_caller_headers(route, key),
|
||||
)
|
||||
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
|
||||
forwarded: Final = rig.upstream.drain()
|
||||
marked_bodies: Final = {
|
||||
cell: tuple(request.body for request in forwarded if marker.encode() in request.body)
|
||||
for cell, marker in markers.items()
|
||||
}
|
||||
assert all(len(bodies) == 1 and b"base_model" not in bodies[0] for bodies in marked_bodies.values()), marked_bodies
|
||||
expected_price: Final = {
|
||||
"vertex_ai": {"allowed": _list_price("gemini-2.5-flash-lite"), "plain": _list_price(f"vertex_ai/{_GEMINI}")},
|
||||
"gemini": {"allowed": _list_price("gemini/gemini-2.5-flash-lite"), "plain": _list_price(f"gemini/{_GEMINI}")},
|
||||
}
|
||||
assert spend == pytest.approx({cell: expected_price[cell[0]][cell[2]] for cell in cells})
|
||||
|
||||
|
||||
def test_a_non_string_body_base_model_bills_the_list_price_instead_of_nothing(rig: _Rig) -> None:
|
||||
routes: Final = MappingProxyType(
|
||||
{
|
||||
"vertex_ai": f"/vertex_ai/v1/projects/{_PROJECT}/locations/global/publishers/google/models/{_GEMINI}",
|
||||
"gemini": f"/gemini/v1beta/models/{_GEMINI}",
|
||||
"v1beta": "/v1beta/models/audit-vertex-global",
|
||||
}
|
||||
)
|
||||
cells: Final = tuple((route, action) for route in routes for action in _GEMINI_ACTIONS)
|
||||
with rig.proxy.scenario() as scenario:
|
||||
keys: Final = {
|
||||
cell: scenario.key()
|
||||
if cell[0] == "v1beta"
|
||||
else scenario.key(metadata={"allow_client_pricing_override": True})
|
||||
for cell in cells
|
||||
}
|
||||
for (route, action), key in keys.items():
|
||||
_post_and_read(
|
||||
rig,
|
||||
f"{routes[route]}:{action}",
|
||||
{"contents": [{"role": "user", "parts": [{"text": uuid.uuid4().hex}]}], "base_model": 123},
|
||||
_caller_headers(route, key),
|
||||
)
|
||||
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
|
||||
list_price: Final = {
|
||||
"vertex_ai": _list_price(f"vertex_ai/{_GEMINI}"),
|
||||
"gemini": _list_price(f"gemini/{_GEMINI}"),
|
||||
"v1beta": _list_price(f"vertex_ai/{_GEMINI}"),
|
||||
}
|
||||
assert spend == pytest.approx({cell: list_price[cell[0]] for cell in cells})
|
||||
|
||||
|
||||
def test_gemini_passthrough_through_a_gemini_api_base_gateway_logs_and_bills_as_gemini_in_both_modes(
|
||||
gateway_rig: _Rig,
|
||||
) -> None:
|
||||
body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
|
||||
with gateway_rig.proxy.scenario() as scenario:
|
||||
keys: Final = {action: scenario.key() for action in _GEMINI_ACTIONS}
|
||||
for action, key in keys.items():
|
||||
_post_and_read(gateway_rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key})
|
||||
providers: Final = {action: _provider_for_key(key) for action, key in keys.items()}
|
||||
spend: Final = {action: _spend_for_key(key) for action, key in keys.items()}
|
||||
forwarded: Final = tuple(
|
||||
path
|
||||
for path in (urlsplit(request.target).path for request in gateway_rig.upstream.drain())
|
||||
if path.startswith("/google/")
|
||||
)
|
||||
assert sorted(forwarded) == sorted(
|
||||
f"/google/v1beta/models/{_GEMINI}:{action.split('?')[0]}" for action in _GEMINI_ACTIONS
|
||||
)
|
||||
assert providers == {action: "gemini" for action in _GEMINI_ACTIONS}
|
||||
assert spend == pytest.approx({action: _list_price(f"gemini/{_GEMINI}") for action in _GEMINI_ACTIONS})
|
||||
|
||||
|
||||
def test_a_configured_passthrough_to_a_non_google_gateway_bills_as_vertex_whatever_the_path(gateway_rig: _Rig) -> None:
|
||||
body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
|
||||
routes: Final = MappingProxyType(
|
||||
{
|
||||
"vertex resource": f"/v1/projects/{_PROJECT}/locations/us/publishers/google/models/{_GEMINI}",
|
||||
"short gateway path": f"/vx/models/{_GEMINI}",
|
||||
}
|
||||
)
|
||||
cells: Final = tuple((route, action) for route in routes for action in _GEMINI_ACTIONS)
|
||||
with gateway_rig.proxy.scenario() as scenario:
|
||||
keys: Final = {cell: scenario.key(allowed_passthrough_routes=[_CONFIGURED_GATEWAY_PATH]) for cell in cells}
|
||||
for (route, action), key in keys.items():
|
||||
_post_and_read(
|
||||
gateway_rig,
|
||||
f"{_CONFIGURED_GATEWAY_PATH}{routes[route]}:{action}",
|
||||
body,
|
||||
{"Authorization": f"Bearer {key}"},
|
||||
)
|
||||
providers: Final = {cell: _provider_for_key(key) for cell, key in keys.items()}
|
||||
spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
|
||||
short_path_spend: Final = {action: spend[("short gateway path", action)] for action in _GEMINI_ACTIONS}
|
||||
assert providers == {cell: "vertex_ai" for cell in cells}
|
||||
assert {action: spend[("vertex resource", action)] for action in _GEMINI_ACTIONS} == pytest.approx(
|
||||
{action: _list_price(f"vertex_ai/{_GEMINI}") * _uplift(f"vertex_ai/{_GEMINI}") for action in _GEMINI_ACTIONS}
|
||||
)
|
||||
assert min(short_path_spend.values()) > 0
|
||||
assert short_path_spend == pytest.approx(
|
||||
{action: short_path_spend[_GEMINI_ACTIONS[0]] for action in _GEMINI_ACTIONS}
|
||||
)
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
import itertools
|
||||
import json
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, eventually, gateway_from_environment
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.tls import server_context, write_self_signed_cert
|
||||
from integration._support.vertex import service_account_json
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
_MODEL: Final = "gemini-3.5-flash-lite"
|
||||
_PROJECT: Final = "caller-project"
|
||||
_CALLER_GOOGLE_TOKEN: Final = "caller-google-token"
|
||||
_GLOBAL_HOST: Final = "aiplatform.googleapis.com"
|
||||
_US_HOST: Final = "aiplatform.us.rep.googleapis.com"
|
||||
_GENERATE: Final = "generateContent"
|
||||
_STREAM: Final = "streamGenerateContent?alt=sse"
|
||||
_LOCATIONS: Final = ("global", "us")
|
||||
_ACTIONS: Final = (_GENERATE, _STREAM)
|
||||
_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json")
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_BODY: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
|
||||
_CUT_PROMPT: Final = "end this stream early"
|
||||
_CUT_BODY: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": _CUT_PROMPT}]}]}
|
||||
_PROMPT_TOKENS: Final = 1200
|
||||
_COMPLETION_TOKENS: Final = 300
|
||||
_USAGE: Final = {
|
||||
"promptTokenCount": _PROMPT_TOKENS,
|
||||
"candidatesTokenCount": _COMPLETION_TOKENS,
|
||||
"totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS,
|
||||
}
|
||||
_REPLY: Final = json.dumps(
|
||||
{
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": _USAGE,
|
||||
"modelVersion": _MODEL,
|
||||
"responseId": "scripted-response",
|
||||
}
|
||||
)
|
||||
_UNFINISHED_FRAME: Final = json.dumps(
|
||||
{
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}}],
|
||||
"usageMetadata": _USAGE,
|
||||
"modelVersion": _MODEL,
|
||||
"responseId": "scripted-cut-response",
|
||||
}
|
||||
)
|
||||
_CUSTOM_DEPLOYMENT: Final = "vertex-custom-priced"
|
||||
_CUSTOM_INPUT_RATE: Final = 1e-06
|
||||
_CUSTOM_OUTPUT_RATE: Final = 2e-06
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rig:
|
||||
proxy: Gateway
|
||||
tunnel: ForwardProxy
|
||||
|
||||
|
||||
def _vertex(request: Request) -> Reply:
|
||||
assert f"/v1/projects/{_PROJECT}/locations/" in request.target, request.target
|
||||
if _CUT_PROMPT.encode() in request.body:
|
||||
frame: Final = f"data: {_UNFINISHED_FRAME}\r\n\r\n".encode()
|
||||
return Reply(chunks=(frame, frame), content_type="text/event-stream", abort_after=1)
|
||||
if ":streamGenerateContent" in request.target:
|
||||
return Reply(chunks=(f"data: {_REPLY}\r\n\r\n".encode(),), content_type="text/event-stream")
|
||||
return Reply(body=_REPLY.encode())
|
||||
|
||||
|
||||
def _port(url: str) -> int:
|
||||
port: Final = urlsplit(url).port
|
||||
assert port is not None, url
|
||||
return port
|
||||
|
||||
|
||||
def _uplift() -> float:
|
||||
row: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[f"vertex_ai/{_MODEL}"])
|
||||
multiplier: Final = row.get("regional_endpoint_uplift_multiplier")
|
||||
assert isinstance(multiplier, int | float) and multiplier > 1, row
|
||||
return float(multiplier)
|
||||
|
||||
|
||||
def _config_with_custom_priced_deployment(directory: Path, token_url: str) -> Path:
|
||||
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
existing: Final = base.get("model_list") or []
|
||||
assert isinstance(existing, list), base
|
||||
deployment: Final = {
|
||||
"model_name": _CUSTOM_DEPLOYMENT,
|
||||
"litellm_params": {
|
||||
"model": f"vertex_ai/{_MODEL}",
|
||||
"vertex_project": _PROJECT,
|
||||
"vertex_location": "us",
|
||||
"vertex_credentials": service_account_json(_PROJECT, token_url),
|
||||
"use_in_pass_through": True,
|
||||
"input_cost_per_token": _CUSTOM_INPUT_RATE,
|
||||
"output_cost_per_token": _CUSTOM_OUTPUT_RATE,
|
||||
},
|
||||
}
|
||||
path: Final = directory / "config.yaml"
|
||||
path.write_text(yaml.safe_dump({**base, "model_list": [*existing, deployment]}))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("vertex-passthrough-uplift")
|
||||
certificate: Final = write_self_signed_cert(directory, (_GLOBAL_HOST, _US_HOST))
|
||||
with (
|
||||
wire_server(_vertex, tls=server_context(*certificate)) as vertex,
|
||||
tunnelling_forward_proxy(_port(vertex.url)) as tunnel,
|
||||
gateway_from_environment() as base,
|
||||
owned_proxy(
|
||||
base,
|
||||
directory,
|
||||
{
|
||||
"HTTPS_PROXY": tunnel.url,
|
||||
"NO_PROXY": "127.0.0.1,localhost",
|
||||
"AIOHTTP_TRUST_ENV": "True",
|
||||
"SSL_VERIFY": "False",
|
||||
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
|
||||
},
|
||||
config=_config_with_custom_priced_deployment(directory, base.upstream_url.rstrip("/")),
|
||||
workers=2,
|
||||
) as proxy,
|
||||
):
|
||||
yield _Rig(proxy, tunnel)
|
||||
|
||||
|
||||
def _path(location: str, model: str, action: str) -> str:
|
||||
return f"/vertex_ai/v1/projects/{_PROJECT}/locations/{location}/publishers/google/models/{model}:{action}"
|
||||
|
||||
|
||||
def _headers(key: str) -> dict[str, str]:
|
||||
return {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"}
|
||||
|
||||
|
||||
def _spend(rig: _Rig, key: str, location: str, action: str, model: str = _MODEL) -> float:
|
||||
with rig.proxy.client.stream("POST", _path(location, model, action), json=_BODY, headers=_headers(key)) as response:
|
||||
body: Final = response.read()
|
||||
assert response.status_code == 200, body
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
return _logged_spend(call_id)
|
||||
|
||||
|
||||
def _logged_spend(call_id: str) -> float:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)),
|
||||
lambda found: len(found) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
spend: Final = rows[0]["spend"]
|
||||
assert isinstance(spend, float), rows
|
||||
return spend
|
||||
|
||||
|
||||
def test_vertex_passthrough_spend_carries_the_regional_uplift_whether_or_not_the_call_streams(rig: _Rig) -> None:
|
||||
with rig.proxy.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
spend: Final = {cell: _spend(rig, key, *cell) for cell in itertools.product(_LOCATIONS, _ACTIONS)}
|
||||
standard: Final = spend[("global", _GENERATE)]
|
||||
regional: Final = standard * _uplift()
|
||||
assert standard > 0, spend
|
||||
assert spend == pytest.approx(
|
||||
{
|
||||
("global", _GENERATE): standard,
|
||||
("global", _STREAM): standard,
|
||||
("us", _GENERATE): regional,
|
||||
("us", _STREAM): regional,
|
||||
}
|
||||
)
|
||||
assert set(rig.tunnel.targets()) == {f"{_GLOBAL_HOST}:443", f"{_US_HOST}:443"}
|
||||
|
||||
|
||||
def test_custom_priced_vertex_passthrough_bills_the_deployment_price_even_when_the_stream_is_cut_off(rig: _Rig) -> None:
|
||||
with rig.proxy.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
finished: Final = {action: _spend(rig, key, "us", action, _CUSTOM_DEPLOYMENT) for action in _ACTIONS}
|
||||
with rig.proxy.client.stream(
|
||||
"POST", _path("us", _CUSTOM_DEPLOYMENT, _STREAM), json=_CUT_BODY, headers=_headers(key)
|
||||
) as response:
|
||||
assert response.status_code == 200
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
with pytest.raises(httpx.RemoteProtocolError):
|
||||
response.read()
|
||||
cut_off: Final = _logged_spend(call_id)
|
||||
custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE
|
||||
assert {**finished, "cut off": cut_off} == pytest.approx(
|
||||
{_GENERATE: custom_price, _STREAM: custom_price, "cut off": custom_price}
|
||||
)
|
||||
|
|
@ -4,6 +4,8 @@ Test to verify the Google GenAI generate_content adapter functionality
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -286,3 +288,97 @@ def test_vertex_generate_content_routes_by_deployment_base_model(base_model, use
|
|||
)
|
||||
|
||||
assert (setup.generate_content_provider_config is not None) is uses_native_api
|
||||
|
||||
|
||||
_UPLIFT_PROBE: Final = "gemini-regional-uplift-probe"
|
||||
_UPLIFT_INPUT_RATE: Final = 1e-06
|
||||
_UPLIFT_OUTPUT_RATE: Final = 2e-06
|
||||
_UPLIFT_MULTIPLIER: Final = 1.1
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vertex_regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
rates: Final = {
|
||||
"input_cost_per_token": _UPLIFT_INPUT_RATE,
|
||||
"output_cost_per_token": _UPLIFT_OUTPUT_RATE,
|
||||
"mode": "chat",
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, _UPLIFT_PROBE, {**rates, "litellm_provider": "vertex_ai-language-models"})
|
||||
monkeypatch.setattr(litellm, "vertex_language_models", {*litellm.vertex_language_models, _UPLIFT_PROBE})
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"vertex_ai/{_UPLIFT_PROBE}",
|
||||
{
|
||||
**rates,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER,
|
||||
},
|
||||
)
|
||||
monkeypatch.delenv("VERTEXAI_LOCATION", raising=False)
|
||||
monkeypatch.delenv("VERTEX_LOCATION", raising=False)
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
async def _logged_vertex_generate_content_cost(location: str, *, stream: bool) -> float:
|
||||
import asyncio
|
||||
import datetime
|
||||
|
||||
from litellm.google_genai.main import GenerateContentHelper
|
||||
from litellm.google_genai.streaming_iterator import BaseGoogleGenAIGenerateContentStreamingIterator
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.google_genai.main import GenerateContentResponse
|
||||
|
||||
reply: Final = {
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "hi"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8},
|
||||
"modelVersion": _UPLIFT_PROBE,
|
||||
}
|
||||
logging_obj: Final = Logging(
|
||||
model=f"vertex_ai/{_UPLIFT_PROBE}",
|
||||
messages=[],
|
||||
stream=stream,
|
||||
call_type="agenerate_content_stream" if stream else "agenerate_content",
|
||||
start_time=datetime.datetime(2026, 1, 1),
|
||||
litellm_call_id="uplift-test",
|
||||
function_id="uplift-test",
|
||||
)
|
||||
setup: Final = GenerateContentHelper.setup_generate_content_call(
|
||||
model=f"vertex_ai/{_UPLIFT_PROBE}",
|
||||
contents=[{"role": "user", "parts": [{"text": "hi"}]}],
|
||||
litellm_logging_obj=logging_obj,
|
||||
vertex_location=location,
|
||||
stream=stream,
|
||||
)
|
||||
if stream:
|
||||
iterator: Final = BaseGoogleGenAIGenerateContentStreamingIterator(
|
||||
litellm_logging_obj=logging_obj,
|
||||
request_body={},
|
||||
model=setup.model,
|
||||
custom_llm_provider=setup.custom_llm_provider,
|
||||
)
|
||||
iterator.collected_chunks.append(f"data: {json.dumps(reply)}\n\n".encode())
|
||||
await iterator._handle_async_streaming_logging()
|
||||
await asyncio.gather(*(task for task in asyncio.all_tasks() if task is not asyncio.current_task()))
|
||||
else:
|
||||
logging_obj._process_hidden_params_and_response_cost(
|
||||
GenerateContentResponse(**reply),
|
||||
datetime.datetime(2026, 1, 1),
|
||||
datetime.datetime(2026, 1, 1),
|
||||
build_logging_payload=False,
|
||||
)
|
||||
return logging_obj.model_call_details["response_cost"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us", _UPLIFT_MULTIPLIER)])
|
||||
async def test_vertex_generate_content_spend_follows_the_deployment_location(
|
||||
vertex_regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float
|
||||
) -> None:
|
||||
expected: Final = (7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * multiplier
|
||||
|
||||
recorded: Final = await _logged_vertex_generate_content_cost(location, stream=stream)
|
||||
|
||||
assert recorded == pytest.approx(expected), (location, stream)
|
||||
|
|
|
|||
|
|
@ -6,9 +6,12 @@ Ensures backward compatibility after sparse kwargs extraction optimization.
|
|||
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai import OpenAI
|
||||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.constants import CONTROL_OPTIONS_KEY
|
||||
from litellm.litellm_core_utils.get_litellm_params import (
|
||||
_OPTIONAL_KWARGS_KEYS,
|
||||
|
|
@ -75,10 +78,39 @@ class TestGetBaseModelFromLitellmCallMetadata:
|
|||
)
|
||||
assert result == "gpt-4"
|
||||
|
||||
def test_returns_non_string_base_model_without_filtering(self):
|
||||
result = get_base_model_from_litellm_call_metadata({"model_info": {"base_model": 123}})
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[{"base_model": 123}, {"base_model": ["gpt-4"]}, {"base_model": {"model": "gpt-4"}}],
|
||||
)
|
||||
def test_ignores_a_non_string_base_model(self, model_info: object):
|
||||
assert get_base_model_from_litellm_call_metadata({"model_info": model_info}) is None
|
||||
|
||||
assert result == 123
|
||||
|
||||
@pytest.mark.parametrize("model_info", ["malformed", ["malformed"], 123])
|
||||
def test_a_call_with_a_non_object_model_info_fails_before_reaching_the_provider(model_info: object) -> None:
|
||||
sent: Final[list[httpx.Request]] = [] # mutable-ok: the transport records each outbound request
|
||||
|
||||
def record(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
return httpx.Response(500)
|
||||
|
||||
client: Final = OpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://api.openai.test/v1",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(record)),
|
||||
max_retries=0,
|
||||
)
|
||||
|
||||
with pytest.raises(AttributeError):
|
||||
litellm.completion(
|
||||
model="openai/gpt-5.4-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata={"model_info": model_info},
|
||||
client=client,
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
assert sent == []
|
||||
|
||||
|
||||
class TestGetLitellmParamsKwargsExtraction:
|
||||
|
|
|
|||
|
|
@ -2066,6 +2066,56 @@ async def test_arealtime_marks_litellm_params_async(monkeypatch):
|
|||
assert LitellmLogging.is_sync_litellm_request(captured["litellm_params"]) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_websocket_turn_reaches_a_custom_logger_through_the_async_hook_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from litellm.litellm_core_utils import thread_pool_executor
|
||||
from litellm.responses.main import base_llm_http_handler
|
||||
|
||||
class InlineExecutor:
|
||||
def submit(self, fn, *args, **kwargs):
|
||||
fn(*args, **kwargs)
|
||||
|
||||
sessions: list[LitellmLogging] = []
|
||||
|
||||
async def open_session(**kwargs):
|
||||
sessions.append(kwargs["logging_obj"])
|
||||
|
||||
async_logged = asyncio.Event()
|
||||
sync_call_ids: list[str] = []
|
||||
async_call_ids: list[str] = []
|
||||
|
||||
class CaptureLogger(CustomLogger):
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
sync_call_ids.append(kwargs["litellm_call_id"])
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
async_call_ids.append(kwargs["litellm_call_id"])
|
||||
async_logged.set()
|
||||
|
||||
logger = CaptureLogger()
|
||||
monkeypatch.setattr(litellm, "callbacks", [logger])
|
||||
monkeypatch.setattr(litellm, "success_callback", ["cache", logger])
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [logger])
|
||||
monkeypatch.setattr(thread_pool_executor, "executor", InlineExecutor())
|
||||
with patch.object( # test-quality-ok: the provider socket is the seam; how a turn of the session logs is under test
|
||||
base_llm_http_handler, "async_responses_websocket", AsyncMock(side_effect=open_session)
|
||||
):
|
||||
await litellm._aresponses_websocket(model="openai/gpt-4o", websocket=MagicMock(), api_key="sk-test")
|
||||
turn = sessions[0].copy_for_new_call("ws-turn-2", dt_object.now())
|
||||
|
||||
await turn.dispatch_success_handlers(
|
||||
litellm.ResponsesAPIResponse(
|
||||
id="resp_turn_2", created_at=0, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[]
|
||||
),
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
await asyncio.wait_for(async_logged.wait(), timeout=10)
|
||||
|
||||
assert (async_call_ids, sync_call_ids) == (["ws-turn-2"], [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_websocket_hands_back_the_provider_failure_without_a_success_log(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
57
tests/unit/llms/azure/realtime/test_http_transformation.py
Normal file
57
tests/unit/llms/azure/realtime/test_http_transformation.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
_DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview"
|
||||
_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai")
|
||||
|
||||
|
||||
class _AzureThatRecordsTheRequest:
|
||||
def __init__(self) -> None:
|
||||
self.requests: Final[list[httpx.Request]] = []
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
self.requests.append(request)
|
||||
return httpx.Response(200, json={"value": "ek_upstream"})
|
||||
|
||||
def client(self) -> AsyncHTTPHandler:
|
||||
return AsyncHTTPHandler(transport=httpx.MockTransport(self))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("api_base", _API_BASES)
|
||||
async def test_azure_client_secret_targets_the_ga_v1_surface_even_with_a_dated_api_version(api_base: str) -> None:
|
||||
azure: Final = _AzureThatRecordsTheRequest()
|
||||
await litellm.acreate_realtime_client_secret(
|
||||
model="azure/gpt-realtime-deployment",
|
||||
api_base=api_base,
|
||||
api_key="azure-key",
|
||||
api_version=_DATED_PREVIEW_API_VERSION,
|
||||
client=azure.client(),
|
||||
)
|
||||
(request,) = azure.requests
|
||||
assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/client_secrets"
|
||||
assert request.headers.get("api-key") == "azure-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("api_base", _API_BASES)
|
||||
async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_key_as_bearer(api_base: str) -> None:
|
||||
azure: Final = _AzureThatRecordsTheRequest()
|
||||
await litellm.arealtime_calls(
|
||||
openai_ephemeral_key="ek_upstream",
|
||||
sdp_body=b"v=0",
|
||||
model="azure/gpt-realtime-deployment",
|
||||
api_base=api_base,
|
||||
api_key="azure-key",
|
||||
api_version=_DATED_PREVIEW_API_VERSION,
|
||||
client=azure.client(),
|
||||
)
|
||||
(request,) = azure.requests
|
||||
assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls"
|
||||
assert request.headers.get("authorization") == "Bearer ek_upstream"
|
||||
assert "api-key" not in request.headers
|
||||
|
|
@ -1410,6 +1410,26 @@ def test_get_azure_ad_token_with_token_refresh(setup_mocks, monkeypatch):
|
|||
assert token == "mock-default-token"
|
||||
|
||||
|
||||
def test_get_azure_ad_token_with_token_refresh_reuses_provider_per_scope(setup_mocks):
|
||||
"""Repeated requests reuse one token provider per scope instead of building a
|
||||
new credential, and fetching a new token, on every call (#46419)"""
|
||||
setup_mocks["litellm"].enable_azure_ad_token_refresh = True
|
||||
default_scope = "https://cognitiveservices.azure.com/.default"
|
||||
other_scope = "api://other-resource/.default"
|
||||
|
||||
tokens = [
|
||||
get_azure_ad_token(GenericLiteLLMParams()),
|
||||
get_azure_ad_token(GenericLiteLLMParams()),
|
||||
get_azure_ad_token(GenericLiteLLMParams(azure_scope=other_scope)),
|
||||
]
|
||||
|
||||
assert tokens == ["mock-default-token"] * 3
|
||||
assert [c.kwargs for c in setup_mocks["token_provider"].call_args_list] == [
|
||||
{"azure_scope": default_scope},
|
||||
{"azure_scope": other_scope},
|
||||
]
|
||||
|
||||
|
||||
def test_get_azure_ad_token_with_token_refresh_error(setup_mocks):
|
||||
"""Test get_azure_ad_token with token refresh enabled but raising an error."""
|
||||
# Reset mocks to ensure clean state
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import logging
|
|||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, create_autospec, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -136,6 +136,77 @@ async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_
|
|||
assert metadata["original_key"] == "original_value"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("metadata_buckets", "expected_model_group"),
|
||||
[
|
||||
({"metadata": {}, "litellm_metadata": {"model_group": "ws-budget-sock"}}, "ws-budget-sock"),
|
||||
({"metadata": {"model_group": "caller-group"}, "litellm_metadata": {"model_group": "routed"}}, "caller-group"),
|
||||
({"metadata": {}, "litellm_metadata": {}}, ""),
|
||||
],
|
||||
)
|
||||
async def test_async_post_call_failure_hook_logs_the_routed_model_group_on_the_failure_row(
|
||||
metadata_buckets: dict[str, dict[str, str]], expected_model_group: str
|
||||
):
|
||||
request_data: Final = {
|
||||
"model": "ws-budget-sock",
|
||||
"input": "Reply with exactly the word two",
|
||||
"proxy_server_request": {"request_id": "test_request_id"},
|
||||
**metadata_buckets,
|
||||
}
|
||||
|
||||
spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
|
||||
await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=litellm.RateLimitError(message="crossed budget", llm_provider="", model="ws-budget-sock"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
|
||||
)
|
||||
|
||||
row: Final = get_logging_payload(
|
||||
kwargs=spend_writer.update_database.call_args.kwargs["kwargs"],
|
||||
response_obj={},
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
assert row["model_group"] == expected_model_group
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("metadata_buckets", "expected_tags"),
|
||||
[
|
||||
({"metadata": {}, "litellm_metadata": {"tags": ["ws-tag"]}}, ["ws-tag"]),
|
||||
({"metadata": {"tags": ["caller-tag"]}, "litellm_metadata": {"tags": ["routed-tag"]}}, ["caller-tag"]),
|
||||
({"metadata": {}, "litellm_metadata": {"tags": "not-a-list"}}, []),
|
||||
({"metadata": {}, "litellm_metadata": {"tags": ["ws-tag", 1, {"a": 1}]}}, []),
|
||||
],
|
||||
)
|
||||
async def test_async_post_call_failure_hook_logs_the_routed_tags_on_the_failure_row(
|
||||
metadata_buckets: dict[str, dict[str, object]], expected_tags: list[str]
|
||||
):
|
||||
request_data: Final = {
|
||||
"model": "ws-budget-sock",
|
||||
"input": "Reply with exactly the word two",
|
||||
"proxy_server_request": {"request_id": "test_request_id"},
|
||||
**metadata_buckets,
|
||||
}
|
||||
|
||||
spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
|
||||
await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=litellm.RateLimitError(message="crossed budget", llm_provider="", model="ws-budget-sock"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"),
|
||||
)
|
||||
|
||||
row: Final = get_logging_payload(
|
||||
kwargs=spend_writer.update_database.call_args.kwargs["kwargs"],
|
||||
response_obj={},
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
assert json.loads(row["request_tags"]) == expected_tags
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_hook_does_not_clobber_guardrail_info_in_metadata():
|
||||
logger = ProxyDBLogger()
|
||||
|
|
@ -1105,6 +1176,38 @@ async def test_track_cost_callback_skips_when_no_standard_logging_object():
|
|||
mock_proxy_logging.failed_tracking_alert.assert_not_called()
|
||||
|
||||
|
||||
class _FailedTrackingAlerts:
|
||||
def __init__(self) -> None:
|
||||
self.messages: tuple[str, ...] = ()
|
||||
|
||||
async def failed_tracking_alert(self, error_message: str, failing_model: str) -> None:
|
||||
self.messages = (*self.messages, error_message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("call_type", "alerted"), [("_aresponses_websocket", True), ("_arealtime", False)])
|
||||
async def test_track_cost_callback_alerts_on_a_websocket_turn_with_no_logging_object(call_type: str, alerted: bool):
|
||||
kwargs: Final = {
|
||||
"call_type": call_type,
|
||||
"model": "gpt-5-mini",
|
||||
"litellm_call_id": "turn-call-id",
|
||||
"litellm_params": {},
|
||||
"stream": False,
|
||||
}
|
||||
alerts: Final = _FailedTrackingAlerts()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.proxy_logging_obj", alerts):
|
||||
await ProxyDBLogger()._PROXY_track_cost_callback(
|
||||
kwargs=kwargs,
|
||||
completion_response=None,
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert any(f"call_type: {call_type}" in message for message in alerts.messages) is alerted
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_track_cost_callback_defers_in_progress_background_interaction(): # test-quality-ok: writing no spend row and raising no alert is the whole observable contract of the deferral path
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -11,6 +13,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.gemini_passthrough_logging_handler import (
|
||||
GeminiPassthroughLoggingHandler,
|
||||
)
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers
|
||||
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
|
||||
from litellm.proxy.pass_through_endpoints.success_handler import (
|
||||
PassThroughEndpointLogging,
|
||||
)
|
||||
|
|
@ -186,7 +190,6 @@ class TestGeminiPassthroughLoggingHandler:
|
|||
mock_completion_cost.assert_called_once()
|
||||
|
||||
# Verify logging object was updated
|
||||
assert mock_logging_obj.model_call_details["response_cost"] == 0.000045
|
||||
assert mock_logging_obj.model_call_details["model"] == "gemini-1.5-flash"
|
||||
assert mock_logging_obj.model_call_details["custom_llm_provider"] == "gemini"
|
||||
|
||||
|
|
@ -310,8 +313,6 @@ class TestGeminiPassthroughLoggingHandler:
|
|||
# Assert - The success handler returns None on success (following the pattern from other tests)
|
||||
assert result is None
|
||||
|
||||
# Verify that the logging object has the cost set (from Gemini handler)
|
||||
assert mock_logging_obj.model_call_details["response_cost"] == 0.000050
|
||||
assert mock_logging_obj.model_call_details["model"] == "gemini-2.0-flash"
|
||||
assert mock_logging_obj.model_call_details["custom_llm_provider"] == "gemini"
|
||||
|
||||
|
|
@ -479,3 +480,114 @@ def test_build_complete_streaming_response_assembles_gemini_sse_chunks():
|
|||
assert response.choices[0].message.content == "Hello there!"
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (10, 8, 18)
|
||||
|
||||
|
||||
_PROBE_MODEL: Final = "gemini-pricing-probe"
|
||||
_LIST_INPUT_RATE: Final = 1e-06
|
||||
_LIST_OUTPUT_RATE: Final = 4e-06
|
||||
_CUSTOM_RATES: Final = {"input_cost_per_token": 2e-09, "output_cost_per_token": 3e-09}
|
||||
_CUSTOM_PRICED_DEPLOYMENT: Final = "gemini-custom-priced-deployment"
|
||||
_PROBE_REPLY: Final = {
|
||||
"candidates": [{"content": {"parts": [{"text": "pong"}], "role": "model"}, "finishReason": "STOP", "index": 0}],
|
||||
"usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8},
|
||||
"modelVersion": _PROBE_MODEL,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def probe_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
rates: Final = {
|
||||
"input_cost_per_token": _LIST_INPUT_RATE,
|
||||
"output_cost_per_token": _LIST_OUTPUT_RATE,
|
||||
"litellm_provider": "gemini",
|
||||
"mode": "chat",
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, f"gemini/{_PROBE_MODEL}", rates)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**_CUSTOM_RATES, "litellm_provider": "gemini", "mode": "chat"}
|
||||
)
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def _cost_the_logger_records_for_gemini(*, stream: bool, model_info: dict[str, object] | None) -> float:
|
||||
method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent"
|
||||
url_route: Final = f"https://generativelanguage.googleapis.com/v1beta/models/{_PROBE_MODEL}:{method}"
|
||||
logging_obj: Final = LiteLLMLoggingObj(
|
||||
model="unknown",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
stream=stream,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime(2026, 1, 1),
|
||||
litellm_call_id="call-1",
|
||||
function_id="fn-1",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={"metadata": {"model_info": model_info}} if model_info is not None else {}, optional_params={}
|
||||
)
|
||||
if stream:
|
||||
asyncio.run(
|
||||
PassThroughStreamingHandler.route_streaming_logging_to_handler(
|
||||
litellm_logging_obj=logging_obj,
|
||||
passthrough_success_handler_obj=PassThroughEndpointLogging(),
|
||||
url_route=url_route,
|
||||
request_body={},
|
||||
endpoint_type=HttpPassThroughEndpointHelpers.get_endpoint_type(url_route),
|
||||
start_time=datetime(2026, 1, 1),
|
||||
raw_bytes=[f"data: {json.dumps(_PROBE_REPLY)}\n\n".encode()],
|
||||
end_time=datetime(2026, 1, 1),
|
||||
)
|
||||
)
|
||||
return logging_obj.model_call_details["response_cost"]
|
||||
response: Final = httpx.Response(200, json=_PROBE_REPLY)
|
||||
result: Final = GeminiPassthroughLoggingHandler.gemini_passthrough_handler(
|
||||
httpx_response=response,
|
||||
response_body=_PROBE_REPLY,
|
||||
logging_obj=logging_obj,
|
||||
url_route=url_route,
|
||||
result=response.text,
|
||||
start_time=datetime(2026, 1, 1),
|
||||
end_time=datetime(2026, 1, 1),
|
||||
cache_hit=False,
|
||||
request_body={},
|
||||
)
|
||||
logging_obj._process_hidden_params_and_response_cost(
|
||||
result["result"], datetime(2026, 1, 1), datetime(2026, 1, 1), build_logging_payload=False
|
||||
)
|
||||
return logging_obj.model_call_details["response_cost"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
@pytest.mark.parametrize(
|
||||
"model_info, input_rate, output_rate",
|
||||
[
|
||||
(None, _LIST_INPUT_RATE, _LIST_OUTPUT_RATE),
|
||||
(
|
||||
{"id": _CUSTOM_PRICED_DEPLOYMENT, **_CUSTOM_RATES},
|
||||
_CUSTOM_RATES["input_cost_per_token"],
|
||||
_CUSTOM_RATES["output_cost_per_token"],
|
||||
),
|
||||
],
|
||||
ids=["list-price", "custom-price"],
|
||||
)
|
||||
def test_gemini_passthrough_spend_matches_whether_or_not_the_call_streams(
|
||||
probe_rate_card: None,
|
||||
stream: bool,
|
||||
model_info: dict[str, object] | None,
|
||||
input_rate: float,
|
||||
output_rate: float,
|
||||
) -> None:
|
||||
recorded: Final = _cost_the_logger_records_for_gemini(stream=stream, model_info=model_info)
|
||||
|
||||
assert recorded == pytest.approx(7 * input_rate + 1 * output_rate), stream
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
@pytest.mark.parametrize("base_model", [123, [f"gemini/{_PROBE_MODEL}"], {"model": f"gemini/{_PROBE_MODEL}"}])
|
||||
def test_gemini_passthrough_bills_the_list_price_when_the_model_info_base_model_is_not_a_string(
|
||||
probe_rate_card: None, stream: bool, base_model: object
|
||||
) -> None:
|
||||
recorded: Final = _cost_the_logger_records_for_gemini(stream=stream, model_info={"base_model": base_model})
|
||||
|
||||
assert recorded == pytest.approx(7 * _LIST_INPUT_RATE + 1 * _LIST_OUTPUT_RATE), stream
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -6,11 +8,15 @@ import httpx
|
|||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import (
|
||||
VertexPassthroughLoggingHandler,
|
||||
)
|
||||
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
|
||||
from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
|
||||
from litellm.types.utils import EmbeddingResponse, ModelResponse
|
||||
|
||||
_PREDICT_ROUTE = "/v1/projects/p/locations/us-central1/publishers/google/models/text-embedding-004:predict"
|
||||
|
|
@ -97,6 +103,359 @@ def test_interactions_usage_object_is_read_into_prompt_and_completion_tokens():
|
|||
assert result["kwargs"]["custom_llm_provider"] == "vertex_ai"
|
||||
|
||||
|
||||
_UPLIFT_MODEL: Final = "gemini-regional-uplift-probe"
|
||||
_UPLIFT_INPUT_RATE: Final = 1e-06
|
||||
_UPLIFT_OUTPUT_RATE: Final = 4e-06
|
||||
_UPLIFT_MULTIPLIER: Final = 1.1
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
rates: Final = {
|
||||
"input_cost_per_token": _UPLIFT_INPUT_RATE,
|
||||
"output_cost_per_token": _UPLIFT_OUTPUT_RATE,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"mode": "chat",
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, _UPLIFT_MODEL, rates)
|
||||
monkeypatch.setattr(litellm, "vertex_language_models", {*litellm.vertex_language_models, _UPLIFT_MODEL})
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"vertex_ai/{_UPLIFT_MODEL}",
|
||||
{**rates, "regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER},
|
||||
)
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def _generate_content_payload() -> dict[str, object]:
|
||||
return {
|
||||
"candidates": [{"content": {"parts": [{"text": "pong"}], "role": "model"}, "finishReason": "STOP", "index": 0}],
|
||||
"usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8},
|
||||
"modelVersion": _UPLIFT_MODEL,
|
||||
}
|
||||
|
||||
|
||||
def _cost_the_logger_records_for(
|
||||
url_route: str,
|
||||
*,
|
||||
stream: bool,
|
||||
reply: dict[str, object],
|
||||
stream_events: list[dict[str, object]],
|
||||
model_info: dict[str, object] | None = None,
|
||||
route_provider: str | None = None,
|
||||
base_model: object = None,
|
||||
) -> float:
|
||||
logging_obj: Final = Logging(
|
||||
model="unknown",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
stream=stream,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime(2026, 1, 1),
|
||||
litellm_call_id="call-1",
|
||||
function_id="fn-1",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={"metadata": {"model_info": model_info}} if model_info is not None else {}, optional_params={}
|
||||
)
|
||||
if route_provider is not None:
|
||||
logging_obj.model_call_details["custom_llm_provider"] = route_provider
|
||||
if base_model is not None:
|
||||
logging_obj.model_call_details["litellm_params"] = {"base_model": base_model}
|
||||
if stream:
|
||||
asyncio.run(
|
||||
PassThroughStreamingHandler.route_streaming_logging_to_handler(
|
||||
litellm_logging_obj=logging_obj,
|
||||
passthrough_success_handler_obj=PassThroughEndpointLogging(),
|
||||
url_route=url_route,
|
||||
request_body={},
|
||||
endpoint_type=EndpointType.VERTEX_AI,
|
||||
start_time=datetime(2026, 1, 1),
|
||||
raw_bytes=[f"data: {json.dumps(event)}\n\n".encode() for event in stream_events],
|
||||
end_time=datetime(2026, 1, 1),
|
||||
)
|
||||
)
|
||||
return logging_obj.model_call_details["response_cost"]
|
||||
response: Final = httpx.Response(200, json=reply)
|
||||
result: Final = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
|
||||
httpx_response=response,
|
||||
logging_obj=logging_obj,
|
||||
url_route=url_route,
|
||||
result=response.text,
|
||||
start_time=datetime(2026, 1, 1),
|
||||
end_time=datetime(2026, 1, 1),
|
||||
cache_hit=False,
|
||||
request_body={},
|
||||
)
|
||||
logging_obj._process_hidden_params_and_response_cost(
|
||||
result["result"], datetime(2026, 1, 1), datetime(2026, 1, 1), build_logging_payload=False
|
||||
)
|
||||
return logging_obj.model_call_details["response_cost"]
|
||||
|
||||
|
||||
def _cost_the_logger_records(location: str, *, stream: bool, model_info: dict[str, object] | None = None) -> float:
|
||||
method: Final = "streamGenerateContent" if stream else "generateContent"
|
||||
url_route: Final = (
|
||||
f"https://aiplatform.googleapis.com/v1/projects/p/locations/{location}"
|
||||
f"/publishers/google/models/{_UPLIFT_MODEL}:{method}"
|
||||
)
|
||||
return _cost_the_logger_records_for(
|
||||
url_route,
|
||||
stream=stream,
|
||||
reply=_generate_content_payload(),
|
||||
stream_events=[_generate_content_payload()],
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us", _UPLIFT_MULTIPLIER)])
|
||||
def test_generate_content_spend_follows_the_vertex_location(
|
||||
regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float
|
||||
) -> None:
|
||||
expected: Final = (7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * multiplier
|
||||
|
||||
assert _cost_the_logger_records(location, stream=stream) == pytest.approx(expected), (location, stream)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"gemini_api_base", ["https://generativelanguage.googleapis.com", "https://llm-egress.internal.example/google"]
|
||||
)
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
def test_gemini_route_spend_has_no_vertex_uplift_whatever_host_serves_it(
|
||||
regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, gemini_api_base: str, stream: bool
|
||||
) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"gemini/{_UPLIFT_MODEL}",
|
||||
{
|
||||
"input_cost_per_token": _UPLIFT_INPUT_RATE,
|
||||
"output_cost_per_token": _UPLIFT_OUTPUT_RATE,
|
||||
"litellm_provider": "gemini",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent"
|
||||
|
||||
recorded: Final = _cost_the_logger_records_for(
|
||||
f"{gemini_api_base}/v1beta/models/{_UPLIFT_MODEL}:{method}",
|
||||
stream=stream,
|
||||
reply=_generate_content_payload(),
|
||||
stream_events=[_generate_content_payload()],
|
||||
route_provider="gemini",
|
||||
)
|
||||
|
||||
assert recorded == pytest.approx(7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE), (gemini_api_base, stream)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
def test_a_vertex_host_that_only_ends_with_the_gemini_host_name_keeps_the_vertex_uplift(
|
||||
regional_uplift_rate_card: None, stream: bool
|
||||
) -> None:
|
||||
method: Final = "streamGenerateContent" if stream else "generateContent"
|
||||
|
||||
recorded: Final = _cost_the_logger_records_for(
|
||||
"https://vertex-generativelanguage.googleapis.com/v1/projects/p/locations/us/publishers/google/models/"
|
||||
f"{_UPLIFT_MODEL}:{method}",
|
||||
stream=stream,
|
||||
reply=_generate_content_payload(),
|
||||
stream_events=[_generate_content_payload()],
|
||||
)
|
||||
|
||||
assert recorded == pytest.approx((7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * _UPLIFT_MULTIPLIER), stream
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
def test_a_subdomain_of_the_gemini_host_bills_the_gemini_price(
|
||||
regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool
|
||||
) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"gemini/{_UPLIFT_MODEL}",
|
||||
{
|
||||
"input_cost_per_token": _UPLIFT_INPUT_RATE,
|
||||
"output_cost_per_token": _UPLIFT_OUTPUT_RATE,
|
||||
"litellm_provider": "gemini",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent"
|
||||
|
||||
recorded: Final = _cost_the_logger_records_for(
|
||||
f"https://eu.generativelanguage.googleapis.com/v1beta/models/{_UPLIFT_MODEL}:{method}",
|
||||
stream=stream,
|
||||
reply=_generate_content_payload(),
|
||||
stream_events=[_generate_content_payload()],
|
||||
)
|
||||
|
||||
assert recorded == pytest.approx(7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE), stream
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [123, ["gemini-2.5-flash-lite"], {"model": "gemini-2.5-flash-lite"}])
|
||||
@pytest.mark.parametrize("carrier", ["root", "model_info"])
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
def test_a_non_string_base_model_bills_the_list_price(
|
||||
regional_uplift_rate_card: None, stream: bool, carrier: str, value: object
|
||||
) -> None:
|
||||
method: Final = "streamGenerateContent" if stream else "generateContent"
|
||||
|
||||
recorded: Final = _cost_the_logger_records_for(
|
||||
f"https://aiplatform.googleapis.com/v1/projects/p/locations/us/publishers/google/models/{_UPLIFT_MODEL}:{method}",
|
||||
stream=stream,
|
||||
reply=_generate_content_payload(),
|
||||
stream_events=[_generate_content_payload()],
|
||||
model_info={"base_model": value} if carrier == "model_info" else None,
|
||||
base_model=value if carrier == "root" else None,
|
||||
)
|
||||
|
||||
assert recorded == pytest.approx((7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * _UPLIFT_MULTIPLIER), stream
|
||||
|
||||
|
||||
_CUSTOM_PRICED_DEPLOYMENT: Final = "vertex-custom-priced-deployment"
|
||||
_CUSTOM_INPUT_RATE: Final = 2e-09
|
||||
_CUSTOM_OUTPUT_RATE: Final = 3e-09
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
@pytest.mark.parametrize("location", ["global", "us"])
|
||||
def test_generate_content_spend_uses_the_deployment_custom_price(
|
||||
regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool, location: str
|
||||
) -> None:
|
||||
custom_rates: Final = {"input_cost_per_token": _CUSTOM_INPUT_RATE, "output_cost_per_token": _CUSTOM_OUTPUT_RATE}
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"}
|
||||
)
|
||||
expected: Final = 7 * _CUSTOM_INPUT_RATE + 1 * _CUSTOM_OUTPUT_RATE
|
||||
|
||||
recorded: Final = _cost_the_logger_records(
|
||||
location, stream=stream, model_info={"id": _CUSTOM_PRICED_DEPLOYMENT, **custom_rates}
|
||||
)
|
||||
|
||||
assert recorded == pytest.approx(expected), (location, stream)
|
||||
|
||||
|
||||
_CLAUDE_UPLIFT_MODEL: Final = "claude-regional-uplift-probe"
|
||||
_CLAUDE_INPUT_RATE: Final = 3e-06
|
||||
_CLAUDE_OUTPUT_RATE: Final = 1.5e-05
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def claude_regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
rates: Final = {
|
||||
"input_cost_per_token": _CLAUDE_INPUT_RATE,
|
||||
"output_cost_per_token": _CLAUDE_OUTPUT_RATE,
|
||||
"mode": "chat",
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, _CLAUDE_UPLIFT_MODEL, {**rates, "litellm_provider": "anthropic"})
|
||||
monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, _CLAUDE_UPLIFT_MODEL})
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"vertex_ai/{_CLAUDE_UPLIFT_MODEL}",
|
||||
{
|
||||
**rates,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER,
|
||||
},
|
||||
)
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def _claude_message() -> dict[str, object]:
|
||||
return {
|
||||
"id": "msg_probe",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": _CLAUDE_UPLIFT_MODEL,
|
||||
"content": [{"type": "text", "text": "pong"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 7, "output_tokens": 1},
|
||||
}
|
||||
|
||||
|
||||
def _claude_stream_events() -> list[dict[str, object]]:
|
||||
message_start: Final = {**_claude_message(), "content": [], "usage": {"input_tokens": 7, "output_tokens": 0}}
|
||||
return [
|
||||
{"type": "message_start", "message": message_start},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}},
|
||||
{"type": "message_stop"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us-east5", _UPLIFT_MULTIPLIER)])
|
||||
def test_claude_raw_predict_spend_follows_the_vertex_location(
|
||||
claude_regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float
|
||||
) -> None:
|
||||
method: Final = "streamRawPredict" if stream else "rawPredict"
|
||||
url_route: Final = (
|
||||
f"https://aiplatform.googleapis.com/v1/projects/p/locations/{location}"
|
||||
f"/publishers/anthropic/models/{_CLAUDE_UPLIFT_MODEL}:{method}"
|
||||
)
|
||||
expected: Final = (7 * _CLAUDE_INPUT_RATE + 1 * _CLAUDE_OUTPUT_RATE) * multiplier
|
||||
|
||||
recorded: Final = _cost_the_logger_records_for(
|
||||
url_route, stream=stream, reply=_claude_message(), stream_events=_claude_stream_events()
|
||||
)
|
||||
|
||||
assert recorded == pytest.approx(expected), (location, stream)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"])
|
||||
def test_claude_raw_predict_spend_uses_the_deployment_custom_price(
|
||||
claude_regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool
|
||||
) -> None:
|
||||
custom_rates: Final = {"input_cost_per_token": _CUSTOM_INPUT_RATE, "output_cost_per_token": _CUSTOM_OUTPUT_RATE}
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"}
|
||||
)
|
||||
method: Final = "streamRawPredict" if stream else "rawPredict"
|
||||
url_route: Final = (
|
||||
"https://aiplatform.googleapis.com/v1/projects/p/locations/us-east5"
|
||||
f"/publishers/anthropic/models/{_CLAUDE_UPLIFT_MODEL}:{method}"
|
||||
)
|
||||
|
||||
recorded: Final = _cost_the_logger_records_for(
|
||||
url_route,
|
||||
stream=stream,
|
||||
reply=_claude_message(),
|
||||
stream_events=_claude_stream_events(),
|
||||
model_info={"id": _CUSTOM_PRICED_DEPLOYMENT, **custom_rates},
|
||||
)
|
||||
|
||||
assert recorded == pytest.approx(7 * _CUSTOM_INPUT_RATE + 1 * _CUSTOM_OUTPUT_RATE), stream
|
||||
|
||||
|
||||
def test_a_claude_stream_through_a_configured_vertex_gateway_with_a_short_path_bills_the_vertex_price(
|
||||
claude_regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
f"vertex_ai/{_CLAUDE_UPLIFT_MODEL}",
|
||||
{
|
||||
"input_cost_per_token": _CLAUDE_INPUT_RATE,
|
||||
"output_cost_per_token": _CLAUDE_OUTPUT_RATE,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
recorded: Final = _cost_the_logger_records_for(
|
||||
f"https://vertex-gateway.internal.example/vxc/models/{_CLAUDE_UPLIFT_MODEL}:streamRawPredict",
|
||||
stream=True,
|
||||
reply=_claude_message(),
|
||||
stream_events=_claude_stream_events(),
|
||||
)
|
||||
|
||||
assert recorded == pytest.approx(7 * _CLAUDE_INPUT_RATE + 1 * _CLAUDE_OUTPUT_RATE)
|
||||
|
||||
|
||||
def test_build_complete_streaming_response_assembles_stream_generate_content_sse_chunks():
|
||||
logging_obj: Final = Logging(
|
||||
model="gemini-2.5-flash",
|
||||
|
|
|
|||
|
|
@ -8452,7 +8452,9 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
|
|||
_PROXY_SERVER_REQUEST: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _split_pass_through_body(body: str) -> _PassThroughSplit:
|
||||
def _split_pass_through_body(
|
||||
body: str, user_api_key_dict: UserAPIKeyAuth = UserAPIKeyAuth(api_key="hashed-key")
|
||||
) -> _PassThroughSplit:
|
||||
mock_request: Final = MagicMock(spec=Request)
|
||||
mock_request.method = "POST"
|
||||
mock_request.url = httpx.URL("http://0.0.0.0:4000/gemini/v1beta/models/gemini-2.5-flash:generateContent")
|
||||
|
|
@ -8462,7 +8464,7 @@ def _split_pass_through_body(body: str) -> _PassThroughSplit:
|
|||
init_kwargs_for_pass_through_endpoint: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # untyped legacy helper
|
||||
kwargs: Final = init_kwargs_for_pass_through_endpoint(
|
||||
request=mock_request,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
passthrough_logging_payload=MagicMock(),
|
||||
logging_obj=MagicMock(),
|
||||
_parsed_body=json.loads(body),
|
||||
|
|
@ -8500,6 +8502,7 @@ def test_passthrough_moves_every_litellm_owned_key_from_the_forwarded_body_into_
|
|||
assert split.forwarded_body == json.loads(GEMINI_BODY)
|
||||
|
||||
|
||||
PRICING_MODEL_OVERRIDE_NAMES: Final = frozenset(("base_model",))
|
||||
PROXY_STAMPED_NAMES: Final = frozenset(
|
||||
(
|
||||
"proxy_server_request",
|
||||
|
|
@ -8515,7 +8518,12 @@ PROXY_STAMPED_NAMES: Final = frozenset(
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
sorted(frozenset(litellm.all_litellm_params) - frozenset(("metadata", "litellm_metadata")) - PROXY_STAMPED_NAMES),
|
||||
sorted(
|
||||
frozenset(litellm.all_litellm_params)
|
||||
- frozenset(("metadata", "litellm_metadata"))
|
||||
- PROXY_STAMPED_NAMES
|
||||
- PRICING_MODEL_OVERRIDE_NAMES
|
||||
),
|
||||
)
|
||||
def test_passthrough_keeps_each_registered_litellm_owned_name_out_of_the_forwarded_body(name: str) -> None:
|
||||
split: Final = _split_pass_through_body(json.dumps({name: "owned", **json.loads(GEMINI_BODY)}))
|
||||
|
|
@ -8525,6 +8533,41 @@ def test_passthrough_keeps_each_registered_litellm_owned_name_out_of_the_forward
|
|||
assert split.forwarded_body == json.loads(GEMINI_BODY)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("allows_client_pricing", [True, False], ids=["key-allows-client-pricing", "plain-key"])
|
||||
def test_passthrough_prices_with_a_client_base_model_only_for_keys_that_allow_client_pricing(
|
||||
allows_client_pricing: bool,
|
||||
) -> None:
|
||||
key: Final = UserAPIKeyAuth(
|
||||
api_key="hashed-key", metadata={"allow_client_pricing_override": True} if allows_client_pricing else {}
|
||||
)
|
||||
|
||||
split: Final = _split_pass_through_body(
|
||||
json.dumps({"base_model": "vertex_ai/gemini-2.5-flash-lite", **json.loads(GEMINI_BODY)}), key
|
||||
)
|
||||
|
||||
expected: Final = "vertex_ai/gemini-2.5-flash-lite" if allows_client_pricing else None
|
||||
assert split.litellm_params.get("base_model") == expected
|
||||
assert split.forwarded_body == json.loads(GEMINI_BODY)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", ["malformed", ["malformed"], 123])
|
||||
def test_passthrough_drops_a_client_model_info_that_is_not_an_object(model_info: object) -> None:
|
||||
key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"allow_client_pricing_override": True})
|
||||
plain_metadata: Final = _metadata_of(_split_pass_through_body(json.dumps({"metadata": {"tag": "a"}}), key))
|
||||
|
||||
split: Final = _split_pass_through_body(json.dumps({"metadata": {"tag": "a", "model_info": model_info}}), key)
|
||||
|
||||
assert _metadata_of(split) == plain_metadata
|
||||
|
||||
|
||||
def test_passthrough_keeps_a_client_model_info_object_for_keys_that_allow_client_pricing() -> None:
|
||||
key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"allow_client_pricing_override": True})
|
||||
|
||||
split: Final = _split_pass_through_body(json.dumps({"metadata": {"model_info": {"base_model": "gemini/x"}}}), key)
|
||||
|
||||
assert _metadata_of(split)["model_info"] == {"base_model": "gemini/x"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(PROXY_STAMPED_NAMES))
|
||||
def test_passthrough_drops_a_client_supplied_proxy_stamped_name(name: str) -> None:
|
||||
split: Final = _split_pass_through_body(json.dumps({name: {"forged": "by-client"}, **json.loads(GEMINI_BODY)}))
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import json
|
|||
import logging
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -13,6 +14,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrou
|
|||
VertexPassthroughLoggingHandler,
|
||||
)
|
||||
from litellm.proxy.pass_through_endpoints.streaming_handler import (
|
||||
PassThroughStreamContext,
|
||||
PassThroughStreamingHandler,
|
||||
)
|
||||
from litellm.proxy.pass_through_endpoints.success_handler import (
|
||||
|
|
@ -235,3 +237,40 @@ async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(
|
|||
partial_usage = logging_obj.record_partial_usage_for_failure.call_args.kwargs["usage"]
|
||||
assert partial_usage.completion_tokens > 100_000
|
||||
assert_loop_stayed_free(took, lags)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interrupted_vertex_stream_bills_the_deployment_custom_price(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
deployment_id: Final = "vertex-custom-priced-deployment"
|
||||
custom_rates: Final = {"input_cost_per_token": 2e-09, "output_cost_per_token": 3e-09}
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, deployment_id, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"}
|
||||
)
|
||||
logging_obj: Final = LiteLLMLoggingObj(
|
||||
model="unknown",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
stream=True,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="interrupted-stream",
|
||||
function_id="interrupted-stream",
|
||||
)
|
||||
logging_obj.optional_params = {}
|
||||
logging_obj.litellm_params = {"metadata": {"model_info": {"id": deployment_id, **custom_rates}}}
|
||||
url_route: Final = (
|
||||
f"https://aiplatform.googleapis.com/v1/projects/p/locations/us"
|
||||
f"/publishers/google/models/{MODEL}:streamGenerateContent?alt=sse"
|
||||
)
|
||||
|
||||
await PassThroughStreamingHandler.schedule_stream_failure_logging(
|
||||
litellm_logging_obj=logging_obj,
|
||||
endpoint_type=EndpointType.VERTEX_AI,
|
||||
request_body={},
|
||||
raw_bytes=[chunk.encode("utf-8") for chunk in _chunks()],
|
||||
exception=RuntimeError("upstream closed the stream"),
|
||||
stream_context=PassThroughStreamContext(PassThroughEndpointLogging(), url_route, datetime.now()),
|
||||
)
|
||||
recorded: Final = logging_obj.model_call_details["response_cost"]
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
|
||||
assert recorded == pytest.approx(PROMPT_TOKENS * 2e-09 + COMPLETION_TOKENS * 3e-09)
|
||||
|
|
|
|||
|
|
@ -972,6 +972,54 @@ async def test_proxy_startup_event_honors_disable_model_info_refresh(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_shutdown_commits_spend_before_waiting_on_a_slow_logging_task_and_again_after_it() -> None:
|
||||
"""A worker killed while shutdown waits on a slow logging task keeps the spend it had already counted,
|
||||
and a worker that finishes the wait also commits the spend that task adds."""
|
||||
from litellm.responses import streaming_iterator as streaming_module
|
||||
|
||||
task_may_finish: Final = asyncio.Event()
|
||||
task_finished: Final = asyncio.Event()
|
||||
flushes: Final[list[tuple[str, bool]]] = [] # mutable-ok: records what each shutdown flush saw, which is the assertion
|
||||
|
||||
async def slow_logging_task() -> None:
|
||||
await task_may_finish.wait()
|
||||
task_finished.set()
|
||||
|
||||
def recording_flush(name: str) -> Callable[[], Awaitable[None]]:
|
||||
async def flush() -> None:
|
||||
flushes.append((name, task_finished.is_set()))
|
||||
task_may_finish.set()
|
||||
|
||||
return flush
|
||||
|
||||
scheduler: Final = AsyncIOScheduler()
|
||||
clean_env: Final = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | {
|
||||
"LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true"
|
||||
}
|
||||
with (
|
||||
patch.dict(os.environ, clean_env, clear=True),
|
||||
patch.object(ps, "scheduler", scheduler),
|
||||
patch.object(ps, "flush_spend_counters_on_shutdown", recording_flush("spend counters")),
|
||||
patch.object(ps, "_flush_spend_logs_queue_on_shutdown", recording_flush("spend logs")),
|
||||
):
|
||||
try:
|
||||
async with proxy_startup_event(app=None):
|
||||
streaming_module.spawn_logging_task(
|
||||
asyncio.get_running_loop(), slow_logging_task(), task_name="slow spend row"
|
||||
)
|
||||
finally:
|
||||
if scheduler.running:
|
||||
scheduler.shutdown(wait=False)
|
||||
|
||||
assert flushes == [
|
||||
("spend counters", False),
|
||||
("spend logs", False),
|
||||
("spend counters", True),
|
||||
("spend logs", True),
|
||||
]
|
||||
|
||||
|
||||
def test_otel_global_provider_published_after_callback_init():
|
||||
"""The OTel V2 global-provider publish must run after callback
|
||||
initialization in ``proxy_startup_event``.
|
||||
|
|
|
|||
|
|
@ -9,17 +9,20 @@ import json
|
|||
import unittest
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Final, Literal, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import Response
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.router import RouterErrors
|
||||
from tests._master_key import MASTER_KEY
|
||||
|
||||
|
||||
|
|
@ -1648,7 +1651,7 @@ class TestManagedResponsesWSFirstMessage:
|
|||
|
||||
processed: list = []
|
||||
|
||||
async def fake_process(msg: str) -> None:
|
||||
async def fake_process(msg: str, turn_admission: object = None) -> None:
|
||||
processed.append(msg)
|
||||
|
||||
handler = ManagedResponsesWebSocketHandler(
|
||||
|
|
@ -1693,30 +1696,32 @@ class TestResponsesWSStreamingFirstMessage:
|
|||
backend_ws.send.assert_awaited_once_with(first)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("failure", "expected_status", "expected_type"),
|
||||
[
|
||||
(ValueError("No deployments available - crossed budget: Exceeded budget for deployment"), 429, "rate_limit_exceeded"),
|
||||
(litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error"),
|
||||
(RuntimeError("boom"), 500, "server_error"),
|
||||
(ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error"),
|
||||
(ProxyException(message="Authentication Error", type="auth_error", param=None, code=401), 401, "invalid_request_error"),
|
||||
(HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error"),
|
||||
],
|
||||
ids=["crossed-budget", "bad-request", "unexpected", "tag-routing", "proxy-auth", "http-exception"],
|
||||
)
|
||||
def test_responses_ws_failure_frame_uses_the_http_status_for_the_same_failure(
|
||||
failure: Exception, expected_status: int, expected_type: str
|
||||
):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame
|
||||
from litellm.proxy.utils import handle_exception_on_proxy
|
||||
|
||||
frame: Final = json.loads(_responses_ws_failure_frame(failure))
|
||||
|
||||
assert frame["status"] == expected_status
|
||||
assert frame["error"]["type"] == expected_type
|
||||
assert frame["error"]["message"] == handle_exception_on_proxy(failure).message
|
||||
|
||||
|
||||
class TestWSSessionCostTracking:
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_budget_limiter_skips_aresponses_websocket_call_type(self):
|
||||
"""
|
||||
RouterBudgetLimiting.async_log_success_event must not raise when
|
||||
call_type='_aresponses_websocket', even when standard_logging_object is None.
|
||||
Per-turn costs are tracked by individual aresponses calls inside the session;
|
||||
the outer session wrapper fires with result=None.
|
||||
"""
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
|
||||
limiter = RouterBudgetLimiting.__new__(RouterBudgetLimiting)
|
||||
kwargs = {
|
||||
"call_type": "_aresponses_websocket",
|
||||
"standard_logging_object": None,
|
||||
"litellm_params": {"custom_llm_provider": "vertex_ai"},
|
||||
}
|
||||
await limiter.async_log_success_event(
|
||||
kwargs=kwargs,
|
||||
response_obj=None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_budget_limiter_skips_arealtime_call_type(self):
|
||||
"""Same guard applies to _arealtime WS session wrappers."""
|
||||
|
|
@ -1996,7 +2001,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
"standard_logging_guardrail_information": [{"guardrail_name": "admission"}],
|
||||
},
|
||||
}
|
||||
guard = _responses_ws_frame_guard(admitted, MagicMock(), proxy_logging_obj)
|
||||
guard = _responses_ws_frame_guard(admitted, MagicMock(), proxy_logging_obj, MagicMock())
|
||||
|
||||
first = await guard(json.dumps({"type": "response.create", "model": "alias", "input": "hi"}))
|
||||
second = await guard(json.dumps({"type": "response.create", "input": "again"}))
|
||||
|
|
@ -2027,7 +2032,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
rate_limiter.async_post_call_failure_hook = AsyncMock()
|
||||
user_api_key_dict = MagicMock()
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, user_api_key_dict, proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, user_api_key_dict, proxy_logging_obj, MagicMock()
|
||||
)
|
||||
|
||||
with (
|
||||
|
|
@ -2059,7 +2064,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
tools = [{"type": "function", "name": "a"}, {"type": "function", "name": "b"}]
|
||||
|
||||
|
|
@ -2078,7 +2083,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
metadata = {
|
||||
"user_api_key_metadata": {"disable_global_guardrails": True},
|
||||
|
|
@ -2109,6 +2114,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
{"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}},
|
||||
MagicMock(),
|
||||
proxy_logging_obj,
|
||||
MagicMock(),
|
||||
)
|
||||
|
||||
await guard(json.dumps(frame))
|
||||
|
|
@ -2131,6 +2137,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
{"model": "gpt-4o-mini", "litellm_metadata": {"litellm_parent_otel_span": span}},
|
||||
MagicMock(),
|
||||
proxy_logging_obj,
|
||||
MagicMock(),
|
||||
)
|
||||
|
||||
guarded = await guard(json.dumps({"type": "response.create", "input": "hi"}))
|
||||
|
|
@ -2156,7 +2163,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
spend_logger = MagicMock(spec=ProxyDBLogger)
|
||||
spend_logger.async_post_call_failure_hook = AsyncMock()
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
|
||||
with (
|
||||
|
|
@ -2189,7 +2196,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
spend_logger = MagicMock(spec=ProxyDBLogger)
|
||||
spend_logger.async_post_call_failure_hook = AsyncMock()
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
body = {
|
||||
"input": "forbidden",
|
||||
|
|
@ -2215,7 +2222,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
body = {"instructions": "SECRET system prompt", "input": "hi", "store": False}
|
||||
frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body}
|
||||
|
|
@ -2240,7 +2247,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
|
||||
with patch.object(litellm, "callbacks", []):
|
||||
|
|
@ -2264,7 +2271,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
|
||||
with patch.object(litellm, "callbacks", []), pytest.raises(ValueError, match="both at the top level"):
|
||||
|
|
@ -2283,7 +2290,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
spend_logger = MagicMock(spec=ProxyDBLogger)
|
||||
spend_logger.async_post_call_failure_hook = AsyncMock(side_effect=RuntimeError("db down"))
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock()
|
||||
)
|
||||
|
||||
with patch.object(litellm, "callbacks", [spend_logger]), pytest.raises(ValueError, match="blocked"):
|
||||
|
|
@ -2360,6 +2367,345 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
assert booked["user_api_key_dict"] is user_api_key_dict
|
||||
assert booked["request_data"]["model"] == "gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("ending", ["refusal", "crash", "session_limit"])
|
||||
async def test_close_time_failure_row_bills_the_guardrails_of_the_turns_the_deployment_refused(self, ending: str):
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
|
||||
|
||||
ws = MagicMock()
|
||||
ws.headers = {}
|
||||
ws.query_params = {}
|
||||
ws.scope = {"headers": []}
|
||||
ws.url = "ws://testserver/v1/responses"
|
||||
ws.accept = AsyncMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
|
||||
)
|
||||
ws.close = AsyncMock()
|
||||
processor = MagicMock()
|
||||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {"model_group": "gpt-4o-mini"}}, MagicMock())
|
||||
)
|
||||
guardrail_costs = {"two": 0.2, "three": 0.3, "four": 0.4}
|
||||
refusal = RateLimitError(message="crossed budget", llm_provider="", model="")
|
||||
later_refusal = RateLimitError(message="still over budget", llm_provider="", model="")
|
||||
crash = RuntimeError("relay crashed")
|
||||
|
||||
async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only):
|
||||
recorded = [{"guardrail_cost": guardrail_costs[data["input"]]}]
|
||||
return {
|
||||
**data,
|
||||
"litellm_metadata": {**data["litellm_metadata"], "standard_logging_guardrail_information": recorded},
|
||||
}
|
||||
|
||||
def route_request(data, **kwargs):
|
||||
async def relay():
|
||||
for turn in ("two", "three", "four"):
|
||||
guarded = await data["responses_ws_frame_guard"](
|
||||
json.dumps({"type": "response.create", "input": turn})
|
||||
)
|
||||
if turn != "three":
|
||||
guarded.bill_refusal(refusal if turn == "two" else later_refusal)
|
||||
if ending == "crash":
|
||||
raise crash
|
||||
if ending == "session_limit":
|
||||
await asyncio.Event().wait()
|
||||
return refusal
|
||||
|
||||
return relay()
|
||||
|
||||
row_written = asyncio.Event()
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=lambda **kwargs: row_written.set())
|
||||
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None)
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above
|
||||
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
|
||||
new_callable=AsyncMock,
|
||||
),
|
||||
patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test
|
||||
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
|
||||
return_value=processor,
|
||||
),
|
||||
patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome
|
||||
"litellm.proxy.route_llm_request.route_request",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=route_request,
|
||||
),
|
||||
patch( # test-quality-ok: a short session limit stands in for the configured one so the cap fires mid-session
|
||||
"litellm.proxy.response_api_endpoints.endpoints._resolve_responses_ws_session_limit_seconds",
|
||||
return_value=0.05 if ending == "session_limit" else 3600.0,
|
||||
),
|
||||
patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj",
|
||||
proxy_logging_obj,
|
||||
),
|
||||
):
|
||||
await responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock())
|
||||
await asyncio.wait_for(row_written.wait(), timeout=1)
|
||||
|
||||
proxy_logging_obj.post_call_failure_hook.assert_awaited_once()
|
||||
proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once()
|
||||
booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs
|
||||
assert booked["original_exception"] is (crash if ending == "crash" else refusal)
|
||||
assert booked["request_data"]["litellm_metadata"] == {
|
||||
"model_group": "gpt-4o-mini",
|
||||
"standard_logging_guardrail_information": [{"guardrail_cost": 0.2}, {"guardrail_cost": 0.4}],
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"relay_outcome",
|
||||
["still_relaying", "already_returned", "returns_despite_the_cancel", "crashes_despite_the_cancel"],
|
||||
)
|
||||
async def test_session_limit_frees_the_slot_at_once_and_never_cuts_the_failure_row(self, relay_outcome: str):
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
|
||||
|
||||
ws = MagicMock()
|
||||
ws.headers = {}
|
||||
ws.query_params = {}
|
||||
ws.scope = {"headers": []}
|
||||
ws.url = "ws://testserver/v1/responses"
|
||||
ws.accept = AsyncMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
|
||||
)
|
||||
async def close_over_the_network(**kwargs):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
ws.close = AsyncMock(side_effect=close_over_the_network)
|
||||
processor = MagicMock()
|
||||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
|
||||
)
|
||||
refusal = RateLimitError(message="crossed budget", llm_provider="", model="")
|
||||
crash = RuntimeError("relay crashed")
|
||||
row_may_finish = asyncio.Event()
|
||||
row_written = asyncio.Event()
|
||||
|
||||
async def slow_failure_hook(**kwargs):
|
||||
await row_may_finish.wait()
|
||||
row_written.set()
|
||||
|
||||
def route_request(data, **kwargs):
|
||||
async def relay():
|
||||
guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"}))
|
||||
guarded.bill_refusal(refusal)
|
||||
if relay_outcome == "still_relaying":
|
||||
await asyncio.Event().wait()
|
||||
if relay_outcome in ("returns_despite_the_cancel", "crashes_despite_the_cancel"):
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await asyncio.Event().wait()
|
||||
if relay_outcome == "crashes_despite_the_cancel":
|
||||
raise crash
|
||||
return refusal
|
||||
|
||||
return relay()
|
||||
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(
|
||||
side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data
|
||||
)
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=slow_failure_hook)
|
||||
async def release_over_the_network(user_api_key_dict):
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(side_effect=release_over_the_network)
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above
|
||||
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
|
||||
new_callable=AsyncMock,
|
||||
),
|
||||
patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test
|
||||
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
|
||||
return_value=processor,
|
||||
),
|
||||
patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome
|
||||
"litellm.proxy.route_llm_request.route_request",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=route_request,
|
||||
),
|
||||
patch( # test-quality-ok: a short session limit stands in for the configured one so the cap fires mid-session
|
||||
"litellm.proxy.response_api_endpoints.endpoints._resolve_responses_ws_session_limit_seconds",
|
||||
return_value=0.05,
|
||||
),
|
||||
patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj",
|
||||
proxy_logging_obj,
|
||||
),
|
||||
):
|
||||
endpoint = asyncio.ensure_future(
|
||||
responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock())
|
||||
)
|
||||
finished, _ = await asyncio.wait({endpoint}, timeout=1)
|
||||
assert endpoint in finished
|
||||
limit_close = call(code=1000, reason="Session duration limit reached")
|
||||
crash_close = call(code=1011, reason="Internal server error")
|
||||
crashed = relay_outcome == "crashes_despite_the_cancel"
|
||||
assert ws.close.await_args_list == ([limit_close, crash_close] if crashed else [limit_close])
|
||||
proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once()
|
||||
assert not row_written.is_set()
|
||||
row_may_finish.set()
|
||||
await asyncio.wait_for(row_written.wait(), timeout=1)
|
||||
|
||||
assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"] is (crash if crashed else refusal)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("ending", ["client_closed", "crash"])
|
||||
async def test_a_closed_socket_frees_its_slot_before_its_failure_row_and_waits_for_the_row(self, ending: str):
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
|
||||
|
||||
ws = MagicMock()
|
||||
ws.headers = {}
|
||||
ws.query_params = {}
|
||||
ws.scope = {"headers": []}
|
||||
ws.url = "ws://testserver/v1/responses"
|
||||
ws.accept = AsyncMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
|
||||
)
|
||||
ws.close = AsyncMock()
|
||||
processor = MagicMock()
|
||||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
|
||||
)
|
||||
refusal = RateLimitError(message="crossed budget", llm_provider="", model="")
|
||||
crash = RuntimeError("relay crashed")
|
||||
row_started = asyncio.Event()
|
||||
row_may_finish = asyncio.Event()
|
||||
row_written = asyncio.Event()
|
||||
|
||||
async def slow_failure_hook(**kwargs):
|
||||
row_started.set()
|
||||
await row_may_finish.wait()
|
||||
row_written.set()
|
||||
|
||||
def route_request(data, **kwargs):
|
||||
async def relay():
|
||||
guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"}))
|
||||
guarded.bill_refusal(refusal)
|
||||
if ending == "crash":
|
||||
raise crash
|
||||
return refusal
|
||||
|
||||
return relay()
|
||||
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(
|
||||
side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data
|
||||
)
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=slow_failure_hook)
|
||||
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None)
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above
|
||||
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
|
||||
new_callable=AsyncMock,
|
||||
),
|
||||
patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test
|
||||
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
|
||||
return_value=processor,
|
||||
),
|
||||
patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome
|
||||
"litellm.proxy.route_llm_request.route_request",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=route_request,
|
||||
),
|
||||
patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj",
|
||||
proxy_logging_obj,
|
||||
),
|
||||
):
|
||||
endpoint = asyncio.ensure_future(
|
||||
responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock())
|
||||
)
|
||||
await asyncio.wait_for(row_started.wait(), timeout=1)
|
||||
proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once()
|
||||
finished_early, _ = await asyncio.wait({endpoint}, timeout=0.05)
|
||||
assert endpoint not in finished_early
|
||||
row_may_finish.set()
|
||||
finished, _ = await asyncio.wait({endpoint}, timeout=1)
|
||||
assert endpoint in finished
|
||||
assert row_written.is_set()
|
||||
|
||||
booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs
|
||||
assert booked["original_exception"] is (crash if ending == "crash" else refusal)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_crashed_socket_whose_failure_row_cannot_be_built_still_gets_its_error_frame_and_close(self):
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
|
||||
|
||||
ws = MagicMock()
|
||||
ws.headers = {}
|
||||
ws.query_params = {}
|
||||
ws.scope = {"headers": []}
|
||||
ws.url = "ws://testserver/v1/responses"
|
||||
ws.accept = AsyncMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
|
||||
)
|
||||
sent = []
|
||||
closes = []
|
||||
ws.send_text = AsyncMock(side_effect=sent.append)
|
||||
ws.close = AsyncMock(side_effect=lambda code, reason: closes.append((code, reason)))
|
||||
processor = MagicMock()
|
||||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
|
||||
)
|
||||
refusal = RateLimitError(message="crossed budget", llm_provider="", model="")
|
||||
|
||||
def route_request(data, **kwargs):
|
||||
async def relay():
|
||||
guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"}))
|
||||
guarded.bill_refusal(refusal)
|
||||
data["litellm_metadata"]["standard_logging_guardrail_information"] = "written by a broken callback"
|
||||
raise RuntimeError("relay crashed")
|
||||
|
||||
return relay()
|
||||
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(
|
||||
side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: {
|
||||
**data,
|
||||
"litellm_metadata": {"standard_logging_guardrail_information": [{"guardrail_name": "turn-check"}]},
|
||||
}
|
||||
)
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock()
|
||||
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None)
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above
|
||||
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
|
||||
new_callable=AsyncMock,
|
||||
),
|
||||
patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test
|
||||
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
|
||||
return_value=processor,
|
||||
),
|
||||
patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome
|
||||
"litellm.proxy.route_llm_request.route_request",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=route_request,
|
||||
),
|
||||
patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj",
|
||||
proxy_logging_obj,
|
||||
),
|
||||
):
|
||||
await asyncio.wait_for(
|
||||
responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()), timeout=1
|
||||
)
|
||||
|
||||
assert [json.loads(frame)["error"]["message"] for frame in sent] == ["relay crashed"]
|
||||
assert closes == [(1011, "Internal server error")]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
|
|
|
|||
|
|
@ -235,6 +235,29 @@ async def test_proxy_shutdown_closes_admin_connector_when_drain_is_cancelled(
|
|||
|
||||
|
||||
# Your bearer token
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_commits_spend_a_logging_task_queued_while_the_first_commit_was_writing():
|
||||
from litellm.responses.streaming_iterator import spawn_logging_task
|
||||
|
||||
queued: list[str] = []
|
||||
committed: list[str] = []
|
||||
|
||||
async def commit_spend() -> None:
|
||||
committed.extend(queued)
|
||||
queued.clear()
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
async def failure_row() -> None:
|
||||
await asyncio.sleep(0.01)
|
||||
queued.append("failure row spend")
|
||||
|
||||
spawn_logging_task(asyncio.get_running_loop(), failure_row(), task_name="failure row")
|
||||
|
||||
await litellm.proxy.proxy_server._flush_spend_around_logging_drain_on_shutdown(flush_spend=commit_spend)
|
||||
|
||||
assert committed == ["failure row spend"]
|
||||
|
||||
|
||||
token = MASTER_KEY
|
||||
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ Tests that:
|
|||
3. Providers without native websocket support use ManagedResponsesWebSocketHandler
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
|
@ -1331,7 +1332,7 @@ class TestWebSocketProjectQuotaEnforcement:
|
|||
callback, not just the first one."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_handler_blocks_frame_rejected_by_quota_callback(self, monkeypatch):
|
||||
async def test_managed_handler_blocks_frame_rejected_by_quota_callback(self, monkeypatch: pytest.MonkeyPatch):
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import litellm
|
||||
|
|
@ -1377,11 +1378,11 @@ class TestWebSocketProjectQuotaEnforcement:
|
|||
quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
|
||||
assert aresponses_called is False
|
||||
mock_websocket.send_text.assert_called_once()
|
||||
error_event = mock_websocket.send_text.call_args[0][0]
|
||||
assert "rate_limit_exceeded" in error_event
|
||||
error_event = json.loads(mock_websocket.send_text.call_args[0][0])
|
||||
assert (error_event["status"], error_event["error"]["type"]) == (429, "rate_limit_exceeded")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_handler_forwards_frame_allowed_by_quota_callback(self, monkeypatch):
|
||||
async def test_managed_handler_forwards_frame_allowed_by_quota_callback(self, monkeypatch: pytest.MonkeyPatch):
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import litellm
|
||||
|
|
@ -1462,7 +1463,8 @@ class TestWebSocketProjectQuotaEnforcement:
|
|||
assert allowed is False
|
||||
mock_backend_ws.send.assert_not_called()
|
||||
mock_websocket.send_text.assert_called_once()
|
||||
assert "rate_limit_exceeded" in mock_websocket.send_text.call_args[0][0]
|
||||
error_event = json.loads(mock_websocket.send_text.call_args[0][0])
|
||||
assert (error_event["status"], error_event["error"]["type"]) == (429, "rate_limit_exceeded")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_handler_runs_the_frame_guard_on_every_turn_and_bills_each_turn_its_own_guardrail_records(
|
||||
|
|
@ -1580,6 +1582,461 @@ class TestWebSocketProjectQuotaEnforcement:
|
|||
quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
|
||||
|
||||
|
||||
def _turn_frames() -> tuple[str, str, str]:
|
||||
return tuple(json.dumps({"type": "response.create", "input": word}) for word in ("one", "two", "three"))
|
||||
|
||||
|
||||
def _admission_refusing_the_next_turn():
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from litellm.exceptions import RateLimitError
|
||||
|
||||
return AsyncMock(side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), None])
|
||||
|
||||
|
||||
class _ClientSocket:
|
||||
"""A client socket that, like Starlette's, stops receiving once the server closes it."""
|
||||
|
||||
def __init__(self, *frames: str) -> None:
|
||||
self._frames: Final = iter(frames)
|
||||
self.sent: Final[list[dict[str, object]]] = []
|
||||
self.close_code: int | None = None
|
||||
|
||||
async def send_text(self, data: str) -> None:
|
||||
self.sent.append(json.loads(data))
|
||||
|
||||
async def receive_text(self) -> str:
|
||||
frame: Final = next(self._frames, None)
|
||||
if self.close_code is not None or frame is None:
|
||||
raise RuntimeError("WebSocket is not connected")
|
||||
return frame
|
||||
|
||||
async def close(self, code: int = 1000, reason: str | None = None) -> None:
|
||||
self.close_code = code
|
||||
|
||||
|
||||
class _StalledClientSocket(_ClientSocket):
|
||||
"""A client socket whose sends never finish, like a client that stopped reading."""
|
||||
|
||||
def __init__(self, *frames: str) -> None:
|
||||
super().__init__(*frames)
|
||||
self.send_started: Final = asyncio.Event()
|
||||
|
||||
async def send_text(self, data: str) -> None:
|
||||
self.send_started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
|
||||
def _admission_crashing_the_next_turn():
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
return AsyncMock(side_effect=[ValueError("custom check failed"), None])
|
||||
|
||||
|
||||
def _guard_blocking(word: str):
|
||||
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame
|
||||
|
||||
async def guard(frame: str) -> GuardedResponsesWsFrame:
|
||||
if word in json.loads(frame)["input"]:
|
||||
raise ValueError("blocked by guardrail")
|
||||
return GuardedResponsesWsFrame(frame=frame)
|
||||
|
||||
return guard
|
||||
|
||||
|
||||
def _guard_recording(bill_refusal: MagicMock):
|
||||
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame
|
||||
|
||||
async def guard(frame: str) -> GuardedResponsesWsFrame:
|
||||
return GuardedResponsesWsFrame(
|
||||
frame=frame, guardrail_information=({"guardrail_cost": 0.5},), bill_refusal=bill_refusal
|
||||
)
|
||||
|
||||
return guard
|
||||
|
||||
|
||||
def _errors(websocket: _ClientSocket) -> list[tuple[object, object]]:
|
||||
return [(frame.get("status"), frame["error"]) for frame in websocket.sent if frame.get("type") == "error"]
|
||||
|
||||
|
||||
class TestWebSocketTurnAdmission:
|
||||
"""A turn after the connection's first is admitted by the deployment again before it reaches the provider."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_drops_a_later_turn_the_deployment_refuses(self):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
first, second, third = _turn_frames()
|
||||
cancel: Final = json.dumps({"type": "response.cancel"})
|
||||
turn_admission: Final = _admission_refusing_the_next_turn()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
websocket: Final = MagicMock()
|
||||
websocket.send_text = AsyncMock()
|
||||
websocket.receive_text = AsyncMock(side_effect=[second, cancel, third, Exception("stop")])
|
||||
handler: Final = _make_streaming(
|
||||
websocket=websocket,
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
first_message=first,
|
||||
turn_admission=turn_admission,
|
||||
)
|
||||
|
||||
await handler.client_to_backend()
|
||||
|
||||
assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [
|
||||
json.loads(first),
|
||||
json.loads(cancel),
|
||||
json.loads(third),
|
||||
]
|
||||
assert turn_admission.await_count == 2
|
||||
refusal: Final = json.loads(websocket.send_text.await_args.args[0])
|
||||
assert refusal["status"] == 429
|
||||
assert refusal["error"]["type"] == "rate_limit_exceeded"
|
||||
assert "crossed budget" in refusal["error"]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_reports_a_refused_turn_as_the_socket_failure(self):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from litellm.exceptions import RateLimitError
|
||||
|
||||
first, second, _ = _turn_frames()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.close = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=Exception("stop"))
|
||||
websocket: Final = MagicMock()
|
||||
websocket.send_text = AsyncMock()
|
||||
websocket.receive_text = AsyncMock(side_effect=[second, Exception("client gone")])
|
||||
handler: Final = _make_streaming(
|
||||
websocket=websocket,
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
first_message=first,
|
||||
turn_admission=_admission_refusing_the_next_turn(),
|
||||
)
|
||||
|
||||
failure: Final = await handler.bidirectional_forward()
|
||||
|
||||
assert isinstance(failure, RateLimitError)
|
||||
assert "crossed budget" in str(failure)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler
|
||||
|
||||
answered: Final[list[object]] = []
|
||||
|
||||
async def fake_aresponses(*args: object, **kwargs: object):
|
||||
answered.append(kwargs.get("input"))
|
||||
|
||||
async def _empty():
|
||||
return
|
||||
yield
|
||||
|
||||
return _empty()
|
||||
|
||||
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
|
||||
first, second, third = _turn_frames()
|
||||
turn_admission: Final = _admission_refusing_the_next_turn()
|
||||
websocket: Final = MagicMock()
|
||||
websocket.send_text = AsyncMock()
|
||||
websocket.receive_text = AsyncMock(side_effect=[second, third, Exception("stop")])
|
||||
handler: Final = ManagedResponsesWebSocketHandler(
|
||||
websocket=websocket,
|
||||
model="test-model",
|
||||
logging_obj=Logging(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=True,
|
||||
call_type="aresponses",
|
||||
start_time=0,
|
||||
litellm_call_id="test-id",
|
||||
function_id="test-func",
|
||||
),
|
||||
first_message=first,
|
||||
turn_admission=turn_admission,
|
||||
)
|
||||
|
||||
failure: Final = await handler.run()
|
||||
|
||||
assert len(answered) == 2
|
||||
assert "two" not in json.dumps(answered)
|
||||
assert turn_admission.await_count == 2
|
||||
assert isinstance(failure, RateLimitError)
|
||||
assert "crossed budget" in str(failure)
|
||||
refusals: Final = [
|
||||
json.loads(call.args[0])
|
||||
for call in websocket.send_text.await_args_list
|
||||
if json.loads(call.args[0]).get("type") == "error"
|
||||
]
|
||||
assert [(refusal["status"], refusal["error"]["type"]) for refusal in refusals] == [(429, "rate_limit_exceeded")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_runs_the_guardrail_before_the_deployment_counts_a_turn(self):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
first, second, third = _turn_frames()
|
||||
turn_admission: Final = _admission_refusing_the_next_turn()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
websocket: Final = _ClientSocket(second, third)
|
||||
handler: Final = _make_streaming(
|
||||
websocket=websocket,
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
first_message=first,
|
||||
frame_guard=_guard_blocking("two"),
|
||||
turn_admission=turn_admission,
|
||||
)
|
||||
|
||||
await handler.client_to_backend()
|
||||
|
||||
assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [json.loads(first)]
|
||||
assert turn_admission.await_count == 1
|
||||
assert [(status, error["type"]) for status, error in _errors(websocket)] == [
|
||||
(None, "invalid_request_error"),
|
||||
(429, "rate_limit_exceeded"),
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_relay_runs_the_guardrail_before_the_deployment_counts_a_turn(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler
|
||||
|
||||
answered: Final[list[object]] = []
|
||||
|
||||
async def fake_aresponses(*args: object, **kwargs: object):
|
||||
answered.append(kwargs.get("input"))
|
||||
|
||||
async def _empty():
|
||||
return
|
||||
yield
|
||||
|
||||
return _empty()
|
||||
|
||||
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
|
||||
first, second, third = _turn_frames()
|
||||
turn_admission: Final = _admission_refusing_the_next_turn()
|
||||
websocket: Final = _ClientSocket(second, third)
|
||||
handler: Final = ManagedResponsesWebSocketHandler(
|
||||
websocket=websocket,
|
||||
model="test-model",
|
||||
logging_obj=Logging(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=True,
|
||||
call_type="aresponses",
|
||||
start_time=0,
|
||||
litellm_call_id="test-id",
|
||||
function_id="test-func",
|
||||
),
|
||||
first_message=first,
|
||||
frame_guard=_guard_blocking("two"),
|
||||
turn_admission=turn_admission,
|
||||
)
|
||||
|
||||
await handler.run()
|
||||
|
||||
assert answered == ["one"]
|
||||
assert turn_admission.await_count == 1
|
||||
assert [(status, error["type"]) for status, error in _errors(websocket)] == [
|
||||
(None, "invalid_request_error"),
|
||||
(429, "rate_limit_exceeded"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row(self):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from litellm.exceptions import RateLimitError
|
||||
|
||||
first, second, third = _turn_frames()
|
||||
refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="")
|
||||
bill_refusal: Final = MagicMock()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.close = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait)
|
||||
handler: Final = _make_streaming(
|
||||
websocket=_ClientSocket(second, third),
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
first_message=first,
|
||||
frame_guard=_guard_recording(bill_refusal),
|
||||
turn_admission=AsyncMock(side_effect=[refusal, None]),
|
||||
)
|
||||
|
||||
close_time_failure: Final = await handler.bidirectional_forward()
|
||||
|
||||
assert [json.loads(sent.args[0])["input"] for sent in backend_ws.send.await_args_list] == ["one", "three"]
|
||||
bill_refusal.assert_called_once_with(refusal)
|
||||
assert close_time_failure is refusal
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_refused_turn_is_recorded_even_when_the_relay_is_cancelled_while_answering_it(self):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from litellm.exceptions import RateLimitError
|
||||
|
||||
first, second, third = _turn_frames()
|
||||
refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="")
|
||||
bill_refusal: Final = MagicMock()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.close = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait)
|
||||
client: Final = _StalledClientSocket(second, third)
|
||||
handler: Final = _make_streaming(
|
||||
websocket=client,
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
first_message=first,
|
||||
frame_guard=_guard_recording(bill_refusal),
|
||||
turn_admission=AsyncMock(side_effect=[refusal, None]),
|
||||
)
|
||||
relay: Final = asyncio.ensure_future(handler.bidirectional_forward())
|
||||
await asyncio.wait_for(client.send_started.wait(), timeout=1)
|
||||
|
||||
relay.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await relay
|
||||
|
||||
bill_refusal.assert_called_once_with(refusal)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler
|
||||
|
||||
answered: Final[list[object]] = []
|
||||
|
||||
async def fake_aresponses(*args: object, **kwargs: object):
|
||||
answered.append(kwargs.get("input"))
|
||||
|
||||
async def _empty():
|
||||
return
|
||||
yield
|
||||
|
||||
return _empty()
|
||||
|
||||
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
|
||||
first, second, third = _turn_frames()
|
||||
refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="")
|
||||
bill_refusal: Final = MagicMock()
|
||||
handler: Final = ManagedResponsesWebSocketHandler(
|
||||
websocket=_ClientSocket(second, third),
|
||||
model="test-model",
|
||||
logging_obj=Logging(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=True,
|
||||
call_type="aresponses",
|
||||
start_time=0,
|
||||
litellm_call_id="test-id",
|
||||
function_id="test-func",
|
||||
),
|
||||
first_message=first,
|
||||
frame_guard=_guard_recording(bill_refusal),
|
||||
turn_admission=AsyncMock(side_effect=[refusal, None]),
|
||||
)
|
||||
|
||||
close_time_failure: Final = await handler.run()
|
||||
|
||||
assert answered == ["one", "three"]
|
||||
bill_refusal.assert_called_once_with(refusal)
|
||||
assert close_time_failure is refusal
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_answers_a_crashed_turn_check_with_a_500_and_keeps_serving(self):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
first, second, third = _turn_frames()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.close = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait)
|
||||
websocket: Final = _ClientSocket(second, third)
|
||||
handler: Final = _make_streaming(
|
||||
websocket=websocket,
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
first_message=first,
|
||||
turn_admission=_admission_crashing_the_next_turn(),
|
||||
)
|
||||
|
||||
failure: Final = await handler.bidirectional_forward()
|
||||
|
||||
assert isinstance(failure, ValueError)
|
||||
assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [
|
||||
json.loads(first),
|
||||
json.loads(third),
|
||||
]
|
||||
assert _errors(websocket) == [(500, {"type": "server_error", "message": "custom check failed"})]
|
||||
assert websocket.close_code is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_relay_answers_a_crashed_turn_check_with_a_500_and_keeps_serving(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler
|
||||
|
||||
answered: Final[list[object]] = []
|
||||
|
||||
async def fake_aresponses(*args: object, **kwargs: object):
|
||||
answered.append(kwargs.get("input"))
|
||||
|
||||
async def _empty():
|
||||
return
|
||||
yield
|
||||
|
||||
return _empty()
|
||||
|
||||
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
|
||||
first, second, third = _turn_frames()
|
||||
websocket: Final = _ClientSocket(second, third)
|
||||
handler: Final = ManagedResponsesWebSocketHandler(
|
||||
websocket=websocket,
|
||||
model="test-model",
|
||||
logging_obj=Logging(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=True,
|
||||
call_type="aresponses",
|
||||
start_time=0,
|
||||
litellm_call_id="test-id",
|
||||
function_id="test-func",
|
||||
),
|
||||
first_message=first,
|
||||
turn_admission=_admission_crashing_the_next_turn(),
|
||||
)
|
||||
|
||||
failure: Final = await handler.run()
|
||||
|
||||
assert isinstance(failure, ValueError)
|
||||
assert answered == ["one", "three"]
|
||||
assert _errors(websocket) == [(500, {"type": "server_error", "message": "custom check failed"})]
|
||||
assert websocket.close_code is None
|
||||
|
||||
def _deployment_defaults():
|
||||
from types import MappingProxyType
|
||||
|
||||
|
|
|
|||
|
|
@ -2594,3 +2594,28 @@ def test_faked_stream_builds_each_refusal_summary_and_custom_tool_input_once_for
|
|||
(1, "content", 1): "No forecasts for Mars.",
|
||||
(2, "input", 0): "curl wttr.in/Berlin",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_drain_waits_for_a_pending_logging_task():
|
||||
row_written = asyncio.Event()
|
||||
|
||||
async def write_row():
|
||||
await asyncio.sleep(0.05)
|
||||
row_written.set()
|
||||
|
||||
streaming_module.spawn_logging_task(asyncio.get_running_loop(), write_row(), task_name="row")
|
||||
|
||||
await streaming_module.drain_logging_tasks(timeout=1)
|
||||
|
||||
assert row_written.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout():
|
||||
stuck = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck")
|
||||
|
||||
await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1)
|
||||
|
||||
assert not stuck.done()
|
||||
stuck.cancel()
|
||||
|
|
|
|||
|
|
@ -60,7 +60,9 @@ async def _log_success(limiter: RouterBudgetLimiting, kwargs: dict[str, object])
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("call_type", ["aresponses", "anthropic_messages", "aembedding", "arerank"])
|
||||
@pytest.mark.parametrize(
|
||||
"call_type", ["aresponses", "_aresponses_websocket", "anthropic_messages", "aembedding", "arerank"]
|
||||
)
|
||||
async def test_provider_spend_tracked_when_litellm_params_omits_provider(disable_budget_sync, call_type):
|
||||
"""Non-chat surfaces carry the provider only on the standard logging payload."""
|
||||
limiter = RouterBudgetLimiting(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -13,6 +14,7 @@ from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_
|
|||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
ModelResponse,
|
||||
StandardLoggingHiddenParams,
|
||||
StandardLoggingMetadata,
|
||||
|
|
@ -482,6 +484,227 @@ async def test_deployment_callback_on_success(sync_mode):
|
|||
assert tpm_key is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deployment_callback_on_success_counts_a_responses_websocket_turn():
|
||||
import time
|
||||
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "gpt-5-mini",
|
||||
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-fake", "rpm": 100},
|
||||
"model_info": {"id": "ws-deployment"},
|
||||
}
|
||||
]
|
||||
router = Router(model_list=model_list)
|
||||
standard_logging_payload = create_standard_logging_payload()
|
||||
standard_logging_payload["total_tokens"] = 17
|
||||
|
||||
def turn_kwargs(call_type: str) -> dict:
|
||||
return {
|
||||
"call_type": call_type,
|
||||
"litellm_params": {
|
||||
"metadata": {"model_group": "gpt-5-mini"},
|
||||
"model_info": {"id": "ws-deployment"},
|
||||
},
|
||||
"standard_logging_object": standard_logging_payload,
|
||||
}
|
||||
|
||||
tpm_key = await router.deployment_callback_on_success(
|
||||
kwargs=turn_kwargs("_aresponses_websocket"),
|
||||
completion_response=None,
|
||||
start_time=time.time(),
|
||||
end_time=time.time(),
|
||||
)
|
||||
assert tpm_key is not None
|
||||
assert await router.cache.async_get_cache(key=tpm_key) == 17
|
||||
|
||||
realtime_key = await router.deployment_callback_on_success(
|
||||
kwargs=turn_kwargs("_arealtime"),
|
||||
completion_response=None,
|
||||
start_time=time.time(),
|
||||
end_time=time.time(),
|
||||
)
|
||||
assert realtime_key is None
|
||||
assert await router.cache.async_get_cache(key=tpm_key) == 17
|
||||
|
||||
|
||||
async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]:
|
||||
routed: Final[dict[str, object]] = {}
|
||||
|
||||
async def call(**kwargs: object) -> None:
|
||||
routed.update(kwargs)
|
||||
|
||||
call.__name__ = function_name
|
||||
await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call)
|
||||
return routed
|
||||
|
||||
|
||||
async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None:
|
||||
import time
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
turn: Final = Logging(
|
||||
model="gpt-5-mini",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type=CallTypes.aresponses_websocket.value,
|
||||
start_time=time.time(),
|
||||
litellm_call_id=call_id,
|
||||
function_id=call_id,
|
||||
)
|
||||
turn.update_environment_variables(litellm_params={"litellm_metadata": routed["litellm_metadata"]}, optional_params={})
|
||||
await turn.dispatch_success_handlers(
|
||||
[{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35}}}],
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_budget():
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ws-budget",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5-mini",
|
||||
"api_key": "sk-fake",
|
||||
"max_budget": 0.001,
|
||||
"budget_duration": "1d",
|
||||
},
|
||||
"model_info": {"id": "ws-budget-dep"},
|
||||
}
|
||||
]
|
||||
)
|
||||
assert "turn_admission" not in await _kwargs_routed_to(router, "ws-budget", "aresponses")
|
||||
admit_turn: Final = (await _kwargs_routed_to(router, "ws-budget", "_aresponses_websocket"))["turn_admission"]
|
||||
assert callable(admit_turn)
|
||||
|
||||
await admit_turn()
|
||||
await router.cache.async_set_cache("deployment_spend:ws-budget-dep:1d", 0.002)
|
||||
|
||||
with pytest.raises(litellm.RateLimitError, match="crossed budget"):
|
||||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turn_after_the_first_counts_toward_deployment_rpm():
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ws-rpm",
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "rpm": 1},
|
||||
"model_info": {"id": "ws-rpm-dep"},
|
||||
}
|
||||
],
|
||||
optional_pre_call_checks=["enforce_model_rate_limits"],
|
||||
)
|
||||
admit_turn: Final = (await _kwargs_routed_to(router, "ws-rpm", "_aresponses_websocket"))["turn_admission"]
|
||||
assert callable(admit_turn)
|
||||
|
||||
with pytest.raises(litellm.RateLimitError, match="RPM limit=1"):
|
||||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turn_is_checked_against_limits_changed_after_the_socket_opened():
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ws-rpm-lowered",
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "rpm": 100},
|
||||
"model_info": {"id": "ws-rpm-lowered-dep"},
|
||||
}
|
||||
],
|
||||
optional_pre_call_checks=["enforce_model_rate_limits"],
|
||||
)
|
||||
admit_turn: Final = (await _kwargs_routed_to(router, "ws-rpm-lowered", "_aresponses_websocket"))["turn_admission"]
|
||||
assert callable(admit_turn)
|
||||
await admit_turn()
|
||||
|
||||
router.upsert_deployment(
|
||||
Deployment(
|
||||
model_name="ws-rpm-lowered",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5-mini", api_key="sk-fake", rpm=1),
|
||||
model_info=ModelInfo(id="ws-rpm-lowered-dep"),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.RateLimitError, match="RPM limit=1"):
|
||||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turn_counts_toward_deployment_tpm(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ws-tpm",
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": 30},
|
||||
"model_info": {"id": "ws-tpm-dep"},
|
||||
}
|
||||
],
|
||||
optional_pre_call_checks=["enforce_model_rate_limits"],
|
||||
)
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", list(litellm.callbacks))
|
||||
routed: Final = await _kwargs_routed_to(router, "ws-tpm", "_aresponses_websocket")
|
||||
admit_turn: Final = routed["turn_admission"]
|
||||
assert callable(admit_turn)
|
||||
await admit_turn()
|
||||
|
||||
await _finish_35_token_websocket_turn(routed, "ws-tpm-turn")
|
||||
|
||||
with pytest.raises(litellm.RateLimitError, match="TPM limit=30, current usage=35"):
|
||||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turn_is_refused_by_usage_based_routing_once_deployment_tpm_is_used(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ws-tpm-v2",
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": 30},
|
||||
"model_info": {"id": "ws-tpm-v2-dep"},
|
||||
}
|
||||
],
|
||||
routing_strategy="usage-based-routing-v2",
|
||||
)
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", list(litellm.callbacks))
|
||||
routed: Final = await _kwargs_routed_to(router, "ws-tpm-v2", "_aresponses_websocket")
|
||||
admit_turn: Final = routed["turn_admission"]
|
||||
assert callable(admit_turn)
|
||||
await admit_turn()
|
||||
|
||||
await _finish_35_token_websocket_turn(routed, "ws-tpm-v2-turn")
|
||||
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turns_do_not_reserve_deployment_otpm():
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ws-otpm",
|
||||
"litellm_params": {"model": "openai/ws-otpm-unpriced", "api_key": "sk-fake", "otpm": 5000},
|
||||
"model_info": {"id": "ws-otpm-dep"},
|
||||
}
|
||||
],
|
||||
optional_pre_call_checks=["enforce_model_rate_limits"],
|
||||
)
|
||||
admit_turn: Final = (await _kwargs_routed_to(router, "ws-otpm", "_aresponses_websocket"))["turn_admission"]
|
||||
assert callable(admit_turn)
|
||||
|
||||
for _ in range(3):
|
||||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deployment_callback_on_success_tracks_tpm_for_io_deployment():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -114,10 +114,22 @@ from litellm.utils import (
|
|||
)
|
||||
|
||||
|
||||
def test_get_base_model_from_metadata_returns_unvalidated_root_value():
|
||||
@pytest.mark.parametrize("root_base_model", [42, ["gemini-3.5-flash"], {"model": "gemini-3.5-flash"}])
|
||||
def test_get_base_model_from_metadata_ignores_a_non_string_root_value(root_base_model: object):
|
||||
from litellm.utils import get_base_model_from_metadata
|
||||
|
||||
assert get_base_model_from_metadata({"litellm_params": {"base_model": 42}}) == 42
|
||||
assert get_base_model_from_metadata({"litellm_params": {"base_model": root_base_model}}) is None
|
||||
assert (
|
||||
get_base_model_from_metadata(
|
||||
{
|
||||
"litellm_params": {
|
||||
"base_model": root_base_model,
|
||||
"metadata": {"model_info": {"base_model": "vertex_ai/gemini-3.5-flash"}},
|
||||
}
|
||||
}
|
||||
)
|
||||
== "vertex_ai/gemini-3.5-flash"
|
||||
)
|
||||
|
||||
|
||||
def _reject_short_post_call_response(input: str, model: str | None = None) -> dict[str, object]:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue