From 8d200a4f981a62ff5f4be5a7c63fcedfedd51181 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Sun, 15 Nov 2026 16:58:31 -0800 Subject: [PATCH] feat(messages): serve bedrock invoke messages on rust only (#51330) --- litellm/messages/dispatch.py | 10 +- litellm/proxy/proxy_server.py | 5 +- litellm/rust/Cargo.lock | 64 + litellm/rust/crates/auth-aws/Cargo.toml | 4 +- litellm/rust/crates/auth-aws/src/aws.rs | 1009 ++++++++++++--- .../auth-aws/src/bedrock_session_policy.json | 1 + litellm/rust/crates/auth-aws/src/constants.rs | 5 +- litellm/rust/crates/auth-aws/src/error.rs | 60 + litellm/rust/crates/auth-aws/src/lib.rs | 4 +- litellm/rust/crates/auth-aws/src/region.rs | 239 ++++ litellm/rust/crates/auth-aws/src/signer.rs | 442 ++++++- litellm/rust/crates/auth-types/src/lib.rs | 2 +- .../rust/crates/auth-types/src/params/aws.rs | 17 + .../rust/crates/auth-types/src/params/mod.rs | 2 +- .../crates/auth-types/tests/aws_params.rs | 9 +- .../python_contract.json | 3 +- .../callbacks-legacy-python/src/adapter.rs | 22 +- .../src/test_support.rs | 2 +- .../crates/callbacks-s3-log/src/settings.rs | 2 + .../rust/crates/gateway-inference/Cargo.toml | 1 + .../crates/gateway-inference/src/error.rs | 4 +- .../crates/gateway-inference/src/messages.rs | 2 +- litellm/rust/crates/host-python/AGENTS.md | 6 +- litellm/rust/crates/host-python/src/driver.rs | 50 +- litellm/rust/crates/host-python/src/native.rs | 7 +- .../rust/crates/host-python/tests/driver.rs | 150 ++- .../crates/host-python/tests/support/mod.rs | 101 +- litellm/rust/crates/host/Cargo.toml | 2 +- litellm/rust/crates/host/src/call.rs | 55 +- litellm/rust/crates/host/src/protocol.rs | 39 +- litellm/rust/crates/host/tests/call.rs | 83 +- litellm/rust/crates/http/src/error.rs | 56 +- litellm/rust/crates/http/src/outbound.rs | 51 +- .../crates/inference-chat/tests/request.rs | 48 +- .../rust/crates/inference-messages/Cargo.toml | 1 + .../inference-messages/src/common_utils.rs | 80 +- .../crates/inference-messages/src/handler.rs | 282 ++++- .../rust/crates/inference-messages/src/lib.rs | 1 + .../crates/inference-messages/src/relay.rs | 202 +++ .../crates/inference-messages/src/route.rs | 14 + .../inference-messages/src/safeguards.rs | 2 + .../crates/inference-messages/src/types.rs | 45 +- .../tests/bedrock_native/main.rs | 51 + .../tests/messages/request.rs | 13 +- .../tests/messages/response.rs | 35 +- litellm/rust/crates/inference/src/error.rs | 15 - .../src/formats/messages/streaming.rs | 2 + litellm/rust/crates/llms/Cargo.toml | 2 + .../llms/src/anthropic/beta_gated_fields.rs | 222 +++- .../crates/llms/src/anthropic/beta_headers.rs | 57 +- .../crates/llms/src/anthropic/common_utils.rs | 48 +- .../llms/src/anthropic/messages/handler.rs | 53 +- .../src/anthropic/messages/transformation.rs | 120 +- .../llms/src/anthropic/model_capabilities.rs | 288 ++++- .../src/azure_ai/messages/transformation.rs | 56 +- litellm/rust/crates/llms/src/base_llm/auth.rs | 8 +- .../llms/src/base_llm/messages/context.rs | 92 +- .../llms/src/base_llm/messages/streaming.rs | 17 +- .../src/base_llm/messages/transformation.rs | 31 +- .../src/bedrock/audio_transcription/mod.rs | 8 +- .../crates/llms/src/bedrock/chat/AGENTS.md | 3 +- .../bedrock/chat/converse_transformation.rs | 29 +- .../llms/src/bedrock/chat/invoke_handler.rs | 727 +++++++++-- .../base_invoke_transformation.rs | 186 +++ .../chat/invoke_transformations/mod.rs | 1 + .../rust/crates/llms/src/bedrock/chat/mod.rs | 1 + .../messages/transformation.rs | 12 +- .../crates/llms/src/bedrock/common_utils.rs | 281 ++++- .../bedrock/common_utils/structured_output.rs | 904 ++++++++++++++ .../llms/src/bedrock/messages/AGENTS.md | 3 +- .../anthropic_claude_transformation.rs | 892 ++++++++------ .../messages/invoke_transformations/betas.rs | 702 +++++++++++ .../messages/invoke_transformations/mod.rs | 3 + .../invoke_transformations/request.rs | 1088 +++++++++++++++++ .../messages/invoke_transformations/stream.rs | 443 +++++++ litellm/rust/crates/llms/src/bedrock/mod.rs | 1 + .../llms/src/bedrock/request_metadata.rs | 336 +++++ .../bedrock_mantle/messages/transformation.rs | 3 +- litellm/rust/crates/llms/src/error.rs | 4 + litellm/rust/crates/python-bridge/Cargo.toml | 1 + .../rust/crates/python-bridge/src/errors.rs | 251 +++- litellm/rust/crates/python-bridge/src/http.rs | 15 +- .../python-bridge/src/logger/machine.rs | 8 +- .../crates/python-bridge/src/logger/mod.rs | 65 +- .../crates/python-bridge/src/logger/tests.rs | 107 +- .../python-bridge/src/routes/messages/host.rs | 327 +++-- .../python-bridge/src/routes/messages/mod.rs | 1 + .../src/routes/messages/model_catalog.rs | 62 + .../crates/python-bridge/src/secrets/local.rs | 238 ++++ .../crates/python-bridge/src/secrets/mod.rs | 95 +- .../python-bridge/src/secrets/seeded.rs | 84 +- .../python-bridge/src/secrets/web_identity.rs | 30 + .../router-types/tests/litellm_params.rs | 5 +- litellm/rust/crates/secrets-aws/src/auth.rs | 2 + litellm/rust_bridge/catalog.py | 11 + .../host/callbacks_legacy_python.py | 10 +- .../rust_bridge/host/model_capabilities.py | 21 +- litellm/rust_bridge/host/routes/messages.py | 369 +++++- litellm/rust_bridge/host/settings.py | 4 + litellm/rust_bridge/messages/entrypoints.py | 1 + .../messages/test_bedrock.py | 713 +++++++++++ .../messages/test_dispatch.py | 89 +- .../test_anthropic_cache_control_hook.py | 3 + .../native/test_anthropic_messages_handler.py | 6 +- .../chat/test_bedrock_converse_handler.py | 3 + .../test_anthropic_claude_transformation.py | 7 + .../custom_httpx/test_llm_http_handler.py | 2 +- .../rust_bridge/host/routes/test_messages.py | 414 ++++++- .../host/test_callbacks_legacy_python.py | 19 +- .../host/test_model_capabilities.py | 49 +- tests/unit/rust_bridge/host/test_settings.py | 4 + .../unit/rust_bridge/messages/test_secrets.py | 82 +- tests/unit/rust_bridge/test_catalog.py | 36 +- tests/unit/test_router/test_router_timeout.py | 116 +- 114 files changed, 11703 insertions(+), 1468 deletions(-) create mode 100644 litellm/rust/crates/auth-aws/src/bedrock_session_policy.json create mode 100644 litellm/rust/crates/auth-aws/src/region.rs create mode 100644 litellm/rust/crates/inference-messages/src/relay.rs create mode 100644 litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/base_invoke_transformation.rs create mode 100644 litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/mod.rs create mode 100644 litellm/rust/crates/llms/src/bedrock/common_utils/structured_output.rs create mode 100644 litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/betas.rs create mode 100644 litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/request.rs create mode 100644 litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/stream.rs create mode 100644 litellm/rust/crates/llms/src/bedrock/request_metadata.rs create mode 100644 litellm/rust/crates/python-bridge/src/routes/messages/model_catalog.rs create mode 100644 litellm/rust/crates/python-bridge/src/secrets/local.rs create mode 100644 litellm/rust/crates/python-bridge/src/secrets/web_identity.rs create mode 100644 tests/test_litellm_rust/messages/test_bedrock.py diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 333c4deca69..0ba1311c567 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -6,6 +6,9 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele from litellm.exceptions import BadRequestError, UnsupportedParamsError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.anthropic.messages.native import handler as main +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( + AmazonAnthropicClaudeMessagesConfig, +) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.rust_bridge import catalog, detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target @@ -105,8 +108,11 @@ def _route_target( api_base: str | None, compacting: bool, ) -> Target: - if main._provider_messages_config(model, provider, api_base) is not None: # pyright: ignore[reportPrivateUsage] # the handler's own config lookup - return Target.MESSAGES + provider_config: Final = main._provider_messages_config(model, provider, api_base) # pyright: ignore[reportPrivateUsage] # the handler's own config lookup + if provider_config is not None: + return ( + Target.BEDROCK_INVOKE if type(provider_config) is AmazonAnthropicClaudeMessagesConfig else Target.MESSAGES + ) if main._deployment_passes_through_anthropic_messages(model_info): # pyright: ignore[reportPrivateUsage] # the handler's own passthrough opt-in return Target.MESSAGES_PASSTHROUGH if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bb71779fd70..cf0306ab005 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -213,7 +213,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( snapshot_tuning_baselines, tuning_limit_violation, ) -from litellm.rust_bridge.catalog import Target +from litellm.rust_bridge.catalog import NATIVE_MESSAGES_TARGETS, Target from litellm.types.utils import ( PRICING_OVERRIDES_KEY, CallTypes, @@ -16130,7 +16130,6 @@ def _count_context_management( _GATEWAY_EDIT_ROUTES: Final = frozenset({Target.CHAT_COMPLETIONS, Target.RESPONSES}) -_MESSAGES_ROUTES: Final = frozenset({Target.MESSAGES, Target.MESSAGES_PASSTHROUGH}) def _messages_route(deployment: _TokenCountDeployment, request_model: str) -> Target: @@ -16669,7 +16668,7 @@ async def _count_request_tokens( local_view: Final = ( None - if route not in _MESSAGES_ROUTES or context_edits is None or request_messages is None + if route not in NATIVE_MESSAGES_TARGETS or context_edits is None or request_messages is None else await _bridged_request_view( request_messages, system, diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock index b12c184a9e7..dc09a8c06a0 100644 --- a/litellm/rust/Cargo.lock +++ b/litellm/rust/Cargo.lock @@ -258,6 +258,8 @@ checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319" dependencies = [ "aws-credential-types", "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", @@ -269,11 +271,14 @@ dependencies = [ "aws-types", "bytes", "fastrand", + "hex", "http 1.4.2", + "sha1 0.10.7", "time", "tokio", "tracing", "url", + "zeroize", ] [[package]] @@ -428,6 +433,58 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-sso" +version = "1.111.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.113.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sts" version = "1.108.0" @@ -3455,6 +3512,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", + "chrono", "futures-util", "litellm-auth-types", "litellm-error", @@ -3467,6 +3525,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "tempfile", "thiserror 2.0.19", "tokio", "url", @@ -4004,6 +4063,7 @@ dependencies = [ "litellm-auth", "litellm-cache-memory", "litellm-cache-response", + "litellm-core-utils", "litellm-error", "litellm-gateway-auth", "litellm-host", @@ -4269,6 +4329,7 @@ dependencies = [ "litellm-inference-testing", "litellm-llms", "litellm-llms-types", + "litellm-python-compat", "litellm-router-types", "litellm-secret-value", "litellm-secrets", @@ -4391,6 +4452,7 @@ dependencies = [ "aws-smithy-types", "base64 0.22.1", "bytes", + "crc32fast", "data-url", "fancy-regex 0.19.2", "futures-util", @@ -4420,6 +4482,7 @@ dependencies = [ "thiserror 2.0.19", "time", "tokio", + "tokio-util", "url", ] @@ -4488,6 +4551,7 @@ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ "async-trait", + "aws-credential-types", "aws-sdk-secretsmanager", "azure_core", "bytes", diff --git a/litellm/rust/crates/auth-aws/Cargo.toml b/litellm/rust/crates/auth-aws/Cargo.toml index 1a7392afc52..cac0d62e5c1 100644 --- a/litellm/rust/crates/auth-aws/Cargo.toml +++ b/litellm/rust/crates/auth-aws/Cargo.toml @@ -16,12 +16,13 @@ litellm-secret-value.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } sha2.workspace = true +chrono.workspace = true thiserror.workspace = true tokio.workspace = true url.workspace = true veil.workspace = true -aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] } +aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process", "sso"] } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] } aws-sigv4 = "1.5.1" @@ -33,6 +34,7 @@ litellm-secrets.workspace = true aws-smithy-types = "1.6.4" aws-runtime = { version = "1.9.4", default-features = false } rstest.workspace = true +tempfile = "3" litellm-http = { workspace = true, features = ["test-support"] } bytes.workspace = true futures-util.workspace = true diff --git a/litellm/rust/crates/auth-aws/src/aws.rs b/litellm/rust/crates/auth-aws/src/aws.rs index a94a4e8d66b..c088ebc4546 100644 --- a/litellm/rust/crates/auth-aws/src/aws.rs +++ b/litellm/rust/crates/auth-aws/src/aws.rs @@ -2,6 +2,7 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use aws_credential_types::provider::{SharedCredentialsProvider, error::CredentialsError}; +use aws_smithy_runtime_api::client::http::SharedHttpClient; use litellm_secret_value::SecretValue; use moka::{Expiry, future::Cache}; use std::{future::Future, pin::Pin, sync::Arc}; @@ -16,19 +17,28 @@ use aws_sigv4::sign::v4; use aws_smithy_runtime_api::client::identity::Identity; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; -use litellm_auth_types::AwsParams; +use litellm_auth_types::{AwsParams, AwsSessionTag, ParamSpec}; use super::constants::{ - AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_ROLE_ARN, - AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN, - AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE, - DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES, + AWS_ACCESS_KEY_ID, AWS_DEFAULT_REGION, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION, + AWS_REGION_NAME, AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, + AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, + AWS_WEB_IDENTITY_TOKEN_FILE, BEDROCK_SESSION_POLICY, DEFAULT_BEDROCK_REGION, + DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES, WEB_IDENTITY_SESSION_SECONDS, }; use crate::error::{CredentialFlow, Error, SigningStage}; +use crate::region::{session_region, sts_endpoint_region}; const CREDENTIAL_CACHE_TTL: Duration = Duration::from_secs(600); const EXPIRY_BUFFER: Duration = Duration::from_secs(60); +pub type WebIdentityTokenFuture<'a> = + Pin, litellm_error::Source>> + Send + 'a>>; + +pub trait WebIdentityTokenSource: Send + Sync + std::fmt::Debug { + fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a>; +} + #[derive(Clone)] pub struct AwsAuthService { providers: Cache, @@ -38,10 +48,7 @@ pub struct AwsAuthService { impl Default for AwsAuthService { fn default() -> Self { - Self::new(Arc::new(NativeProviderLoader { - token: None, - policy: None, - })) + Self::new(Arc::new(NativeProviderLoader::default())) } } @@ -59,13 +66,21 @@ impl AwsAuthService { } } + pub fn with_web_identity_tokens(tokens: Arc) -> Self { + Self::new(Arc::new(NativeProviderLoader { + tokens: Some(tokens), + ..NativeProviderLoader::default() + })) + } + pub fn for_web_identity( resolver: WebIdentityTokenResolver, session_policy: Option, ) -> Self { Self::new(Arc::new(NativeProviderLoader { - token: Some(resolver), - policy: session_policy, + tokens: Some(Arc::new(ResolverWebIdentityTokens(resolver))), + session_policy, + ..NativeProviderLoader::default() })) } } @@ -96,19 +111,99 @@ trait ProviderLoader: Send + Sync { fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_>; } +#[derive(Clone, Default)] struct NativeProviderLoader { - token: Option, - policy: Option, + tokens: Option>, + session_policy: Option, + http_client: Option, + ambient: Option, +} + +impl std::fmt::Debug for NativeProviderLoader { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeProviderLoader") + .finish_non_exhaustive() + } +} + +struct ResolverWebIdentityTokens(WebIdentityTokenResolver); + +impl std::fmt::Debug for ResolverWebIdentityTokens { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ResolverWebIdentityTokens") + .finish_non_exhaustive() + } +} + +impl WebIdentityTokenSource for ResolverWebIdentityTokens { + fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a> { + let resolve = Arc::clone(&self.0); + let reference = reference.to_owned(); + Box::pin(async move { + tokio::task::spawn_blocking(move || resolve(&reference)) + .await + .map_err(litellm_error::Source::new)? + .map(|token| Some(SecretValue::new(token))) + .map_err(|message| litellm_error::Source::new(std::io::Error::other(message))) + }) + } } impl ProviderLoader for NativeProviderLoader { fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> { - Box::pin(build_provider( - config, - flow, - self.token.clone(), - self.policy.clone(), - )) + Box::pin(build_provider(config, flow, self)) + } +} + +impl NativeProviderLoader { + async fn ambient(&self) -> SharedCredentialsProvider { + match &self.ambient { + Some(provider) => provider.clone(), + None => SharedCredentialsProvider::new( + aws_config::default_provider::credentials::DefaultCredentialsChain::builder() + .build() + .await, + ), + } + } + + async fn sts_config( + &self, + config: &AwsAuthConfig, + region: Option, + ) -> aws_types::SdkConfig { + let loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); + let loader = match region { + Some(region) => loader.region(aws_types::region::Region::new(region)), + None => loader, + }; + let loader = match config.sts_endpoint.clone() { + Some(endpoint) => loader.endpoint_url(endpoint), + None => loader, + }; + let loader = match self.http_client.clone() { + Some(client) => loader.http_client(client), + None => loader, + }; + let loader = match (&config.access_key_id, &config.secret_access_key) { + (Some(key), Some(secret)) => loader.credentials_provider(Credentials::new( + key.expose(), + secret.expose(), + config + .session_token + .as_ref() + .map(|token| token.expose().to_owned()), + None, + "litellm-role-source", + )), + _ => match &self.ambient { + Some(ambient) => loader.credentials_provider(ambient.clone()), + None => loader, + }, + }; + loader.load().await } } @@ -124,6 +219,8 @@ pub struct AwsAuthConfig { pub web_identity_token: Option, pub sts_endpoint: Option, pub external_id: Option, + pub session_tags: Option>, + pub session_policy: Option<&'static str>, } impl AwsAuthConfig { @@ -149,8 +246,20 @@ impl AwsAuthConfig { external_id: self .external_id .or_else(|| env_lookup(AWS_EXTERNAL_ID).map(SecretValue::new)), + session_tags: self.session_tags, + session_policy: self.session_policy, } } + + fn sts_region(&self, env_lookup: &(dyn Fn(&str) -> Option + Sync)) -> Option { + let present = |value: Option| value.filter(|value| !value.is_empty()); + self.sts_endpoint + .as_deref() + .and_then(sts_endpoint_region) + .or_else(|| present(env_lookup(AWS_REGION))) + .or_else(|| present(env_lookup(AWS_DEFAULT_REGION))) + .or_else(|| present(self.region_name.clone())) + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -159,11 +268,13 @@ pub enum AwsAuthFlow { token: SecretValue, role: String, session_name: String, + sts_region: Option, }, AssumeRole { role: String, session_name: Option, web_identity_role: Option, + sts_region: Option, }, Profile { name: String, @@ -213,6 +324,7 @@ pub fn classify_auth( .filter(|token_file| !token_file.is_empty()) .and_then(|_| env_lookup(AWS_ROLE_ARN)); let config = config.with_environment(env_lookup); + let sts_region = config.sts_region(env_lookup); if let (Some(token), Some(role), Some(session_name)) = ( config.web_identity_token.clone(), config.role_name.clone(), @@ -222,6 +334,7 @@ pub fn classify_auth( token, role, session_name, + sts_region, }; } if let Some(role) = config.role_name.clone() { @@ -229,6 +342,7 @@ pub fn classify_auth( role, session_name: config.session_name.clone(), web_identity_role, + sts_region, }; } if let Some(name) = config.profile_name { @@ -291,6 +405,11 @@ impl AwsAuthService { ) -> Result { let resolved = config.clone().with_environment(env_lookup); let flow = classify_auth(config, env_lookup); + if let AwsAuthFlow::WebIdentity { token, .. } = &flow + && names_the_environment(token.expose(), env_lookup) + { + return Err(Error::WebIdentityTokenReference); + } self.fresh_credentials(&resolved, &flow).await } @@ -358,6 +477,13 @@ impl AwsAuthService { } } +fn names_the_environment( + reference: &str, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> bool { + reference.starts_with("os.environ/") || env_lookup(reference).is_some() +} + fn fresh(credentials: &Credentials) -> bool { credentials .expiry() @@ -380,8 +506,7 @@ impl AwsAuthFlow { async fn build_provider( resolved: AwsAuthConfig, flow: AwsAuthFlow, - token_resolver: Option, - policy: Option, + loader: &NativeProviderLoader, ) -> Result { match flow { AwsAuthFlow::SessionToken { @@ -428,70 +553,145 @@ async fn build_provider( role, session_name, web_identity_role, + sts_region, } => { - if is_already_running_as_role(&role, web_identity_role.as_deref(), &resolved).await? { - return Ok(SharedCredentialsProvider::new( - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await, - )); + let running_as = RunningAs { + role: role.clone(), + config: resolved.clone(), + sts_region: sts_region.clone(), + loader: loader.clone(), + }; + if running_as.is_target(web_identity_role.as_deref()).await { + return Ok(loader.ambient().await); } - let sdk_config = sdk_config(resolved.clone()).await; + let sdk_config = loader.sts_config(&resolved, sts_region).await; let builder = aws_config::sts::AssumeRoleProvider::builder(role) .session_name(session_name.unwrap_or_else(default_session_name)); let builder = match resolved.external_id { Some(id) => builder.external_id(id.expose()), None => builder, }; - Ok(SharedCredentialsProvider::new( - builder.configure(&sdk_config).build().await, - )) + let builder = match resolved.session_tags { + Some(tags) => builder.tags(tags.into_iter().map(|tag| (tag.key, tag.value))), + None => builder, + }; + Ok(SharedCredentialsProvider::new(AssumedRole { + provider: builder.configure(&sdk_config).build().await, + fallback: web_identity_role.is_none().then_some(running_as), + })) } AwsAuthFlow::WebIdentity { token, role, session_name, + sts_region, } => Ok(SharedCredentialsProvider::new(WebIdentityProvider { - policy, - client: aws_sdk_sts::Client::new(&sdk_config(resolved).await), + client: aws_sdk_sts::Client::new(&loader.sts_config(&resolved, sts_region).await), + tokens: loader.tokens.clone(), token, - token_resolver, role, session_name, + policy: loader + .session_policy + .clone() + .or_else(|| resolved.session_policy.map(str::to_owned)), })), } } -async fn sdk_config(config: AwsAuthConfig) -> aws_types::SdkConfig { - let loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - let loader = match config.region_name { - Some(region) => loader.region(aws_types::region::Region::new(region)), - None => loader, - }; - let loader = match config.sts_endpoint { - Some(endpoint) => loader.endpoint_url(endpoint), - None => loader, - }; - let loader = match (config.access_key_id, config.secret_access_key) { - (Some(key), Some(secret)) => loader.credentials_provider(Credentials::new( - key.expose(), - secret.expose(), - config.session_token.map(|token| token.expose().to_owned()), - None, - "litellm-role-source", - )), - _ => loader, - }; - loader.load().await +#[derive(Debug)] +struct RunningAs { + role: String, + config: AwsAuthConfig, + sts_region: Option, + loader: NativeProviderLoader, +} + +impl RunningAs { + async fn is_target(&self, web_identity_role: Option<&str>) -> bool { + if self.config.access_key_id.is_some() || self.config.secret_access_key.is_some() { + return false; + } + if role_identity(&self.role).is_none() { + return false; + } + if let Some(current_role) = web_identity_role { + return same_role_arns(&self.role, current_role); + } + let caller = AwsAuthConfig { + sts_endpoint: self.config.sts_endpoint.clone(), + ..Default::default() + }; + let sdk_config = self + .loader + .sts_config(&caller, self.sts_region.clone()) + .await; + aws_sdk_sts::Client::new(&sdk_config) + .get_caller_identity() + .send() + .await + .ok() + .and_then(|response| response.arn) + .is_some_and(|caller| same_role_arns(&self.role, &caller)) + } +} + +#[derive(Debug)] +struct AssumedRole { + provider: aws_config::sts::AssumeRoleProvider, + fallback: Option, +} + +impl ProvideCredentials for AssumedRole { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::new(async { + let error = match self.provider.provide_credentials().await { + Ok(credentials) => return Ok(credentials), + Err(error) => error, + }; + match &self.fallback { + Some(running_as) + if format!("{error:?}").contains("AccessDenied") + && running_as.is_target(None).await => + { + running_as + .loader + .ambient() + .await + .provide_credentials() + .await + } + _ => Err(error), + } + }) + } } struct WebIdentityProvider { client: aws_sdk_sts::Client, + tokens: Option>, token: SecretValue, - token_resolver: Option, - policy: Option, role: String, session_name: String, + policy: Option, +} + +impl WebIdentityProvider { + async fn token(&self) -> Result { + let Some(tokens) = &self.tokens else { + return Ok(self.token.clone()); + }; + tokens + .web_identity_token(self.token.expose()) + .await + .map_err(Error::WebIdentityToken)? + .ok_or(Error::MissingWebIdentityToken) + } } impl std::fmt::Debug for WebIdentityProvider { @@ -514,31 +714,18 @@ impl ProvideCredentials for WebIdentityProvider { Self: 'a, { aws_credential_types::provider::future::ProvideCredentials::new(async { - let token = match &self.token_resolver { - Some(resolve) => { - let resolve = Arc::clone(resolve); - let reference = self.token.expose().to_owned(); - tokio::task::spawn_blocking(move || resolve(&reference)) - .await - .map_err(|error| error.to_string()) - .and_then(|token| token) - .map_err(|message| { - CredentialsError::provider_error(std::io::Error::other(message)) - })? - } - None => self.token.expose().to_owned(), - }; - let request = self + let token = self + .token() + .await + .map_err(CredentialsError::provider_error)?; + let response = self .client .assume_role_with_web_identity() .role_arn(&self.role) .role_session_name(&self.session_name) - .web_identity_token(token); - let request = match &self.policy { - Some(policy) => request.policy(policy).duration_seconds(3600), - None => request, - }; - let response = request + .web_identity_token(token.expose()) + .duration_seconds(WEB_IDENTITY_SESSION_SECONDS) + .set_policy(self.policy.clone()) .send() .await .map_err(CredentialsError::provider_error)?; @@ -558,42 +745,6 @@ impl ProvideCredentials for WebIdentityProvider { } } -async fn is_already_running_as_role( - role: &str, - web_identity_role: Option<&str>, - config: &AwsAuthConfig, -) -> Result { - if config.access_key_id.is_some() || config.secret_access_key.is_some() { - return Ok(false); - } - if role_identity(role).is_none() { - return Ok(false); - } - if let Some(current_role) = web_identity_role { - return Ok(same_role_arns(role, current_role)); - } - - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = config.region_name.clone() { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = config.sts_endpoint.clone() { - loader = loader.endpoint_url(endpoint); - } - let sdk_config = loader.load().await; - let response = match aws_sdk_sts::Client::new(&sdk_config) - .get_caller_identity() - .send() - .await - { - Ok(response) => response, - Err(_) => return Ok(false), - }; - Ok(response - .arn() - .is_some_and(|caller| same_role_arns(role, caller))) -} - fn default_session_name() -> String { let seconds = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -814,75 +965,141 @@ pub fn resolve_bedrock_region( model_region: Option<&str>, params: &AwsParams, env_lookup: &dyn Fn(&str) -> Option, +) -> String { + bedrock_region(model_region, params, env_lookup, &|name| { + std::env::var(name).ok() + }) +} + +fn bedrock_region( + model_region: Option<&str>, + params: &AwsParams, + env_lookup: &dyn Fn(&str) -> Option, + process_env: &dyn Fn(&str) -> Option, ) -> String { resolve_aws_region(model_region, params, env_lookup) + .or_else(|| session_region(env_lookup, process_env)) .unwrap_or_else(|| DEFAULT_BEDROCK_REGION.to_string()) } +fn configured( + params: &AwsParams, + spec: &ParamSpec, + env_lookup: &dyn Fn(&str) -> Option, +) -> Option { + spec.wire + .iter() + .find_map(|name| params.get(name)) + .or_else(|| spec.env.iter().find_map(|name| env_lookup(name))) +} + pub fn aws_auth_config( params: &AwsParams, env_lookup: &dyn Fn(&str) -> Option, ) -> AwsAuthConfig { + let secret = |spec: &ParamSpec| configured(params, spec, env_lookup).map(SecretValue::new); + let setting = |spec: &ParamSpec| configured(params, spec, env_lookup); AwsAuthConfig { - access_key_id: params - .resolve(&AwsParams::ACCESS_KEY_ID, env_lookup) - .map(SecretValue::new), - secret_access_key: params - .resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup) - .map(SecretValue::new), - session_token: params - .resolve(&AwsParams::SESSION_TOKEN, env_lookup) - .map(SecretValue::new), - region_name: params.resolve(&AwsParams::REGION, env_lookup), - session_name: params.resolve(&AwsParams::SESSION_NAME, env_lookup), - profile_name: params.resolve(&AwsParams::PROFILE_NAME, env_lookup), - role_name: params.resolve(&AwsParams::ROLE_NAME, env_lookup), - web_identity_token: params - .resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup) - .map(SecretValue::new), - sts_endpoint: params.resolve(&AwsParams::STS_ENDPOINT, env_lookup), - external_id: params - .resolve(&AwsParams::EXTERNAL_ID, env_lookup) - .map(SecretValue::new), + access_key_id: secret(&AwsParams::ACCESS_KEY_ID), + secret_access_key: secret(&AwsParams::SECRET_ACCESS_KEY), + session_token: secret(&AwsParams::SESSION_TOKEN), + region_name: setting(&AwsParams::REGION), + session_name: setting(&AwsParams::SESSION_NAME), + profile_name: setting(&AwsParams::PROFILE_NAME), + role_name: setting(&AwsParams::ROLE_NAME), + web_identity_token: secret(&AwsParams::WEB_IDENTITY_TOKEN), + sts_endpoint: setting(&AwsParams::STS_ENDPOINT), + external_id: secret(&AwsParams::EXTERNAL_ID), + session_tags: params.aws_session_tags.clone().map(|mut tags| { + tags.sort_by(|left, right| left.key.cmp(&right.key)); + tags + }), + session_policy: None, } } +pub fn web_identity_session_policy(service: &str) -> Option<&'static str> { + matches!( + service, + "bedrock" | "bedrock-agentcore" | "bedrock-mantle" | "aws-external-anthropic" + ) + .then_some(BEDROCK_SESSION_POLICY) +} + /// Where the credentials that sign a request come from, decided when the request is /// prepared and resolved when it is sent. #[derive(Clone, Debug, PartialEq)] pub enum AwsCredentialSource { HostSupplied(Credentials), - Chain(AwsAuthConfig), + Chain(Box), } impl AwsCredentialSource { pub fn from_params(params: &AwsParams, env_lookup: &dyn Fn(&str) -> Option) -> Self { match host_supplied_credentials(params) { - Some(credentials) => Self::HostSupplied(credentials), - None => Self::Chain(aws_auth_config(params, env_lookup)), + Some(credentials) if names_only_static_keys(params) => Self::HostSupplied(credentials), + _ => Self::Chain(Box::new(aws_auth_config(params, env_lookup))), } } pub async fn resolve( self, auth: &AwsAuthService, + region: &str, + service: &str, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result { - match self { - Self::HostSupplied(credentials) => Ok(credentials), - Self::Chain(config) => auth.resolve_credentials(config, env_lookup).await, - } + let config = match self { + Self::HostSupplied(credentials) => AwsAuthConfig { + access_key_id: Some(SecretValue::new(credentials.access_key_id())), + secret_access_key: Some(SecretValue::new(credentials.secret_access_key())), + session_token: credentials.session_token().map(SecretValue::new), + ..AwsAuthConfig::default() + }, + Self::Chain(config) => *config, + }; + auth.resolve_credentials( + AwsAuthConfig { + region_name: Some(region.to_owned()), + session_policy: config + .session_policy + .or_else(|| web_identity_session_policy(service)), + ..config + }, + env_lookup, + ) + .await } } -/// Credentials a host resolved through its own chain and handed down verbatim. -/// -/// A host with its own resolution (LiteLLM's Python `BaseAWSLLM`, which reads -/// profiles, STS and boto sessions) passes the result here so the core signs -/// with exactly those. Without this the core would re-derive from ambient -/// state, where an unrelated `AWS_ROLE_NAME` or `AWS_PROFILE_NAME` in the -/// environment outranks explicit keys in [`classify_auth`] and the two sides -/// would sign as different principals. +fn names_only_static_keys(params: &AwsParams) -> bool { + let AwsParams { + aws_session_token, + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + aws_sts_endpoint, + aws_external_id, + aws_session_tags, + .. + } = params; + aws_session_token + .as_deref() + .is_none_or(|token| !token.trim().is_empty()) + && [ + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + aws_sts_endpoint, + aws_external_id, + ] + .iter() + .all(|value| value.is_none()) + && aws_session_tags.is_none() +} + pub fn host_supplied_credentials(params: &AwsParams) -> Option { let value = |value: &Option| { value @@ -1023,16 +1240,17 @@ mod tests { #[case] secret_key: Option<&str>, ) { let role = "arn:aws:iam::123456789012:role/target"; - let config = AwsAuthConfig { - access_key_id: access_key.map(SecretValue::new), - secret_access_key: secret_key.map(SecretValue::new), - ..Default::default() + let running_as = RunningAs { + role: role.to_string(), + config: AwsAuthConfig { + access_key_id: access_key.map(SecretValue::new), + secret_access_key: secret_key.map(SecretValue::new), + ..Default::default() + }, + sts_region: None, + loader: NativeProviderLoader::default(), }; - assert!( - !is_already_running_as_role(role, Some(role), &config) - .await - .unwrap() - ); + assert!(!running_as.is_target(Some(role)).await); } #[test] @@ -1045,7 +1263,7 @@ mod tests { recorded.lock().unwrap().insert(name.to_string()); None }; - resolve_aws_region(None, &AwsParams::default(), &env); + bedrock_region(None, &AwsParams::default(), &env, &|_| None); aws_auth_config(&AwsParams::default(), &env); assert!( seen.lock() @@ -1057,7 +1275,8 @@ mod tests { #[rstest::rstest] #[case::param_wins(Some("from-params"), &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-params"))] - #[case::blank_param_falls_to_environment(Some(" "), &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-env"))] + #[case::blank_param_counts_as_set(Some(" "), &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some(" "))] + #[case::blank_environment_counts_as_set(None, &[(AWS_SECRET_ACCESS_KEY, "")], Some(""))] #[case::absent_param_falls_to_environment(None, &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-env"))] #[case::nothing(None, &[], None)] fn auth_config_reads_each_spec_from_params_then_the_environment( @@ -1096,7 +1315,7 @@ mod tests { #[case::call_params(Some("us-east-2"), Some("eu-west-1"), &[(AWS_REGION_NAME, "ap-south-1")], Some("eu-west-1"))] #[case::model_region(Some("us-east-2"), None, &[(AWS_REGION_NAME, "ap-south-1")], Some("us-east-2"))] #[case::aws_region_name(None, None, &[(AWS_REGION_NAME, "ap-south-1")], Some("ap-south-1"))] - #[case::aws_region(None, None, &[(AWS_REGION, "sa-east-1")], Some("sa-east-1"))] + #[case::aws_region(None, None, &[(AWS_REGION, "sa-east-1"), (AWS_DEFAULT_REGION, "eu-north-1")], Some("sa-east-1"))] #[case::nothing(None, None, &[], None)] fn a_region_comes_from_the_call_then_the_model_then_the_environment( #[case] model_region: Option<&str>, @@ -1120,11 +1339,32 @@ mod tests { expected ); assert_eq!( - resolve_bedrock_region(model_region, ¶ms, &env), + bedrock_region(model_region, ¶ms, &env, &|_| None), expected.unwrap_or(DEFAULT_BEDROCK_REGION) ); } + #[rstest::rstest] + #[case::default_region(&[(AWS_DEFAULT_REGION, "eu-north-1")], "eu-north-1")] + #[case::blank_default_region(&[(AWS_DEFAULT_REGION, "")], DEFAULT_BEDROCK_REGION)] + #[case::aws_region_outranks_it(&[(AWS_DEFAULT_REGION, "eu-north-1"), (AWS_REGION, "sa-east-1")], "sa-east-1")] + fn bedrock_falls_back_to_the_boto3_session_region( + #[case] environment: &[(&str, &str)], + #[case] expected: &str, + ) { + let env = |key: &str| { + environment + .iter() + .find(|(name, _)| *name == key) + .map(|(_, value)| value.to_string()) + }; + + assert_eq!( + bedrock_region(None, &AwsParams::default(), &env, &|_| None), + expected + ); + } + fn parity_inputs() -> (String, Vec, HeaderMap) { ( "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke" @@ -1490,7 +1730,7 @@ mod tests { #[rstest::rstest] #[case::session(AwsAuthFlow::SessionToken { access_key_id: "credential-value".into(), secret_access_key: "credential-value".into(), session_token: "credential-value".into() })] #[case::static_keys(AwsAuthFlow::StaticKeys { access_key_id: "credential-value".into(), secret_access_key: "credential-value".into(), region_name: "region".into() })] - #[case::web_identity(AwsAuthFlow::WebIdentity { token: "credential-value".into(), role: "role".into(), session_name: "session".into() })] + #[case::web_identity(AwsAuthFlow::WebIdentity { token: "credential-value".into(), role: "role".into(), session_name: "session".into(), sts_region: None })] fn auth_flow_debug_never_discloses_credentials(#[case] flow: AwsAuthFlow) { let output = format!("{flow:?}"); assert!(!output.contains("credential-value")); @@ -1673,6 +1913,449 @@ mod tests { assert_eq!(report.matches("profile not found").count(), 1, "{report}"); } + #[derive(Clone, Debug, Default)] + struct Sts { + requests: Arc>>, + deny_assume_role: bool, + caller_arns: Arc>>>, + } + + #[derive(Clone, Debug)] + struct StsRequest { + host: String, + authorization: Option, + form: std::collections::BTreeMap, + } + + impl StsRequest { + fn action(&self) -> &str { + self.form["Action"].as_str() + } + } + + const STS_XMLNS: &str = "https://sts.amazonaws.com/doc/2011-06-15/"; + + impl Sts { + fn response(&self, request: &StsRequest) -> (u16, String) { + let credentials = |prefix: &str| { + format!( + "{prefix}-access{prefix}-secret{prefix}-session2100-01-01T00:00:00Zarn:aws:sts::123456789012:assumed-role/{prefix}/sid:s" + ) + }; + let error = |code: &str| { + format!( + "Sender{code}deniedr" + ) + }; + match request.action() { + "AssumeRole" if self.deny_assume_role => (403, error("AccessDenied")), + "AssumeRole" => ( + 200, + format!( + "{}", + credentials("role") + ), + ), + "AssumeRoleWithWebIdentity" => ( + 200, + format!( + "{}", + credentials("web") + ), + ), + "GetCallerIdentity" => match self.caller_arns.lock().unwrap().remove(0) { + Some(arn) => ( + 200, + format!( + "{arn}u123456789012" + ), + ), + None => (400, error("InvalidClientTokenId")), + }, + action => panic!("unexpected STS action {action}"), + } + } + + fn sent(&self) -> Vec { + self.requests.lock().unwrap().clone() + } + } + + impl aws_smithy_runtime_api::client::http::HttpConnector for Sts { + fn call( + &self, + request: aws_smithy_runtime_api::client::orchestrator::HttpRequest, + ) -> aws_smithy_runtime_api::client::http::HttpConnectorFuture { + let recorded = StsRequest { + host: url::Url::parse(request.uri()) + .unwrap() + .host_str() + .unwrap() + .to_string(), + authorization: request.headers().get("authorization").map(str::to_string), + form: url::form_urlencoded::parse(request.body().bytes().unwrap_or_default()) + .into_owned() + .collect(), + }; + let (status, body) = self.response(&recorded); + self.requests.lock().unwrap().push(recorded); + aws_smithy_runtime_api::client::http::HttpConnectorFuture::new(async move { + Ok( + aws_smithy_runtime_api::client::orchestrator::HttpResponse::new( + status.try_into().unwrap(), + aws_smithy_types::body::SdkBody::from(body), + ), + ) + }) + } + } + + #[derive(Debug)] + struct Tokens(Option<&'static str>); + + impl WebIdentityTokenSource for Tokens { + fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a> { + Box::pin(async move { + assert_eq!(reference, "oidc/file/var/run/secrets/token"); + Ok(self.0.map(SecretValue::new)) + }) + } + } + + fn sts_service(sts: &Sts, tokens: Option>) -> AwsAuthService { + use aws_smithy_runtime_api::client::http::{SharedHttpConnector, http_client_fn}; + let connector = sts.clone(); + AwsAuthService::new(Arc::new(NativeProviderLoader { + tokens, + session_policy: None, + http_client: Some(SharedHttpClient::new(http_client_fn(move |_, _| { + SharedHttpConnector::new(connector.clone()) + }))), + ambient: Some(SharedCredentialsProvider::new(Credentials::new( + "ambient-access", + "ambient-secret", + None, + None, + "ambient", + ))), + })) + } + + fn lookup( + environment: &'static [(&'static str, &'static str)], + ) -> impl Fn(&str) -> Option + Sync { + |key: &str| { + environment + .iter() + .find(|(name, _)| *name == key) + .map(|(_, value)| value.to_string()) + } + } + + const TARGET_ROLE: &str = "arn:aws:iam::123456789012:role/target"; + + #[rstest::rstest] + #[case::regional_endpoint(Some("https://sts.eu-west-1.amazonaws.com"), &[(AWS_REGION, "ap-south-1")], "sts.eu-west-1.amazonaws.com", "eu-west-1")] + #[case::aws_region(None, &[(AWS_REGION, "ap-south-1"), (AWS_DEFAULT_REGION, "ca-central-1")], "sts.ap-south-1.amazonaws.com", "ap-south-1")] + #[case::aws_default_region(None, &[(AWS_DEFAULT_REGION, "ca-central-1")], "sts.ca-central-1.amazonaws.com", "ca-central-1")] + #[case::the_signing_region(None, &[], "sts.us-east-1.amazonaws.com", "us-east-1")] + #[case::custom_endpoint_signs_for_the_signing_region(Some("https://sts.internal.test"), &[], "sts.internal.test", "us-east-1")] + #[tokio::test] + async fn params_keys_assume_the_role_with_tags_in_the_python_sts_region( + #[case] sts_endpoint: Option<&str>, + #[case] environment: &'static [(&'static str, &'static str)], + #[case] host: &str, + #[case] sts_region: &str, + ) { + let sts = Sts::default(); + let params = AwsParams { + aws_access_key_id: Some("source-access".into()), + aws_secret_access_key: Some("source-secret".into()), + aws_role_name: Some(TARGET_ROLE.into()), + aws_session_name: Some("attributed".into()), + aws_sts_endpoint: sts_endpoint.map(str::to_string), + aws_session_tags: Some(vec![ + AwsSessionTag { + key: "team".into(), + value: "genai".into(), + }, + AwsSessionTag { + key: "cost".into(), + value: "42".into(), + }, + ]), + ..AwsParams::default() + }; + + let credentials = AwsCredentialSource::from_params(¶ms, &no_env) + .resolve( + &sts_service(&sts, None), + "us-east-1", + BEDROCK_SERVICE, + &lookup(environment), + ) + .await + .unwrap(); + + assert_eq!(credentials.access_key_id(), "role-access"); + let [request] = sts.sent().try_into().unwrap(); + assert_eq!(request.action(), "AssumeRole"); + assert_eq!(request.host, host); + assert!( + request + .authorization + .as_deref() + .unwrap() + .contains("Credential=source-access/") + ); + assert!( + request + .authorization + .unwrap() + .contains(&format!("/{sts_region}/sts/aws4_request")) + ); + assert_eq!(request.form["RoleArn"], TARGET_ROLE); + assert_eq!(request.form["RoleSessionName"], "attributed"); + assert_eq!( + [ + request.form["Tags.member.1.Key"].as_str(), + request.form["Tags.member.1.Value"].as_str(), + request.form["Tags.member.2.Key"].as_str(), + request.form["Tags.member.2.Value"].as_str(), + ], + ["cost", "42", "team", "genai"] + ); + } + + #[rstest::rstest] + #[case::caller_is_the_target_role( + vec![None, Some("arn:aws:sts::123456789012:assumed-role/target/i-1")], + Ok("ambient-access") + )] + #[case::caller_is_another_role( + vec![None, Some("arn:aws:sts::123456789012:assumed-role/other/i-1")], + Err(()) + )] + #[tokio::test] + async fn an_access_denied_assume_role_falls_back_only_when_already_running_as_the_role( + #[case] caller_arns: Vec>, + #[case] expected: Result<&str, ()>, + ) { + let sts = Sts { + deny_assume_role: true, + caller_arns: Arc::new(std::sync::Mutex::new(caller_arns)), + ..Sts::default() + }; + let source = AwsCredentialSource::Chain(Box::new(AwsAuthConfig { + role_name: Some(TARGET_ROLE.into()), + ..AwsAuthConfig::default() + })); + + let result = source + .resolve( + &sts_service(&sts, None), + "us-east-1", + BEDROCK_SERVICE, + &no_env, + ) + .await; + + assert_eq!( + result + .as_ref() + .map(Credentials::access_key_id) + .map_err(|_| ()), + expected + ); + assert_eq!( + sts.sent() + .iter() + .map(StsRequest::action) + .collect::>(), + ["GetCallerIdentity", "AssumeRole", "GetCallerIdentity"] + ); + } + + fn web_identity_source(token: &str) -> AwsCredentialSource { + AwsCredentialSource::from_params( + &AwsParams { + aws_access_key_id: Some("ignored-access".into()), + aws_secret_access_key: Some("ignored-secret".into()), + aws_web_identity_token: Some(token.into()), + aws_role_name: Some(TARGET_ROLE.into()), + aws_session_name: Some("oidc-session".into()), + ..AwsParams::default() + }, + &no_env, + ) + } + + #[rstest::rstest] + #[case::bedrock_carries_the_session_policy(BEDROCK_SERVICE, Some(BEDROCK_SESSION_POLICY))] + #[case::another_service_does_not("textract", None)] + #[tokio::test] + async fn a_web_identity_reference_is_resolved_before_sts_sees_it( + #[case] service: &'static str, + #[case] policy: Option<&str>, + ) { + let sts = Sts::default(); + + let credentials = web_identity_source("oidc/file/var/run/secrets/token") + .resolve( + &sts_service(&sts, Some(Arc::new(Tokens(Some("resolved.jwt.value"))))), + "us-east-1", + service, + &no_env, + ) + .await + .unwrap(); + + assert_eq!(credentials.access_key_id(), "web-access"); + let [request] = sts.sent().try_into().unwrap(); + assert_eq!(request.action(), "AssumeRoleWithWebIdentity"); + assert_eq!(request.form["WebIdentityToken"], "resolved.jwt.value"); + assert_eq!(request.form["RoleArn"], TARGET_ROLE); + assert_eq!(request.form["RoleSessionName"], "oidc-session"); + assert_eq!(request.form["DurationSeconds"], "3600"); + assert_eq!(request.form.get("Policy").map(String::as_str), policy); + } + + #[rstest::rstest] + #[case::environment_prefix("os.environ/OIDC_TOKEN")] + #[case::environment_name("OIDC_TOKEN")] + #[tokio::test] + async fn a_web_identity_reference_to_the_environment_is_refused(#[case] token: &str) { + let sts = Sts::default(); + + let error = web_identity_source(token) + .resolve( + &sts_service(&sts, Some(Arc::new(Tokens(Some("unused"))))), + "us-east-1", + BEDROCK_SERVICE, + &lookup(&[("OIDC_TOKEN", "raw.jwt.value")]), + ) + .await + .unwrap_err(); + + assert_eq!(error, Error::WebIdentityTokenReference); + assert!(sts.sent().is_empty()); + } + + #[tokio::test] + async fn a_web_identity_reference_that_resolves_to_nothing_never_reaches_sts() { + let sts = Sts::default(); + + let error = web_identity_source("oidc/file/var/run/secrets/token") + .resolve( + &sts_service(&sts, Some(Arc::new(Tokens(None)))), + "us-east-1", + BEDROCK_SERVICE, + &no_env, + ) + .await + .unwrap_err(); + + assert!(matches!( + litellm_error::find::(&error), + Some(Error::Credentials { + flow: CredentialFlow::WebIdentity, + .. + }) + )); + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("resolved to no token") + ); + assert!(sts.sent().is_empty()); + } + + fn recorded_flow( + source: AwsCredentialSource, + environment: &'static [(&'static str, &'static str)], + ) -> (AwsAuthConfig, AwsAuthFlow) { + let loader = Arc::new(RecordingLoader::default()); + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(source.resolve( + &AwsAuthService::new(loader.clone()), + "eu-central-1", + BEDROCK_SERVICE, + &lookup(environment), + )) + .unwrap(); + loader.loads.lock().unwrap().remove(0) + } + + fn params_keys(extra: AwsParams) -> AwsParams { + AwsParams { + aws_access_key_id: Some("params-access".into()), + aws_secret_access_key: Some("params-secret".into()), + ..extra + } + } + + #[rstest::rstest] + #[case::keys_alone_sign_as_static_keys_in_the_signing_region( + params_keys(AwsParams::default()), + &[], + AwsAuthFlow::StaticKeys { access_key_id: "params-access".into(), secret_access_key: "params-secret".into(), region_name: "eu-central-1".into() } + )] + #[case::keys_pick_up_an_ambient_session_token( + params_keys(AwsParams::default()), + &[(AWS_SESSION_TOKEN, "ambient-token")], + AwsAuthFlow::SessionToken { access_key_id: "params-access".into(), secret_access_key: "params-secret".into(), session_token: "ambient-token".into() } + )] + #[case::keys_with_a_role_assume_it( + params_keys(AwsParams { aws_role_name: Some(TARGET_ROLE.into()), ..AwsParams::default() }), + &[], + AwsAuthFlow::AssumeRole { role: TARGET_ROLE.into(), session_name: None, web_identity_role: None, sts_region: Some("eu-central-1".into()) } + )] + #[case::keys_under_an_ambient_role_assume_it( + params_keys(AwsParams::default()), + &[(AWS_ROLE_NAME, TARGET_ROLE)], + AwsAuthFlow::AssumeRole { role: TARGET_ROLE.into(), session_name: None, web_identity_role: None, sts_region: Some("eu-central-1".into()) } + )] + #[case::keys_with_a_profile_use_the_profile( + params_keys(AwsParams { aws_profile_name: Some("work".into()), ..AwsParams::default() }), + &[], + AwsAuthFlow::Profile { name: "work".into() } + )] + #[case::a_blank_role_is_still_a_role( + params_keys(AwsParams { aws_role_name: Some(String::new()), ..AwsParams::default() }), + &[], + AwsAuthFlow::AssumeRole { role: String::new(), session_name: None, web_identity_role: None, sts_region: Some("eu-central-1".into()) } + )] + fn params_keys_follow_python_credential_precedence( + #[case] params: AwsParams, + #[case] environment: &'static [(&'static str, &'static str)], + #[case] expected: AwsAuthFlow, + ) { + let (config, flow) = recorded_flow( + AwsCredentialSource::from_params(¶ms, &no_env), + environment, + ); + + assert_eq!(flow, expected); + assert_eq!( + config.access_key_id.as_ref().map(SecretValue::expose), + Some("params-access") + ); + assert_eq!(config.region_name.as_deref(), Some("eu-central-1")); + } + + #[rstest::rstest] + #[case::bedrock(BEDROCK_SERVICE, Some(BEDROCK_SESSION_POLICY))] + #[case::claude_platform("aws-external-anthropic", Some(BEDROCK_SESSION_POLICY))] + #[case::textract("textract", None)] + fn the_session_policy_follows_the_signing_service( + #[case] service: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(web_identity_session_policy(service), expected); + } + #[ignore] #[tokio::test] async fn live_bedrock_invoke_model_returns_200() -> Result<(), Box> { diff --git a/litellm/rust/crates/auth-aws/src/bedrock_session_policy.json b/litellm/rust/crates/auth-aws/src/bedrock_session_policy.json new file mode 100644 index 00000000000..f80abfb9511 --- /dev/null +++ b/litellm/rust/crates/auth-aws/src/bedrock_session_policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream","bedrock:CountTokens","bedrock:Rerank","bedrock:Retrieve","bedrock:ListKnowledgeBases","bedrock:InvokeAgent","bedrock:ApplyGuardrail","bedrock:GetGuardrail","bedrock:ListGuardrails"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"BedrockAgentCoreLiteLLM","Effect":"Allow","Action":["bedrock-agentcore:InvokeAgentRuntime","bedrock-agentcore:InvokeAgentRuntimeForUser","bedrock-agentcore:InvokeGateway"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformLiteLLM","Effect":"Allow","Action":["aws-external-anthropic:CreateInference","aws-external-anthropic:CreateBatchInference","aws-external-anthropic:CancelBatchInference","aws-external-anthropic:DeleteBatchInference","aws-external-anthropic:CountTokens","aws-external-anthropic:Get*","aws-external-anthropic:List*"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformOutboundIdentityLiteLLM","Effect":"Allow","Action":["sts:GetWebIdentityToken","sts:TagGetWebIdentityToken"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringEquals":{"aws:CalledViaLast":"aws-external-anthropic.amazonaws.com"}}},{"Sid":"BedrockMantleLiteLLM","Effect":"Allow","Action":["bedrock-mantle:CreateInference","bedrock-mantle:CountTokens"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]} \ No newline at end of file diff --git a/litellm/rust/crates/auth-aws/src/constants.rs b/litellm/rust/crates/auth-aws/src/constants.rs index 0e4662eda9e..7f1ad330976 100644 --- a/litellm/rust/crates/auth-aws/src/constants.rs +++ b/litellm/rust/crates/auth-aws/src/constants.rs @@ -42,5 +42,6 @@ pub const SIGV4_COMPUTED_HEADER_NAMES: &[&str] = &[ pub const BEDROCK_SERVICE: &str = "bedrock"; pub const DEFAULT_SESSION_NAME_PREFIX: &str = "litellm-session"; pub const DEFAULT_BEDROCK_REGION: &str = "us-west-2"; -pub const BEDROCK_RUNTIME_ENDPOINT_TEMPLATE: &str = - "https://bedrock-runtime.{region}.amazonaws.com"; +pub const BEDROCK_SESSION_POLICY: &str = include_str!("bedrock_session_policy.json"); +pub const WEB_IDENTITY_SESSION_SECONDS: i32 = 3600; +pub const AWS4_AUTHORIZATION_PREFIX: &str = "AWS4-HMAC-SHA256"; diff --git a/litellm/rust/crates/auth-aws/src/error.rs b/litellm/rust/crates/auth-aws/src/error.rs index 5445de1e6c3..96caddee754 100644 --- a/litellm/rust/crates/auth-aws/src/error.rs +++ b/litellm/rust/crates/auth-aws/src/error.rs @@ -1,3 +1,4 @@ +use aws_credential_types::provider::error::CredentialsError; use litellm_error::Source; #[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] @@ -10,6 +11,14 @@ pub enum Error { #[source] source: Source, }, + #[error( + "the AWS web identity token must name a secret or an oidc/ provider, not an environment variable" + )] + WebIdentityTokenReference, + #[error("the AWS web identity token reference resolved to no token")] + MissingWebIdentityToken, + #[error("could not read the AWS web identity token")] + WebIdentityToken(#[source] Source), #[error("AWS web identity response had no credentials")] MissingWebIdentityCredentials, #[error("AWS web identity response had an invalid expiration")] @@ -49,6 +58,20 @@ pub enum SigningStage { } impl Error { + pub fn is_empty_default_chain(&self) -> bool { + let Self::Credentials { + flow: CredentialFlow::DefaultChain, + source, + } = self + else { + return false; + }; + matches!( + source.downcast_ref::(), + Some(CredentialsError::CredentialsNotLoaded(_)) + ) + } + pub(crate) fn credentials( flow: CredentialFlow, ) -> impl FnOnce(E) -> Self { @@ -73,3 +96,40 @@ impl From for litellm_auth_types::Error { Self::provider(error) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[rstest::rstest] + #[case::empty_default_chain( + CredentialFlow::DefaultChain, + CredentialsError::not_loaded("no providers in chain provided credentials"), + true + )] + #[case::failed_default_chain( + CredentialFlow::DefaultChain, + CredentialsError::provider_error("IMDS answered 500"), + false + )] + #[case::empty_profile( + CredentialFlow::Profile, + CredentialsError::not_loaded("the profile has no credentials"), + false + )] + fn only_a_default_chain_that_loaded_nothing_is_empty( + #[case] flow: CredentialFlow, + #[case] failure: CredentialsError, + #[case] empty: bool, + ) { + assert_eq!( + Error::credentials(flow)(failure).is_empty_default_chain(), + empty + ); + } + + #[test] + fn a_web_identity_failure_is_not_an_empty_default_chain() { + assert!(!Error::MissingWebIdentityToken.is_empty_default_chain()); + } +} diff --git a/litellm/rust/crates/auth-aws/src/lib.rs b/litellm/rust/crates/auth-aws/src/lib.rs index c8c5b5e8653..b467af27a6c 100644 --- a/litellm/rust/crates/auth-aws/src/lib.rs +++ b/litellm/rust/crates/auth-aws/src/lib.rs @@ -2,10 +2,12 @@ mod aws; pub mod constants; mod error; mod provider; +mod region; mod signer; pub use aws::*; pub use aws_credential_types::Credentials; pub use error::{CredentialFlow, Error, SigningStage}; pub use provider::AwsCredentialsProvider; -pub use signer::SigV4Signer; +pub use region::{AwsPartition, aws_partition, bedrock_runtime_endpoint}; +pub use signer::{CallerSigningHeaders, SigV4Signer}; diff --git a/litellm/rust/crates/auth-aws/src/region.rs b/litellm/rust/crates/auth-aws/src/region.rs new file mode 100644 index 00000000000..ab5f49a26ef --- /dev/null +++ b/litellm/rust/crates/auth-aws/src/region.rs @@ -0,0 +1,239 @@ +use std::path::PathBuf; + +use crate::constants::AWS_DEFAULT_REGION; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AwsPartition { + pub partition: &'static str, + pub dns_suffix: &'static str, +} + +const COMMERCIAL_PARTITION: AwsPartition = AwsPartition { + partition: "aws", + dns_suffix: "amazonaws.com", +}; + +const PARTITIONS_BY_REGION_PREFIX: [(&str, AwsPartition); 6] = [ + ( + "cn-", + AwsPartition { + partition: "aws-cn", + dns_suffix: "amazonaws.com.cn", + }, + ), + ( + "us-gov-", + AwsPartition { + partition: "aws-us-gov", + dns_suffix: "amazonaws.com", + }, + ), + ( + "us-isob-", + AwsPartition { + partition: "aws-iso-b", + dns_suffix: "sc2s.sgov.gov", + }, + ), + ( + "us-isof-", + AwsPartition { + partition: "aws-iso-f", + dns_suffix: "csp.hci.ic.gov", + }, + ), + ( + "us-iso-", + AwsPartition { + partition: "aws-iso", + dns_suffix: "c2s.ic.gov", + }, + ), + ( + "eu-isoe-", + AwsPartition { + partition: "aws-iso-e", + dns_suffix: "cloud.adc-e.uk", + }, + ), +]; + +pub fn aws_partition(region: &str) -> AwsPartition { + PARTITIONS_BY_REGION_PREFIX + .iter() + .find(|(prefix, _)| region.starts_with(prefix)) + .map_or(COMMERCIAL_PARTITION, |(_, partition)| *partition) +} + +pub fn bedrock_runtime_endpoint(region: &str) -> String { + format!( + "https://bedrock-runtime.{region}.{}", + aws_partition(region).dns_suffix + ) +} + +pub(crate) fn session_region( + env_lookup: &dyn Fn(&str) -> Option, + process_env: &dyn Fn(&str) -> Option, +) -> Option { + env_lookup(AWS_DEFAULT_REGION) + .filter(|region| !region.is_empty()) + .or_else(|| profile_region(process_env)) +} + +fn profile_region(process_env: &dyn Fn(&str) -> Option) -> Option { + let profile = process_env("AWS_PROFILE") + .or_else(|| process_env("AWS_DEFAULT_PROFILE")) + .filter(|profile| !profile.is_empty()) + .unwrap_or_else(|| "default".to_string()); + let home = process_env("HOME"); + let path = match process_env("AWS_CONFIG_FILE").filter(|path| !path.is_empty()) { + Some(path) => match (path.strip_prefix("~/"), &home) { + (Some(rest), Some(home)) => PathBuf::from(home).join(rest), + _ => PathBuf::from(path), + }, + None => PathBuf::from(home?).join(".aws").join("config"), + }; + config_file_region(&std::fs::read_to_string(path).ok()?, &profile) +} + +fn config_file_region(contents: &str, profile: &str) -> Option { + let named = format!("profile {profile}"); + let selects = |section: &str| section == named || (profile == "default" && section == profile); + let mut selected = false; + for line in contents.lines().map(str::trim) { + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { + continue; + } + if let Some(section) = line + .strip_prefix('[') + .and_then(|line| line.strip_suffix(']')) + { + selected = selects(§ion.split_whitespace().collect::>().join(" ")); + continue; + } + if selected + && let Some((key, value)) = line.split_once('=') + && key.trim() == "region" + { + return Some(value.trim().to_string()).filter(|region| !region.is_empty()); + } + } + None +} + +pub(crate) fn sts_endpoint_region(endpoint: &str) -> Option { + let url = url::Url::parse(endpoint).ok()?; + let labels: Vec<&str> = url.host_str()?.split('.').collect(); + (0..labels.len()).find_map(|index| { + let ["sts" | "sts-fips", region, rest @ ..] = &labels[index..] else { + return None; + }; + let rest = rest.join("."); + let valid_region = !region.is_empty() + && region + .chars() + .all(|char| char.is_ascii_lowercase() || char.is_ascii_digit() || char == '-'); + (valid_region + && (rest.starts_with("amazonaws.com") || rest.starts_with("vpce.amazonaws.com"))) + .then(|| region.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::commercial("us-east-1", "https://bedrock-runtime.us-east-1.amazonaws.com")] + #[case::china("cn-north-1", "https://bedrock-runtime.cn-north-1.amazonaws.com.cn")] + #[case::gov_cloud("us-gov-west-1", "https://bedrock-runtime.us-gov-west-1.amazonaws.com")] + #[case::iso("us-iso-east-1", "https://bedrock-runtime.us-iso-east-1.c2s.ic.gov")] + #[case::iso_b( + "us-isob-east-1", + "https://bedrock-runtime.us-isob-east-1.sc2s.sgov.gov" + )] + #[case::iso_f( + "us-isof-south-1", + "https://bedrock-runtime.us-isof-south-1.csp.hci.ic.gov" + )] + #[case::iso_e( + "eu-isoe-west-1", + "https://bedrock-runtime.eu-isoe-west-1.cloud.adc-e.uk" + )] + fn the_runtime_endpoint_uses_the_partition_dns_suffix( + #[case] region: &str, + #[case] expected: &str, + ) { + assert_eq!(bedrock_runtime_endpoint(region), expected); + } + + #[rstest] + #[case::china("cn-northwest-1", "aws-cn")] + #[case::gov_cloud("us-gov-east-1", "aws-us-gov")] + #[case::commercial("eu-central-1", "aws")] + fn a_region_names_its_partition(#[case] region: &str, #[case] expected: &str) { + assert_eq!(aws_partition(region).partition, expected); + } + + #[rstest] + #[case::regional("https://sts.eu-west-1.amazonaws.com", Some("eu-west-1"))] + #[case::fips("https://sts-fips.us-east-2.amazonaws.com/", Some("us-east-2"))] + #[case::china("https://sts.cn-north-1.amazonaws.com.cn", Some("cn-north-1"))] + #[case::vpc_endpoint( + "https://vpce-0abc-1234.sts.ap-south-1.vpce.amazonaws.com", + Some("ap-south-1") + )] + #[case::global("https://sts.amazonaws.com", None)] + #[case::custom_host("https://sts.internal.example.com", None)] + #[case::no_scheme("sts.eu-west-1.amazonaws.com", None)] + fn the_sts_region_is_read_from_a_regional_endpoint( + #[case] endpoint: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(sts_endpoint_region(endpoint).as_deref(), expected); + } + + const CONFIG: &str = "[default]\nregion = eu-west-3\n\n[profile work]\noutput = json\nregion=ap-northeast-1\n\n[profile empty]\nregion =\n"; + + #[rstest] + #[case::default_profile("default", Some("eu-west-3"))] + #[case::named_profile("work", Some("ap-northeast-1"))] + #[case::blank_region("empty", None)] + #[case::unknown_profile("missing", None)] + fn the_profile_region_comes_from_its_config_section( + #[case] profile: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(config_file_region(CONFIG, profile).as_deref(), expected); + } + + #[rstest] + #[case::default_region_wins(Some("sa-east-1"), None, Some("sa-east-1"))] + #[case::blank_default_region_reads_the_profile(Some(""), Some("work"), Some("ap-northeast-1"))] + #[case::profile_from_the_environment(None, Some("work"), Some("ap-northeast-1"))] + #[case::default_profile(None, None, Some("eu-west-3"))] + fn the_session_region_reads_the_default_region_then_the_profile( + #[case] default_region: Option<&str>, + #[case] profile: Option<&str>, + #[case] expected: Option<&str>, + ) { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(file.path(), CONFIG).unwrap(); + let config = file.path().to_string_lossy().into_owned(); + let env = |name: &str| { + (name == AWS_DEFAULT_REGION) + .then(|| default_region.map(str::to_string)) + .flatten() + }; + let process_env = |name: &str| match name { + "AWS_CONFIG_FILE" => Some(config.clone()), + "AWS_PROFILE" => profile.map(str::to_string), + _ => None, + }; + + assert_eq!(session_region(&env, &process_env).as_deref(), expected); + } +} diff --git a/litellm/rust/crates/auth-aws/src/signer.rs b/litellm/rust/crates/auth-aws/src/signer.rs index 2d9ad9ba123..12867f0c785 100644 --- a/litellm/rust/crates/auth-aws/src/signer.rs +++ b/litellm/rust/crates/auth-aws/src/signer.rs @@ -1,11 +1,18 @@ +use std::collections::BTreeMap; use std::time::SystemTime; use aws_credential_types::Credentials; -use reqwest::{Request, header::HeaderMap}; +use aws_sigv4::sign::v4; +use chrono::{DateTime, Utc}; +use reqwest::{ + Request, + header::{HeaderMap, HeaderName, HeaderValue}, +}; +use sha2::{Digest, Sha256}; use crate::{ - AwsAuthService, AwsCredentialSource, Error, SigningScope, aws_signature_headers, - is_sigv4_computed_header, sign_headers, + AwsAuthService, AwsCredentialSource, Error, SigningScope, SigningStage, aws_signature_headers, + constants::AWS4_AUTHORIZATION_PREFIX, is_sigv4_computed_header, sign_headers, }; use litellm_http::outbound::RequestSigner; @@ -15,6 +22,14 @@ pub struct SigV4Signer { service: &'static str, credentials: Credentials, clock: fn() -> SystemTime, + caller_headers: CallerSigningHeaders, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum CallerSigningHeaders { + #[default] + Dropped, + Botocore, } impl SigV4Signer { @@ -24,6 +39,7 @@ impl SigV4Signer { service, credentials, clock: SystemTime::now, + caller_headers: CallerSigningHeaders::Dropped, } } @@ -31,6 +47,18 @@ impl SigV4Signer { Self { clock, ..self } } + pub fn with_caller_headers(self, caller_headers: CallerSigningHeaders) -> Self { + Self { + caller_headers, + ..self + } + } + + fn keeps_caller_security_token(&self) -> bool { + self.caller_headers == CallerSigningHeaders::Botocore + && self.credentials.session_token().is_none() + } + pub async fn resolve( auth: &AwsAuthService, region: String, @@ -38,48 +66,246 @@ impl SigV4Signer { credentials: AwsCredentialSource, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result { - Ok(Self::new( - region, - service, - credentials.resolve(auth, env_lookup).await?, - )) + let credentials = credentials + .resolve(auth, ®ion, service, env_lookup) + .await?; + Ok(Self::new(region, service, credentials)) } } -impl RequestSigner for SigV4Signer { - fn sign(&self, request: &Request) -> Result { - if let Some(name) = request - .headers() - .keys() - .find(|name| is_sigv4_computed_header(name.as_str())) - { - return Err(litellm_http::Error::computed_header(name.as_str())); +const CALLER_HEADERS_THE_SIGNER_REPLACES: &[&str] = &["x-amz-date", "x-amz-security-token", "date"]; + +const CALLER_DATE_HEADERS: &[&str] = &["x-amz-date", "date"]; + +const SECURITY_TOKEN: &str = "x-amz-security-token"; + +const BOTOCORE_UNSIGNED_HEADERS: &[&str] = &["expect", "user-agent", "x-amzn-trace-id"]; + +fn signable(headers: &HeaderMap, keep_security_token: bool) -> HeaderMap { + aws_signature_headers(headers) + .iter() + .filter(|(name, _)| { + !is_sigv4_computed_header(name.as_str()) + || (keep_security_token && name.as_str() == SECURITY_TOKEN) + }) + .map(|(name, value)| (name.clone(), value.clone())) + .collect() +} + +fn hex_sha256(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn quote_path(path: &str) -> String { + path.bytes() + .map(|byte| match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' | b'/' => { + char::from(byte).to_string() + } + other => format!("%{other:02X}"), + }) + .collect() +} + +fn canonical_uri(path: &str) -> String { + if path.is_empty() { + return "/".to_string(); + } + let segments = path.split('/').fold(Vec::new(), |mut kept, segment| { + match segment { + "" | "." => {} + ".." => { + kept.pop(); + } + other => kept.push(other), } + kept + }); + let leading = if path.starts_with('/') { "/" } else { "" }; + let trailing = if path.ends_with('/') && !segments.is_empty() { + "/" + } else { + "" + }; + quote_path(&format!("{leading}{}{trailing}", segments.join("/"))) +} + +fn canonical_query(query: Option<&str>) -> String { + let mut pairs: Vec<(&str, &str)> = query + .filter(|query| !query.is_empty()) + .map(|query| { + query + .split('&') + .map(|pair| pair.split_once('=').unwrap_or((pair, ""))) + .collect() + }) + .unwrap_or_default(); + pairs.sort_unstable(); + pairs + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join("&") +} + +fn canonical_host(url: &reqwest::Url) -> String { + let host = url.host_str().unwrap_or_default(); + match url.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + } +} + +fn header_pair(name: &str, value: &str) -> Result<(HeaderName, HeaderValue), Error> { + Ok(( + HeaderName::from_bytes(name.as_bytes()).map_err(Error::signing(SigningStage::Output))?, + HeaderValue::from_str(value).map_err(Error::signing(SigningStage::Output))?, + )) +} + +fn sign_with_date_header( + request: &Request, + body: &[u8], + headers: &HeaderMap, + scope: SigningScope<'_>, +) -> Result { + let time: DateTime = scope.time.into(); + let amz_date = time.format("%Y%m%dT%H%M%SZ").to_string(); + let http_date = time.format("%a, %d %b %Y %H:%M:%S -0000").to_string(); + let session_token = scope.credentials.session_token(); + let mut canonical: BTreeMap> = BTreeMap::new(); + for (name, value) in headers { + if BOTOCORE_UNSIGNED_HEADERS.contains(&name.as_str()) { + continue; + } + let value = value.to_str().map_err(|_| Error::NonAsciiHeader { + name: name.to_string(), + })?; + canonical + .entry(name.as_str().to_string()) + .or_default() + .push(value.split_whitespace().collect::>().join(" ")); + } + canonical.insert("date".into(), vec![http_date.clone()]); + if let Some(token) = session_token { + canonical.insert(SECURITY_TOKEN.into(), vec![token.to_string()]); + } + canonical + .entry("host".into()) + .or_insert_with(|| vec![canonical_host(request.url())]); + let payload = headers + .get("x-amz-content-sha256") + .and_then(|value| value.to_str().ok()) + .map_or_else(|| hex_sha256(body), str::to_string); + let canonical_headers: String = canonical + .iter() + .map(|(name, values)| format!("{name}:{}\n", values.join(","))) + .collect(); + let signed_headers = canonical.keys().cloned().collect::>().join(";"); + let canonical_request = format!( + "{}\n{}\n{}\n{canonical_headers}\n{signed_headers}\n{payload}", + request.method().as_str(), + canonical_uri(request.url().path()), + canonical_query(request.url().query()), + ); + let credential_scope = format!( + "{}/{}/{}/aws4_request", + &amz_date[..8], + scope.region, + scope.service + ); + let string_to_sign = format!( + "{AWS4_AUTHORIZATION_PREFIX}\n{amz_date}\n{credential_scope}\n{}", + hex_sha256(canonical_request.as_bytes()) + ); + let signing_key = v4::generate_signing_key( + scope.credentials.secret_access_key(), + scope.time, + scope.region, + scope.service, + ); + let signature = v4::calculate_signature(signing_key, string_to_sign.as_bytes()); + let authorization = format!( + "{AWS4_AUTHORIZATION_PREFIX} Credential={}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}", + scope.credentials.access_key_id() + ); + [ + Some(("date", http_date.as_str())), + session_token.map(|token| (SECURITY_TOKEN, token)), + Some(("authorization", authorization.as_str())), + ] + .into_iter() + .flatten() + .map(|(name, value)| header_pair(name, value)) + .collect() +} + +fn overrides_the_signature(headers: &HeaderMap) -> bool { + headers + .get(reqwest::header::AUTHORIZATION) + .is_some_and(|value| { + !value + .as_bytes() + .starts_with(AWS4_AUTHORIZATION_PREFIX.as_bytes()) + }) +} + +impl RequestSigner for SigV4Signer { + fn owned_headers(&self) -> &'static [&'static str] { + if self.keeps_caller_security_token() { + CALLER_DATE_HEADERS + } else { + CALLER_HEADERS_THE_SIGNER_REPLACES + } + } + + fn sign(&self, request: &Request) -> Result { let body = match request.body() { None => &[], Some(body) => body .as_bytes() .ok_or_else(litellm_http::Error::unsignable_body)?, }; - sign_headers( - request.method().as_str(), - request.url().as_str(), - body, - &aws_signature_headers(request.headers()), - SigningScope { - region: &self.region, - service: self.service, - credentials: &self.credentials, - time: (self.clock)(), - }, - ) + let headers = signable(request.headers(), self.keeps_caller_security_token()); + let scope = SigningScope { + region: &self.region, + service: self.service, + credentials: &self.credentials, + time: (self.clock)(), + }; + let signed = match self.caller_headers == CallerSigningHeaders::Botocore + && request.headers().contains_key(reqwest::header::DATE) + { + true => sign_with_date_header(request, body, &headers, scope), + false => sign_headers( + request.method().as_str(), + request.url().as_str(), + body, + &headers, + scope, + ), + } .map_err(|error| match error { Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error), Error::ExpiredCredentials | Error::Credentials { .. } + | Error::WebIdentityTokenReference + | Error::MissingWebIdentityToken + | Error::WebIdentityToken(_) | Error::MissingWebIdentityCredentials | Error::WebIdentityExpiration(_) | Error::Signing { .. } => litellm_http::Error::signature(error), + })?; + Ok(match overrides_the_signature(request.headers()) { + true => signed + .iter() + .filter(|(name, _)| *name != reqwest::header::AUTHORIZATION) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(), + false => signed, }) } } @@ -176,22 +402,166 @@ mod tests { assert!(original.contains("/us-east-1/textract/aws4_request")); } - #[test] - fn a_forwarded_computed_header_is_refused_instead_of_sent_twice() { - let error = signed( + fn sent(headers: &[(&str, &str)], credentials: Credentials) -> Request { + signed( json( URL, - &[("authorization".into(), "Bearer caller".into())], + &headers + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>(), &json_value!({}), None, ) .unwrap(), - &signer("textract"), + &SigV4Signer::new("us-east-1".into(), "bedrock", credentials).with_clock(fixed_clock), ) - .unwrap_err(); + .unwrap() + } - assert!(error.is_computed_header()); - assert!(error.to_string().contains("authorization"), "{error}"); + fn header<'a>(request: &'a Request, name: &str) -> Vec<&'a str> { + request + .headers() + .get_all(name) + .iter() + .map(|value| value.to_str().unwrap()) + .collect() + } + + #[rstest::rstest] + #[case::caller_date("date", None)] + #[case::caller_amz_date("x-amz-date", Some("20231114T221320Z"))] + #[case::caller_security_token_without_a_session("x-amz-security-token", None)] + fn a_header_the_signer_computes_is_dropped_from_the_callers_copy( + #[case] name: &str, + #[case] expected: Option<&str>, + ) { + let request = sent( + &[(name, "caller-value")], + Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"), + ); + + assert_eq!( + header(&request, name), + expected.into_iter().collect::>() + ); + assert!(header(&request, "authorization")[0].starts_with("AWS4-HMAC-SHA256 ")); + } + + #[test] + fn a_session_token_replaces_the_callers_security_token() { + let request = sent( + &[("X-Amz-Security-Token", "caller-value")], + Credentials::new( + "AKIDEXAMPLE", + "secret", + Some("session".into()), + None, + "test", + ), + ); + + assert_eq!(header(&request, "x-amz-security-token"), ["session"]); + } + + #[rstest::rstest] + #[case::bearer_overrides_the_signature("Bearer gateway", "Bearer gateway")] + #[case::a_stale_signature_is_replaced( + "AWS4-HMAC-SHA256 Credential=stale", + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;host;x-amz-date" + )] + fn a_callers_authorization_overrides_the_signature_unless_it_is_sigv4( + #[case] caller: &str, + #[case] expected: &str, + ) { + let request = sent( + &[("Authorization", caller)], + Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"), + ); + let authorization = header(&request, "authorization"); + + assert_eq!(authorization.len(), 1); + assert_eq!( + authorization[0].split(", Signature=").next(), + Some(expected) + ); + assert_eq!(header(&request, "x-amz-date"), ["20231114T221320Z"]); + } + + const BEDROCK_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke"; + const ARN_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A1%3Ainference-profile%2Fx/invoke?b=2&a=1"; + const CALLER_SIGNING_HEADERS: &[(&str, &str)] = &[ + ("date", "Wed, 01 Jan 2020 00:00:00 GMT"), + ("x-amz-date", "20200101T000000Z"), + ("x-amz-security-token", "caller-token"), + ]; + const CALLER_TOKEN: &[(&str, &str)] = &[("x-amz-security-token", "caller-token")]; + const CALLER_DATE: &[(&str, &str)] = &[("Date", "Wed, 01 Jan 2020 00:00:00 GMT")]; + const HTTP_DATE: &str = "Tue, 14 Nov 2023 22:13:20 -0000"; + const AMZ_DATE: &str = "20231114T221320Z"; + + #[rstest::rstest] + #[case::date_and_caller_token(URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "1b3652d523d5296b3904a80981e936a201167da9cb7bd200c2da8171dadb2c08")] + #[case::caller_token_only(URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "0fbd1a8e98efbfbcdb6fa9d665fe89ffffcdd8a7656865d63def21a5344d02dd")] + #[case::date_only(URL, CALLER_DATE, None, (Some(HTTP_DATE), None, None), "content-type;date;host", "cccdac7b04f20ebfe37a156c4e806a14e6c0a4b0aad11ca67e25dca337237085")] + #[case::session_replaces_the_caller_token_under_a_date(URL, CALLER_SIGNING_HEADERS, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "3406b8200cae95c8b66037dc5a9a838e1fe4da94b57b89fada3c05cde8a2c15c")] + #[case::session_replaces_the_caller_token(URL, CALLER_TOKEN, Some("session"), (None, Some(AMZ_DATE), Some("session")), "content-type;host;x-amz-date;x-amz-security-token", "d75425952d9f5ef7465a6ef3acabf15908c0edfd8ce2529cc3f0f8d2e9232e76")] + #[case::colon_in_the_model_path(BEDROCK_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "47d75a0dcb00499a025fc0f9f010f6dce89c7661eb54cb97c3a293e9c585b153")] + #[case::encoded_arn_and_query(ARN_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "8fb7d97a0ad5df9d1c3bbfc55897266059a5a2f35a1aaed9687f370047e78363")] + #[case::encoded_arn_and_query_without_a_date(ARN_URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "f5eb993af90c7896203bf6b3b85c0087087f712a0634e44ac0a04c9d87d6fe12")] + #[case::encoded_arn_and_query_with_a_session(ARN_URL, CALLER_DATE, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "15e8d5a9f64eb64366363f951db64de2a747959573a8cda9b5faa0fcbabc713c")] + fn botocore_caller_headers_sign_like_botocore( + #[case] url: &str, + #[case] caller: &[(&str, &str)], + #[case] session: Option<&str>, + #[case] sent: (Option<&str>, Option<&str>, Option<&str>), + #[case] signed_headers: &str, + #[case] signature: &str, + ) { + let request = signed( + json( + url, + &caller + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>(), + &json_value!({}), + None, + ) + .unwrap(), + &SigV4Signer::new( + "us-east-1".into(), + "bedrock", + Credentials::new( + "AKIDEXAMPLE", + "secret", + session.map(str::to_string), + None, + "test", + ), + ) + .with_clock(fixed_clock) + .with_caller_headers(CallerSigningHeaders::Botocore), + ) + .unwrap(); + + assert_eq!( + ( + header(&request, "date"), + header(&request, "x-amz-date"), + header(&request, "x-amz-security-token"), + header(&request, "authorization"), + ), + ( + sent.0.into_iter().collect::>(), + sent.1.into_iter().collect::>(), + sent.2.into_iter().collect::>(), + vec![format!( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders={signed_headers}, Signature={signature}" + ) + .as_str()], + ) + ); } #[test] diff --git a/litellm/rust/crates/auth-types/src/lib.rs b/litellm/rust/crates/auth-types/src/lib.rs index ab3bd3febbf..189655e6933 100644 --- a/litellm/rust/crates/auth-types/src/lib.rs +++ b/litellm/rust/crates/auth-types/src/lib.rs @@ -52,6 +52,6 @@ pub use credential::{ }; pub use error::{Error, ErrorDetail}; pub use http::CredentialPlacement; -pub use params::{AwsParams, ParamKind, ParamSpec, VertexParams}; +pub use params::{AwsParams, AwsSessionTag, ParamKind, ParamSpec, VertexParams}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; diff --git a/litellm/rust/crates/auth-types/src/params/aws.rs b/litellm/rust/crates/auth-types/src/params/aws.rs index de21fbf94c9..c3489c2d234 100644 --- a/litellm/rust/crates/auth-types/src/params/aws.rs +++ b/litellm/rust/crates/auth-types/src/params/aws.rs @@ -21,6 +21,14 @@ const fn spec( } } +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct AwsSessionTag { + #[serde(rename = "Key")] + pub key: String, + #[serde(rename = "Value")] + pub value: String, +} + /// The `aws_*` fields of Python's `GenericLiteLLMParams`. #[derive(Redact, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct AwsParams { @@ -50,11 +58,15 @@ pub struct AwsParams { pub aws_external_id: Option, #[serde(default)] pub aws_bedrock_runtime_endpoint: Option, + #[serde(default)] + pub aws_session_tags: Option>, } impl AwsParams { pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN"; pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; + pub const DEFAULT_REGION_ENV: &str = "AWS_DEFAULT_REGION"; + pub const SESSION_TAGS: &str = "aws_session_tags"; pub const ACCESS_KEY_ID: ParamSpec = spec( "access key id", @@ -144,6 +156,7 @@ impl AwsParams { Self::SPECS .iter() .flat_map(|spec| spec.wire.iter().copied()) + .chain([Self::SESSION_TAGS]) } /// The environment names the specs read, for hosts that resolve secrets up front. @@ -153,6 +166,7 @@ impl AwsParams { .iter() .flat_map(|spec| spec.env.iter().copied()) .chain([ + AwsParams::DEFAULT_REGION_ENV, AwsParams::ROLE_ARN_ENV, AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV, ]) @@ -205,6 +219,9 @@ impl AwsParams { aws_sts_endpoint: value("aws_sts_endpoint"), aws_external_id: value("aws_external_id"), aws_bedrock_runtime_endpoint: value("aws_bedrock_runtime_endpoint"), + aws_session_tags: optional_params + .get(Self::SESSION_TAGS) + .and_then(|tags| serde_json::from_value(tags.clone()).ok()), } } } diff --git a/litellm/rust/crates/auth-types/src/params/mod.rs b/litellm/rust/crates/auth-types/src/params/mod.rs index daad1aff8a8..634bc297f03 100644 --- a/litellm/rust/crates/auth-types/src/params/mod.rs +++ b/litellm/rust/crates/auth-types/src/params/mod.rs @@ -1,7 +1,7 @@ mod aws; mod vertex; -pub use aws::AwsParams; +pub use aws::{AwsParams, AwsSessionTag}; pub use vertex::VertexParams; /// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads diff --git a/litellm/rust/crates/auth-types/tests/aws_params.rs b/litellm/rust/crates/auth-types/tests/aws_params.rs index b438984c199..5c806a7aa7b 100644 --- a/litellm/rust/crates/auth-types/tests/aws_params.rs +++ b/litellm/rust/crates/auth-types/tests/aws_params.rs @@ -1,11 +1,16 @@ use litellm_auth_types::AwsParams; use rstest::rstest; -use serde_json::{Map, Value}; +use serde_json::{Map, Value, json}; #[rstest] fn every_named_param_roundtrips_without_loss() { let filled: Map = AwsParams::fields() - .map(|name| (name.to_string(), Value::from(format!("value-of-{name}")))) + .map(|name| match name { + AwsParams::SESSION_TAGS => { + (name.to_string(), json!([{"Key": "team", "Value": "genai"}])) + } + _ => (name.to_string(), Value::from(format!("value-of-{name}"))), + }) .collect(); let typed = AwsParams::from_optional_params(&filled); let serialized = serde_json::to_value(&typed).unwrap(); diff --git a/litellm/rust/crates/callbacks-legacy-python/python_contract.json b/litellm/rust/crates/callbacks-legacy-python/python_contract.json index 505651b5f16..2024400495e 100644 --- a/litellm/rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm/rust/crates/callbacks-legacy-python/python_contract.json @@ -159,6 +159,7 @@ "endpoint_type", "request_body", "chunks", - "error" + "error", + "head" ] } diff --git a/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs index 5f451f59c31..2b36f1ebff5 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs @@ -38,6 +38,7 @@ struct PassThroughStream { /// What the Messages stream iterator keeps for its end-of-stream billing. struct DeliveredStream { + head: Py, chunks: Py, first_chunk: Option>, } @@ -65,22 +66,14 @@ pub struct LegacyLogging { /// When the provider request went out, until its window is recorded (Messages only). api_started: Option, host_answered: bool, - /// Whether the call's Messages stream starts only on its first read, as Python's iterator - /// does. Only `azure_ai`: by the owner's ruling `anthropic` keeps main's billing of an - /// unread stream. + /// Whether the call's stream starts only on its first read, as Python's Messages iterator + /// does. stream_starts_on_first_read: bool, /// Set once the route sends through the host's own client, whose timed `post` already /// reports the exchange. host_client: Arc, } -/// The one provider whose unread Messages stream bills nothing, as Python's does. -const STREAM_STARTS_ON_FIRST_READ_PROVIDER: &str = "azure_ai"; - -/// The one provider whose exchange through the host's client is timed only by that client, as -/// Python's is. By the owner's ruling `anthropic` keeps main's second report. -const HOST_CLIENT_TIMED_PROVIDER: &str = "azure_ai"; - fn retry_reason(reason: &RetryReason) -> Value { match reason { RetryReason::ThinkingSignature => serde_json::json!({"kind": "thinking_signature"}), @@ -504,6 +497,7 @@ impl LegacyLogging { ), &stream.chunks, error, + &stream.head, ), ); match scheduled { @@ -675,8 +669,7 @@ impl LegacyLogging { )?; if self.operation == LoggingOperation::Messages && self.asynchronous - && !(context.custom_llm_provider == HOST_CLIENT_TIMED_PROVIDER - && self.host_client.load(Ordering::Acquire)) + && !self.host_client.load(Ordering::Acquire) && self.times_provider_exchange(py)? { self.api_started = Some(epoch_seconds()); @@ -729,8 +722,7 @@ impl LegacyLogging { ) -> PyResult> { use litellm_host::interceptors::{Cost, ResultSource}; - self.stream_starts_on_first_read = - facts.provider.provider == STREAM_STARTS_ON_FIRST_READ_PROVIDER; + self.stream_starts_on_first_read = self.operation == LoggingOperation::Messages; self.cache_key = match &facts.source { ResultSource::Provider => None, ResultSource::Cache { key } => Some(key.clone()), @@ -940,6 +932,7 @@ impl LegacyLogging { ), )?; self.stream = Some(DeliveredStream { + head: head.clone_ref(py), chunks: PyList::empty(py).unbind(), first_chunk: None, }); @@ -995,6 +988,7 @@ impl PythonOwned for LegacyLogging { visit.call(&self.response)?; visit.call(&self.error)?; if let Some(stream) = &self.stream { + visit.call(&stream.head)?; visit.call(&stream.chunks)?; visit.call(&stream.first_chunk)?; } diff --git a/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs index ab1a4ab9cdb..858d6573faf 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs @@ -107,7 +107,7 @@ FAKES = { 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record( 'stream_success', list(chunks) ), - 'stream_failure': lambda logger, endpoint_type, request_body, chunks, error: logger.record('stream_failure', error), + 'stream_failure': lambda logger, endpoint_type, request_body, chunks, error, head: logger.record('stream_failure', error), } assert FAKES.keys() == CONTRACT.keys(), sorted(FAKES.keys() ^ CONTRACT.keys()) for name, fake in FAKES.items(): diff --git a/litellm/rust/crates/callbacks-s3-log/src/settings.rs b/litellm/rust/crates/callbacks-s3-log/src/settings.rs index d3d32eae77f..1dd09cf675c 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/settings.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/settings.rs @@ -336,6 +336,8 @@ impl Settings { web_identity_token: secret(params, "s3_aws_web_identity_token"), sts_endpoint: text(params, "s3_aws_sts_endpoint"), external_id: None, + session_tags: None, + session_policy: None, }, use_team_prefix: truthy_param("s3_use_team_prefix"), use_key_prefix: truthy_param("s3_use_key_prefix"), diff --git a/litellm/rust/crates/gateway-inference/Cargo.toml b/litellm/rust/crates/gateway-inference/Cargo.toml index 69ca561e4b5..271b668e4f3 100644 --- a/litellm/rust/crates/gateway-inference/Cargo.toml +++ b/litellm/rust/crates/gateway-inference/Cargo.toml @@ -32,6 +32,7 @@ serde_json.workspace = true thiserror.workspace = true [dev-dependencies] +litellm-core-utils.workspace = true litellm-cache-memory.workspace = true futures-util.workspace = true tokio = { workspace = true, features = ["io-util"] } diff --git a/litellm/rust/crates/gateway-inference/src/error.rs b/litellm/rust/crates/gateway-inference/src/error.rs index 190561079f6..d05ecb2ff68 100644 --- a/litellm/rust/crates/gateway-inference/src/error.rs +++ b/litellm/rust/crates/gateway-inference/src/error.rs @@ -324,8 +324,8 @@ mod tests { #[rstest] #[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)] #[case::declined( - Error::Route(InferenceError::from(litellm_http::Error::computed_header( - "authorization" + Error::Route(InferenceError::Declined(litellm_core_utils::decline::Decline::new( + "unrecognized request parameter" ))), StatusCode::NOT_IMPLEMENTED )] diff --git a/litellm/rust/crates/gateway-inference/src/messages.rs b/litellm/rust/crates/gateway-inference/src/messages.rs index 79b787a3c53..56e283d63b9 100644 --- a/litellm/rust/crates/gateway-inference/src/messages.rs +++ b/litellm/rust/crates/gateway-inference/src/messages.rs @@ -108,7 +108,7 @@ fn project( .chain(tools) .collect(); Ok(MessagesCall { - body: messages_body(body, deployment.custom_llm_provider.as_deref())?, + body: messages_body(body)?, api_key: deployment.api_key.clone(), api_base: deployment.api_base.clone(), custom_llm_provider: deployment.custom_llm_provider.clone(), diff --git a/litellm/rust/crates/host-python/AGENTS.md b/litellm/rust/crates/host-python/AGENTS.md index 02b8d4f2dcb..1f7dbca461a 100644 --- a/litellm/rust/crates/host-python/AGENTS.md +++ b/litellm/rust/crates/host-python/AGENTS.md @@ -16,15 +16,15 @@ Boundary tests exercise behavior with a supplied lifecycle binding without impor `HookChain` composes Python runtime hooks in order. Each argument, wire-request and response transformation feeds its result to the next hook. After all argument transformations, the driver calls `arguments_prepared` on every hook in order. Retained callback views must adopt that dictionary before later policy hooks can mutate or reject it. SDK policy is supplied by bridge composition as a hook, never a separate driver phase or parameter. Hooks implement only the stages they need. Default stages preserve the supplied values -Terminal notifications share the selected response or exception. An ordinary notification error is reported as unraisable and does not skip the next hook or replace the selected outcome. Preparation, interception and transformation errors stop the chain. Cancellation stops all further hook dispatch. Suspensions stay inline in the existing driver, and the chain traverses retained event values for GC +Terminal notifications share the selected response or exception. An ordinary notification error is reported as unraisable and does not skip the next hook or replace the selected outcome. Preparation, interception and transformation errors stop the chain. Cancellation stops all further hook dispatch, except on a stream whose head asks for a detached drain: the first cancellation during a read keeps reading to the end, delivers every chunk and the success, then raises the cancellation, and a second cancellation stops it. Suspensions stay inline in the existing driver, and the chain traverses retained event values for GC ## Existing runtime invariants - Keep this crate the CPython runtime adapter and nothing more: Serde marshalling, interpreter detachment, tokio/asyncio glue, the `Execution` handle, the call driver and the `PythonBinding`, `PythonHostCalls` and `PythonOwned` traits, and the `PythonRuntime` specialization of `host::hooks::CallHooks` - No LiteLLM domain dependencies beyond `litellm-host`: no route types, no `Logging` policy, no public API registration, no cdylib build features - `PythonCallHooks` only constrains the shared call-stage interface to `PythonRuntime` and Python ownership. It must not redeclare the stages - - `PythonCallEvent` is a specialization of the shared `CallEvent`, never a separately defined lifecycle. The driver emits `Succeeded` or `Failed` exactly once and never dispatches callbacks after a cancellation. Which Python objects consume those events is the legacy adapter's business - - `CallOptions` can publish snapshots independently of callback delivery. Terminal snapshots follow completed hook dispatch, and cancellation never calls a Python callback. For Python-driven calls, leave the machine's observation publisher unset so the driver is the sole publisher of intercepted provider-response snapshots + - `PythonCallEvent` is a specialization of the shared `CallEvent`, never a separately defined lifecycle. The driver emits `Succeeded` or `Failed` exactly once and never dispatches callbacks after a cancellation, apart from the detached drain above. Which Python objects consume those events is the legacy adapter's business + - `CallOptions` can publish snapshots independently of callback delivery. Terminal snapshots follow completed hook dispatch, and cancellation never calls a Python callback outside a detached drain. For Python-driven calls, leave the machine's observation publisher unset so the driver is the sole publisher of intercepted provider-response snapshots - `PythonBinding::decode_request` receives the keyword view returned by `prepare_arguments` and updated by `arguments_prepared`, not the caller's dict. A binding that decodes from it inherits all composed argument rewrites - A native failure, including one a host op returns as `InvokeError::Native`, is classified exactly once through the binding's `map_error`. A Python exception raised inside the call, and a failure in `prepare_arguments` or `transform_response`, is raised as is - A failing `map_error` is raised with the native error's text as its `__context__`, never swallowed diff --git a/litellm/rust/crates/host-python/src/driver.rs b/litellm/rust/crates/host-python/src/driver.rs index ba0c174c875..541e6e41b0a 100644 --- a/litellm/rust/crates/host-python/src/driver.rs +++ b/litellm/rust/crates/host-python/src/driver.rs @@ -14,7 +14,7 @@ use litellm_host::{ lifecycle::{CallEvent, ExecutionEvent, FailureOrigin, Timing, epoch_seconds}, machine::{HostFailure, Machine, MachineStep}, observation::ObservationSender, - protocol::{HostRequest, InterceptRequest, Protocol, Reply, StreamDelivery}, + protocol::{HostRequest, InterceptRequest, Protocol, Reply, StreamDelivery, UnobservedTail}, }; use crate::PythonHostCalls; @@ -133,6 +133,9 @@ where native: NativeMachine, start: Option, M>>, streaming: bool, + detached: bool, + drains: bool, + unobserved_tail: Option, } impl CallBody @@ -238,6 +241,13 @@ where NativePoll::Ready(result) => result, NativePoll::Suspend(awaitable) => match self.caller.await_python(awaitable).await { Ok(_) => self.native.take_result()?, + Err(error) if self.drains_after(&error) => { + self.detached = true; + Python::attach(|py| { + self.parts().interrupted = Some(error.into_value(py)); + }); + continue; + } Err(error) if self.parts().interrupted.is_none() => { interruption = Some(self.interrupt(error)); continue; @@ -260,6 +270,14 @@ where } } + fn drains_after(&self, error: &PyErr) -> bool { + self.drains + && self.streaming + && !self.detached + && self.parts().interrupted.is_none() + && Python::attach(|py| is_cancellation(py, error)) + } + fn interrupt(&self, error: PyErr) -> HostFailure> { Python::attach(|py| { let cancelled = is_cancellation(py, &error); @@ -320,6 +338,8 @@ where } HostRequest::Stream(StreamDelivery::Open(head, reply)) => { self.streaming = true; + self.drains = ProtocolOf::::detached_drain(&head).is_some(); + self.unobserved_tail = ProtocolOf::::unobserved_tail(&head); let head = { let mut parts = self.parts(); let Parts { binding, hooks, .. } = &mut *parts; @@ -333,16 +353,28 @@ where Ok(()) } HostRequest::Stream(StreamDelivery::Chunk(chunk, reply)) => { + let observed = !self + .unobserved_tail + .as_ref() + .is_some_and(UnobservedTail::begun); let chunk = { let mut parts = self.parts(); let Parts { binding, hooks, .. } = &mut *parts; Python::attach(|py| { let chunk = binding.encode_chunk(py, chunk)?; - hooks.on_stream_chunk(py, &chunk)?; + if observed { + hooks.on_stream_chunk(py, &chunk)?; + } Ok::<_, PyErr>(chunk) })? }; - reply.send(self.caller.yield_chunk(chunk).await); + if self.detached { + reply.send(std::ops::ControlFlow::Break(())); + return Ok(()); + } + let flow = self.caller.yield_chunk(chunk).await; + self.detached = flow.is_break(); + reply.send(flow); Ok(()) } } @@ -384,7 +416,14 @@ where HostedCompletion::Complete(response) => response, HostedCompletion::StreamEnded | HostedCompletion::Detached => { let none = Python::attach(|py| py.None()); - return self.succeeded(none).await; + let succeeded = self.succeeded(none).await; + let cancelled = self.parts().interrupted.take(); + return match cancelled { + Some(cancelled) => Err(Python::attach(|py| { + PyErr::from_value(cancelled.into_bound(py).into_any()) + })), + None => succeeded, + }; } }; let encoded = { @@ -645,6 +684,9 @@ where native: NativeMachine::new(asynchronous), start: Some(Box::new(start)), streaming: false, + detached: false, + drains: false, + unobserved_tail: None, } .run() } diff --git a/litellm/rust/crates/host-python/src/native.rs b/litellm/rust/crates/host-python/src/native.rs index 4e6b59813f9..13168c10e2e 100644 --- a/litellm/rust/crates/host-python/src/native.rs +++ b/litellm/rust/crates/host-python/src/native.rs @@ -67,13 +67,14 @@ where let state = Arc::clone(self.state.as_ref().ok_or(Error::MissingState)?); let future = async move { let mut state = state.lock().await; - let result = match interruption { - Some(failure) => state + let result = match (interruption, state.result.take()) { + (Some(failure), _) => state .machine .interrupt(failure) .await .map(MachineStep::Complete), - None => state.machine.resume().await, + (None, Some(settled)) => settled, + (None, None) => state.machine.resume().await, }; state.result = Some(result); Ok(()) diff --git a/litellm/rust/crates/host-python/tests/driver.rs b/litellm/rust/crates/host-python/tests/driver.rs index 157cfc92148..1f555bcb165 100644 --- a/litellm/rust/crates/host-python/tests/driver.rs +++ b/litellm/rust/crates/host-python/tests/driver.rs @@ -23,8 +23,9 @@ use pyo3::{ use rstest::rstest; use support::{ HookScript, Interpreter, Log, MachineFactory, OpScript, Synthetic, binding, call_options, - failing_machine, handed, hooks, interpreter, lifecycle_module, open_stream, read_all, - run_composed, run_scripted, settled, success_machine, + draining_streaming_machine, failing_machine, handed, hooks, interpreter, lifecycle_module, + open_stream, open_stream_from, read_all, run_composed, run_scripted, settled, success_machine, + tailed_streaming_machine, }; fn observed(capacity: usize) -> (CallOptions, impl FnMut() -> Vec) { @@ -403,6 +404,151 @@ fn a_stream_carries_its_head_before_the_first_chunk( }); } +#[rstest] +fn chunks_on_the_unobserved_tail_reach_the_caller_without_the_stream_hooks( + #[from(interpreter)] _python: Interpreter, + #[values(false, true)] asynchronous: bool, +) { + Python::attach(|py| { + let log = Log::default(); + let stream = open_stream_from( + py, + tailed_streaming_machine(), + hooks(&log, HookScript::Plain), + call_options(asynchronous), + ); + assert_eq!( + read_all(py, &stream, asynchronous), + ["first", "second", "trailer"] + ); + assert_eq!( + log.entries(), + [ + "started", + "begin", + "opened", + "delivered", + "delivered", + "succeeded:None", + "adapter.close" + ] + ); + }); +} + +#[rstest] +fn a_draining_stream_reports_every_chunk_to_the_stream_hooks_after_the_caller_closes( + #[from(interpreter)] _python: Interpreter, + #[values(false, true)] asynchronous: bool, +) { + Python::attach(|py| { + let log = Log::default(); + let stream = open_stream_from( + py, + draining_streaming_machine(true, std::time::Duration::ZERO), + hooks(&log, HookScript::Plain), + call_options(asynchronous), + ); + if asynchronous { + for method in ["__anext__", "aclose"] { + let stop = stream + .call_method0(method) + .unwrap() + .call_method1("send", (py.None(),)) + .unwrap_err(); + assert!(stop.is_instance_of::(py)); + } + } else { + stream.call_method0("__next__").unwrap(); + stream.call_method0("close").unwrap(); + } + assert_eq!( + log.entries(), + [ + "started", + "begin", + "opened", + "delivered", + "delivered", + "delivered", + "succeeded:None", + "adapter.close" + ] + ); + }); +} + +#[rstest] +#[case::draining( + true, + "cancelled", + &["started", "begin", "opened", "delivered", "delivered", "delivered", "succeeded:None", "adapter.close"] +)] +#[case::not_draining(false, "cancelled", &["started", "begin", "opened", "delivered", "adapter.close"])] +fn a_caller_cancelled_mid_read_leaves_only_a_draining_stream_reading_on( + #[from(interpreter)] _python: Interpreter, + #[case] drain: bool, + #[case] outcome: &str, + #[case] logged: &[&str], +) { + Python::attach(|py| { + let log = Log::default(); + let opened = handed( + py, + run_call( + py, + move |_, _, request| { + Ok(draining_streaming_machine( + drain, + std::time::Duration::from_millis(200), + )(request)) + }, + support::StreamingBinding, + hooks(&log, HookScript::Plain), + PyDict::new(py).unbind(), + call_options(true), + ), + true, + ) + .unwrap(); + let locals = PyDict::new(py); + locals.set_item("opened", opened).unwrap(); + py.run( + c" +import asyncio + +async def read_then_cancel(): + stream = await opened + await stream.__anext__() + reading = asyncio.ensure_future(stream.__anext__()) + await asyncio.sleep(0.05) + reading.cancel() + try: + await reading + except asyncio.CancelledError: + outcome = 'cancelled' + else: + outcome = 'read' + await stream.aclose() + return outcome + +outcome = asyncio.run(read_then_cancel()) +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let actual: String = locals + .get_item("outcome") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(actual, outcome); + assert_eq!(log.entries(), logged); + }); +} + struct Classification { provider_fails: bool, op: OpScript, diff --git a/litellm/rust/crates/host-python/tests/support/mod.rs b/litellm/rust/crates/host-python/tests/support/mod.rs index 25b68b782b2..8b9b98d656d 100644 --- a/litellm/rust/crates/host-python/tests/support/mod.rs +++ b/litellm/rust/crates/host-python/tests/support/mod.rs @@ -2,13 +2,15 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use futures_util::StreamExt; + use litellm_host::{ call::{CallOutput, HostedMachine, hosted_call}, hooks::CallHooks, interceptors::{Interceptors, RawResponse, RequestContext, WireRequest}, lifecycle::{ExecutionEvent, Timing}, machine::{CallMachine, MachineFault}, - protocol::{Protocol, Reply}, + protocol::{DetachedDrain, Protocol, Reply, UnobservedTail}, }; use litellm_host_python::{ CallOptions, HookStep, InvokeError, PythonBinding, PythonCallEvent, PythonCallHooks, @@ -559,7 +561,21 @@ impl Protocol for Streaming { type Request = (); type HostCall = std::convert::Infallible; type Chunk = &'static str; - type StreamHead = Vec<(&'static str, &'static str)>; + type StreamHead = StreamingHead; + + fn detached_drain(head: &StreamingHead) -> Option { + head.drain + } + + fn unobserved_tail(head: &StreamingHead) -> Option { + head.tail.clone() + } +} + +pub struct StreamingHead { + headers: Vec<(&'static str, &'static str)>, + tail: Option, + drain: Option, } pub struct StreamingBinding; @@ -576,13 +592,9 @@ impl PythonBinding for StreamingBinding { Ok(()) } - fn encode_stream_head( - &mut self, - py: Python<'_>, - head: Vec<(&'static str, &'static str)>, - ) -> PyResult> { + fn encode_stream_head(&mut self, py: Python<'_>, head: StreamingHead) -> PyResult> { let headers = PyDict::new(py); - for (name, value) in head { + for (name, value) in head.headers { headers.set_item(name, value)?; } let hidden = PyDict::new(py); @@ -629,25 +641,94 @@ pub fn streaming_machine() -> impl FnOnce(()) -> HostedMachine + Send move |()| { hosted_call((), None, |(), _, _, _observations| async { Ok(CallOutput::Stream { - head: vec![("request-id", "req_1")], + head: StreamingHead { + headers: vec![("request-id", "req_1")], + tail: None, + drain: None, + }, chunks: Box::pin(futures_util::stream::iter([Ok("first"), Ok("second")])), }) }) } } +/// Streams `first` and `second`, then `trailer` on its unobserved tail. +pub fn tailed_streaming_machine() -> impl FnOnce(()) -> HostedMachine + Send + Sync { + move |()| { + hosted_call((), None, |(), _, _, _observations| async { + let tail = UnobservedTail::default(); + let begun = tail.clone(); + let trailer = futures_util::stream::once(async move { + begun.begin(); + Ok("trailer") + }); + Ok(CallOutput::Stream { + head: StreamingHead { + headers: Vec::new(), + tail: Some(tail), + drain: None, + }, + chunks: Box::pin(futures_util::StreamExt::chain( + futures_util::stream::iter([Ok("first"), Ok("second")]), + trailer, + )), + }) + }) + } +} + +pub fn draining_streaming_machine( + drain: bool, + delay: std::time::Duration, +) -> impl FnOnce(()) -> HostedMachine + Send + Sync { + move |()| { + hosted_call((), None, move |(), _, _, _observations| async move { + let second = futures_util::stream::once(async move { + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + Ok("second") + }); + Ok(CallOutput::Stream { + head: StreamingHead { + headers: Vec::new(), + tail: None, + drain: drain.then_some(DetachedDrain { + timeout: std::time::Duration::from_secs(5), + limit: 4, + }), + }, + chunks: Box::pin( + futures_util::stream::iter([Ok("first")]) + .chain(second) + .chain(futures_util::stream::iter([Ok("third")])), + ), + }) + }) + } +} + /// Starts a streaming call and returns the caller's stream object. pub fn open_stream( py: Python<'_>, hooks: SyntheticHooks, options: CallOptions, +) -> Bound<'_, PyAny> { + open_stream_from(py, streaming_machine(), hooks, options) +} + +pub fn open_stream_from( + py: Python<'_>, + machine: impl FnOnce(()) -> HostedMachine + Send + Sync + 'static, + hooks: SyntheticHooks, + options: CallOptions, ) -> Bound<'_, PyAny> { let asynchronous = options.asynchronous; let handed = handed( py, run_call( py, - |_, _, request| Ok(streaming_machine()(request)), + move |_, _, request| Ok(machine(request)), StreamingBinding, hooks, PyDict::new(py).unbind(), diff --git a/litellm/rust/crates/host/Cargo.toml b/litellm/rust/crates/host/Cargo.toml index 73392827f31..284c1c5f5f0 100644 --- a/litellm/rust/crates/host/Cargo.toml +++ b/litellm/rust/crates/host/Cargo.toml @@ -13,7 +13,7 @@ litellm-coroutine.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true -tokio = { workspace = true, features = ["sync"] } +tokio = { workspace = true, features = ["sync", "time"] } [dev-dependencies] rstest.workspace = true diff --git a/litellm/rust/crates/host/src/call.rs b/litellm/rust/crates/host/src/call.rs index d8187a216e5..b965a6b6d78 100644 --- a/litellm/rust/crates/host/src/call.rs +++ b/litellm/rust/crates/host/src/call.rs @@ -1,13 +1,58 @@ use crate::observation::ObservationSender; -use std::future::Future; +use std::{ + future::Future, + sync::atomic::{AtomicUsize, Ordering}, +}; use futures_util::{TryStreamExt, stream::BoxStream}; use crate::{ - machine::{CallMachine, ChannelInterceptors, HostServices, MachineFault}, - protocol::Protocol, + machine::{CallMachine, ChannelInterceptors, HostServices, MachineFault, StreamSender}, + protocol::{DetachedDrain, Protocol}, }; +static DETACHED_DRAINS: AtomicUsize = AtomicUsize::new(0); + +struct DrainSlot; + +impl DrainSlot { + fn claim(limit: usize) -> Option { + DETACHED_DRAINS + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |running| { + (running < limit).then_some(running + 1) + }) + .ok() + .map(|_| Self) + } +} + +impl Drop for DrainSlot { + fn drop(&mut self) { + DETACHED_DRAINS.fetch_sub(1, Ordering::AcqRel); + } +} + +async fn drain_detached

( + stream: &StreamSender

, + mut chunks: BoxStream<'static, Result>, + drain: DetachedDrain, +) where + P: Protocol, + P::Error: From, +{ + let Some(_slot) = DrainSlot::claim(drain.limit) else { + return; + }; + let delivered = async { + while let Ok(Some(chunk)) = chunks.try_next().await { + if stream.send_chunk(chunk).await.is_err() { + return; + } + } + }; + let _ = tokio::time::timeout(drain.timeout, delivered).await; +} + pub enum CallOutput { Complete(Response), Stream { @@ -61,11 +106,15 @@ where match execute(request, host.services, host.interceptors, host.observers).await? { CallOutput::Complete(response) => Ok(HostedCompletion::Complete(response)), CallOutput::Stream { head, mut chunks } => { + let drain = P::detached_drain(&head); if host.stream.open_stream(head).await?.is_break() { return Ok(HostedCompletion::Detached); } while let Some(chunk) = chunks.try_next().await? { if host.stream.send_chunk(chunk).await?.is_break() { + if let Some(drain) = drain { + drain_detached(&host.stream, chunks, drain).await; + } return Ok(HostedCompletion::Detached); } } diff --git a/litellm/rust/crates/host/src/protocol.rs b/litellm/rust/crates/host/src/protocol.rs index 5ec566b6a65..8a9bf712543 100644 --- a/litellm/rust/crates/host/src/protocol.rs +++ b/litellm/rust/crates/host/src/protocol.rs @@ -1,4 +1,10 @@ -use std::ops::ControlFlow; +use std::{ + ops::ControlFlow, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; pub use litellm_coroutine::{Abandoned, Answer, Reply, reply}; @@ -11,6 +17,37 @@ pub trait Protocol: Send + Sync + 'static { type HostCall: Send + 'static; type Chunk: Send + 'static; type StreamHead: Send + 'static; + + fn detached_drain(_head: &Self::StreamHead) -> Option { + None + } + + fn unobserved_tail(_head: &Self::StreamHead) -> Option { + None + } +} + +/// Where a stream's chunks stop reaching the stream hooks: once the route begins the tail, the +/// chunks it delivers go to the caller alone, as a frame Python's relay adds after billing does. +#[derive(Clone, Debug, Default)] +pub struct UnobservedTail(Arc); + +impl UnobservedTail { + pub fn begin(&self) { + self.0.store(true, Ordering::Release); + } + + pub fn begun(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + +/// A stream whose upstream is read to its end after the caller detaches, so the provider's +/// terminal usage still reaches the stream hooks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DetachedDrain { + pub timeout: std::time::Duration, + pub limit: usize, } pub enum HostRequest { diff --git a/litellm/rust/crates/host/tests/call.rs b/litellm/rust/crates/host/tests/call.rs index a8ba71e7b07..454c3c9272c 100644 --- a/litellm/rust/crates/host/tests/call.rs +++ b/litellm/rust/crates/host/tests/call.rs @@ -6,6 +6,7 @@ use std::{ Arc, Mutex, atomic::{AtomicUsize, Ordering}, }, + time::Duration, }; use futures_util::{StreamExt, stream}; @@ -13,7 +14,7 @@ use litellm_host::{ call::{CallOutput, HostedCompletion, hosted_call}, lifecycle::{CallEvent, CallObserver, observe_call, observe_unary}, machine::{Machine, MachineFault, MachineStep}, - protocol::{HostRequest, Protocol}, + protocol::{DetachedDrain, HostRequest, Protocol}, }; use rstest::{fixture, rstest}; @@ -94,6 +95,86 @@ async fn delivery_obeys_demand_and_distinguishes_detachment( assert_eq!(delivered, (0..expected_polls).collect::>()); } +struct DrainingProtocol; + +impl Protocol for DrainingProtocol { + type Response = &'static str; + type Error = TestError; + type Request = usize; + type HostCall = Infallible; + type Chunk = usize; + type StreamHead = Option; + + fn detached_drain(head: &Option) -> Option { + *head + } +} + +#[rstest] +#[case::draining( + Some(DetachedDrain { timeout: Duration::from_secs(5), limit: 4 }), + vec![0, 1, 2] +)] +#[case::no_drain(None, vec![0])] +#[case::no_free_slot(Some(DetachedDrain { timeout: Duration::from_secs(5), limit: 0 }), vec![0])] +#[tokio::test] +async fn a_draining_stream_delivers_the_rest_after_the_caller_detaches( + #[case] drain: Option, + #[case] expected: Vec, +) { + let mut machine = + hosted_call::(3, None, move |count, _, _, _| async move { + Ok(CallOutput::Stream { + head: drain, + chunks: stream::iter((0..count).map(Ok)).boxed(), + }) + }); + let mut delivered = Vec::new(); + let completed = loop { + match machine.resume().await.unwrap() { + MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Open(_, reply))) => { + reply.send(ControlFlow::Continue(())); + } + MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Chunk(chunk, reply))) => { + delivered.push(chunk); + reply.send(ControlFlow::Break(())); + } + MachineStep::Complete(result) => break result, + _ => panic!("unexpected operation"), + } + }; + assert_eq!(completed, HostedCompletion::Detached); + assert_eq!(delivered, expected); +} + +#[tokio::test] +async fn a_detached_drain_stops_at_its_timeout() { + let mut machine = + hosted_call::(0, None, move |_, _, _, _| async move { + let pending = stream::iter([Ok(0)]).chain(stream::pending()); + Ok(CallOutput::Stream { + head: Some(DetachedDrain { + timeout: Duration::from_millis(50), + limit: 4, + }), + chunks: pending.boxed(), + }) + }); + let completed = loop { + match machine.resume().await.unwrap() { + MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Open(_, reply))) => { + reply.send(ControlFlow::Continue(())); + } + MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Chunk(_, reply))) => { + reply.send(ControlFlow::Break(())); + } + MachineStep::Complete(result) => break result, + _ => panic!("unexpected operation"), + } + }; + assert_eq!(completed, HostedCompletion::Detached); +} + #[derive(Default)] struct Observer(Observations); struct Observations { diff --git a/litellm/rust/crates/http/src/error.rs b/litellm/rust/crates/http/src/error.rs index d9fdfb5486f..ab3c0a768fd 100644 --- a/litellm/rust/crates/http/src/error.rs +++ b/litellm/rust/crates/http/src/error.rs @@ -52,8 +52,6 @@ pub(crate) enum Repr { RequestUrl(#[source] reqwest::Error), #[error("invalid request URL")] ParseUrl(#[source] url::ParseError), - #[error("request forwards the header {0:?}, which the signer computes")] - ComputedHeader(String), #[error("request signing failed: a streaming body cannot be signed")] UnsignableBody, #[error("request forwards a header that cannot be signed")] @@ -247,10 +245,6 @@ impl Error { } } - pub fn computed_header(name: impl Into) -> Self { - Repr::ComputedHeader(name.into()).into() - } - pub fn request_body(error: serde_json::Error) -> Self { Repr::RequestBody(error).into() } @@ -292,7 +286,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -318,7 +311,6 @@ impl Error { Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableHeader(_) | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -364,7 +356,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -392,51 +383,10 @@ impl Error { } } - pub fn is_computed_header(&self) -> bool { - match &*self.0 { - Repr::ComputedHeader(_) => true, - Repr::Read { .. } - | Repr::InvalidPem { .. } - | Repr::Client(_) - | Repr::TlsConfig(_) - | Repr::RequestBody(_) - | Repr::Request(_) - | Repr::RequestUrl(_) - | Repr::ParseUrl(_) - | Repr::UnsignableBody - | Repr::UnsignableHeader(_) - | Repr::Signature(_) - | Repr::Connect(_) - | Repr::Network(_) - | Repr::BodyCut { .. } - | Repr::Timeout(_) - | Repr::ReadTimeout { .. } - | Repr::Status { .. } - | Repr::WebSocketUrl(_) - | Repr::WebSocketScheme(_) - | Repr::WebSocketSchemeRewrite(_) - | Repr::Handshake(_) - | Repr::HandshakeTimeout(_) - | Repr::WebSocket(_) - | Repr::WebSocketClosed - | Repr::BlockedUrl - | Repr::Resolve(_) - | Repr::DownloadDisabled - | Repr::DownloadTooLarge - | Repr::TooManyRedirects - | Repr::MissingRedirectLocation - | Repr::InvalidRedirect(_) - | Repr::DownloadStatus(_) - | Repr::Download(_) - | Repr::DownloadTimeout(_) => false, - } - } - pub fn is_connect(&self) -> bool { match &*self.0 { Repr::Connect(_) => true, - Repr::ComputedHeader(_) - | Repr::Read { .. } + Repr::Read { .. } | Repr::InvalidPem { .. } | Repr::Client(_) | Repr::TlsConfig(_) @@ -486,7 +436,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -523,7 +472,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -564,7 +512,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -600,7 +547,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) diff --git a/litellm/rust/crates/http/src/outbound.rs b/litellm/rust/crates/http/src/outbound.rs index 49ff7fa6df0..7636eff0f76 100644 --- a/litellm/rust/crates/http/src/outbound.rs +++ b/litellm/rust/crates/http/src/outbound.rs @@ -19,6 +19,10 @@ use crate::{Error, error::Repr}; /// Returns the headers to add to the request; it never sees a mutable request. pub trait RequestSigner: Send + Sync { fn sign(&self, request: &Request) -> Result; + + fn owned_headers(&self) -> &'static [&'static str] { + &[] + } } pub fn json( @@ -65,7 +69,11 @@ pub fn from_http>(request: http::Request) -> Result Result { let signature = signer.sign(&request)?; - request.headers_mut().extend(signature); + let headers = request.headers_mut(); + for name in signer.owned_headers() { + headers.remove(*name); + } + headers.extend(signature); Ok(request) } @@ -159,7 +167,7 @@ mod tests { struct Refuses; impl RequestSigner for Refuses { fn sign(&self, _request: &Request) -> Result { - Err(Error::computed_header("authorization")) + Err(Error::unsignable_body()) } } @@ -168,7 +176,44 @@ mod tests { &Refuses, ) .unwrap_err(); - assert!(error.is_computed_header()); + assert!(!error.dispatched()); + } + + #[test] + fn a_header_the_signer_owns_is_dropped_even_when_the_signer_omits_it() { + struct OwnsDate; + impl RequestSigner for OwnsDate { + fn sign(&self, _request: &Request) -> Result { + Ok(HeaderMap::from_iter([( + reqwest::header::AUTHORIZATION, + HeaderValue::from_static("signed"), + )])) + } + + fn owned_headers(&self) -> &'static [&'static str] { + &["date", "authorization"] + } + } + + let request = signed( + json( + "https://u.test/", + &[ + ("Date".into(), "caller".into()), + ("Authorization".into(), "caller".into()), + ("x-kept".into(), "1".into()), + ], + &json!({}), + None, + ) + .unwrap(), + &OwnsDate, + ) + .unwrap(); + + assert!(!request.headers().contains_key("date")); + assert_eq!(request.headers()["authorization"], "signed"); + assert_eq!(request.headers()["x-kept"], "1"); } #[rstest] diff --git a/litellm/rust/crates/inference-chat/tests/request.rs b/litellm/rust/crates/inference-chat/tests/request.rs index 121f7347c24..d9efbc68419 100644 --- a/litellm/rust/crates/inference-chat/tests/request.rs +++ b/litellm/rust/crates/inference-chat/tests/request.rs @@ -328,13 +328,23 @@ async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() { } #[rstest] -#[case::authorization("Authorization")] -#[case::amz_date("x-amz-date")] -#[case::security_token("x-amz-security-token")] -#[case::date("Date")] +#[case::authorization("Authorization", Some("forged"), false, None)] +#[case::amz_date("x-amz-date", None, false, Some("x-amz-date"))] +#[case::security_token( + "x-amz-security-token", + None, + true, + Some("x-amz-date;x-amz-security-token") +)] +#[case::date("Date", None, false, Some("content-type;date;host"))] #[tokio::test] -async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) { - let upstream = wiremock::MockServer::start().await; +async fn a_forwarded_header_the_signer_computes_is_sent_as_python_sends_it( + #[case] forwarded: &str, + #[case] authorization: Option<&str>, + #[case] kept: bool, + #[case] signed_headers: Option<&str>, +) { + let upstream = upstream([bedrock_response()]).await; let base = upstream.uri(); let call = ChatCompletionsRequest { api_key: None, @@ -351,11 +361,27 @@ async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) }), ) }; - let error = complete(call) - .await - .expect_err("conflicting signing headers fail"); - assert!(error.declined().is_some(), "{error:?}"); - assert!(received(&upstream).await.is_empty()); + complete(call).await.expect("call succeeds"); + + let sent = only_request(&upstream).await; + let signature = sent.header("authorization").expect("an authorization"); + match authorization { + Some(expected) => assert_eq!(signature, expected), + None => { + assert!(signature.starts_with("AWS4-HMAC-SHA256")); + assert_eq!(sent.header(forwarded) == Some("forged"), kept); + assert!(signed_headers.is_some_and(|names| signature.contains(names))); + } + } + let dated_by_date = forwarded == "Date"; + assert_eq!( + ( + sent.header("x-amz-date") + .is_some_and(|date| date != "forged"), + sent.header("date").is_some_and(|date| date != "forged"), + ), + (!dated_by_date, dated_by_date) + ); } #[rstest] diff --git a/litellm/rust/crates/inference-messages/Cargo.toml b/litellm/rust/crates/inference-messages/Cargo.toml index 0ca371c3ce7..ed84b0f864d 100644 --- a/litellm/rust/crates/inference-messages/Cargo.toml +++ b/litellm/rust/crates/inference-messages/Cargo.toml @@ -20,6 +20,7 @@ litellm-http.workspace = true litellm-inference.workspace = true litellm-llms.workspace = true litellm-llms-types.workspace = true +litellm-python-compat.workspace = true litellm-router-types.workspace = true litellm-secrets.workspace = true litellm-tracing.workspace = true diff --git a/litellm/rust/crates/inference-messages/src/common_utils.rs b/litellm/rust/crates/inference-messages/src/common_utils.rs index d0f109c1e99..4f86ff42f06 100644 --- a/litellm/rust/crates/inference-messages/src/common_utils.rs +++ b/litellm/rust/crates/inference-messages/src/common_utils.rs @@ -52,11 +52,15 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option &TENCENT_MESSAGES_CONFIG, LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG, LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG, - LlmProviders::Bedrock if model.starts_with("claude_platform/") => { + LlmProviders::Bedrock if has_route_prefix(model, "claude_platform/") => { &BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG } - LlmProviders::Bedrock if model.starts_with("mantle/") => &BEDROCK_MANTLE_MESSAGES_CONFIG, - LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG, + LlmProviders::Bedrock if has_route_prefix(model, "converse/") => return None, + LlmProviders::Bedrock if has_route_prefix(model, "mantle/") => { + &BEDROCK_MANTLE_MESSAGES_CONFIG + } + LlmProviders::Bedrock if model.contains("claude") => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG, + LlmProviders::Bedrock => return None, LlmProviders::BedrockMantle => &BEDROCK_MANTLE_MESSAGES_CONFIG, LlmProviders::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG, LlmProviders::Edenai => &EDENAI_MESSAGES_CONFIG, @@ -70,6 +74,10 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option bool { + model.starts_with(prefix) || model.contains(&format!("/{prefix}")) +} + pub(super) fn string_headers( extra_headers: Option>, ) -> Result, Error> { @@ -155,6 +163,72 @@ mod tests { assert!(messages_provider(provider, model).is_none()); } + #[rstest] + #[case::invoke_claude("anthropic.claude-sonnet-4-5-20250929-v1:0", Some("bedrock"))] + #[case::invoke_prefix("invoke/anthropic.claude-3-5-sonnet", Some("bedrock"))] + #[case::cross_region_claude("us.anthropic.claude-opus-4-5", Some("bedrock"))] + #[case::claude_arn( + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5", + Some("bedrock") + )] + #[case::converse_claude("converse/anthropic.claude-sonnet-4-5", None)] + #[case::nested_converse("us-west-2/converse/anthropic.claude-sonnet-4-5", None)] + #[case::non_claude("amazon.nova-pro-v1:0", None)] + #[case::non_claude_invoke("invoke/meta.llama3-70b-instruct-v1:0", None)] + #[case::capitalized_claude_is_not_claude("anthropic.Claude-x", None)] + #[case::mantle("mantle/anthropic.claude-sonnet-5", Some("bedrock"))] + #[case::claude_platform("claude_platform/claude-sonnet-4-5", Some("bedrock"))] + #[case::bedrock_mantle_provider_prefix_is_not_the_mantle_route( + "bedrock_mantle/openai.gpt-5", + None + )] + fn bedrock_models_route_as_python_selects_their_messages_config( + #[case] model: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!( + messages_provider(LlmProviders::Bedrock, model).map(|provider| provider.as_str()), + expected + ); + } + + #[rstest] + #[case::invoke("anthropic.claude-3", false, false)] + #[case::mantle("mantle/anthropic.claude-sonnet-5", true, false)] + #[case::claude_platform("claude_platform/claude-sonnet-4-5", false, true)] + fn bedrock_route_prefixes_pick_their_config( + #[case] model: &str, + #[case] mantle: bool, + #[case] claude_platform: bool, + ) { + use litellm_llms::{ + base_llm::messages::transformation::BaseMessagesConfig, + bedrock::claude_platform::messages::transformation::BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG, + bedrock_mantle::messages::transformation::BEDROCK_MANTLE_MESSAGES_CONFIG, + }; + let params = litellm_router_types::LitellmParams::default(); + let region = |key: &str| { + key.starts_with("AWS_REGION") + .then(|| "us-east-1".to_owned()) + }; + let url = |config: &dyn BaseMessagesConfig| { + format!( + "{:?}", + config.get_complete_url(None, model, ¶ms, false, ®ion) + ) + }; + let routed = url(messages_provider(LlmProviders::Bedrock, model) + .unwrap() + .config()); + assert_eq!( + ( + routed == url(&BEDROCK_MANTLE_MESSAGES_CONFIG), + routed == url(&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG), + ), + (mantle, claude_platform) + ); + } + #[rstest] #[case::int(json!(12345), "int")] #[case::float(json!(1.5), "float")] diff --git a/litellm/rust/crates/inference-messages/src/handler.rs b/litellm/rust/crates/inference-messages/src/handler.rs index 2cab709681f..f44ae935da3 100644 --- a/litellm/rust/crates/inference-messages/src/handler.rs +++ b/litellm/rust/crates/inference-messages/src/handler.rs @@ -9,15 +9,18 @@ use litellm_core_utils::{ dot_notation_indexing::delete_nested_value, get_provider_specific_headers::get_provider_specific_headers, settings::Lookup, }; -use litellm_host::interceptors::{ - Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext, - ResponseHead, ResultSource, WireRequest, +use litellm_host::{ + interceptors::{ + Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext, + ResponseHead, ResultSource, WireRequest, + }, + protocol::UnobservedTail, }; use litellm_llms::base_llm::{ auth::{Authenticated, AzureAdTokenHosts, ValidatedEnvironment, resolve_auth}, messages::{ context::MessagesTransformContext, - streaming::{self, ByteStream, StreamDecoder}, + streaming::{ByteStream, StreamDecoder}, transformation::{BaseMessagesConfig, FederatedEnvironment}, }, }; @@ -53,12 +56,38 @@ struct ProviderCall { stream: bool, whole_sse_frames: bool, joined_reads: bool, + detached_drain: Option, transform_context: MessagesTransformContext, + native_output_fallback: Option, ascii_response_headers: bool, unencodable_request_header: Option>, ascii_request_headers: bool, } +struct NativeOutputFallback { + input: MessagesRequest, + context: MessagesTransformContext, + headers: Vec<(String, String)>, + tool_custom_field: litellm_llms::base_llm::messages::tools::ToolCustomField, + drops_unsupported_params: bool, + drop_params: bool, +} + +impl NativeOutputFallback { + fn body(&self, config: &dyn BaseMessagesConfig) -> Result { + let rebuilt = config.request_body( + self.tool_custom_field.apply( + config.transform_anthropic_messages_request(self.input.clone(), &self.context)?, + self.drops_unsupported_params, + ), + &self.headers, + self.drop_params, + )?; + let (rebuilt, _) = config.finalize_request(rebuilt, self.headers.clone(), &self.context)?; + Ok(config.wire_body(serde_json::to_value(&rebuilt).map_err(serialize_failure)?)) + } +} + impl ProviderCall { fn cacheable(&self) -> bool { self.signer.is_none() @@ -290,7 +319,7 @@ async fn provider_call( )?; let transform_context = MessagesTransformContext { drops_unsupported_params: shaping.settings.drops_unsupported_params, - logs_dropped_params: config.logs_dropped_params(), + global_drop_params: shaping.settings.global_drop_params, ..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params) }; let dropped_keys = top_level_paths(&shaping.settings.additional_drop_params); @@ -358,6 +387,9 @@ async fn provider_call( let tool_custom_field = config.tool_custom_field(&trimmed.model); let requested_output_format = trimmed.params.output_format.clone(); + let fallback = config + .native_output_fallback(&trimmed, &transform_context) + .map(|fallback_context| (trimmed.clone(), fallback_context)); let body = config.request_body( tool_custom_field.apply( config.transform_anthropic_messages_request(trimmed, &transform_context)?, @@ -373,17 +405,26 @@ async fn provider_call( body.params.streams(), &env_lookup, )?; - let environment = ValidatedEnvironment { - headers: config.headers_for_url( - config.request_headers( - with_python_default_headers(validated.headers, config.default_headers()), - &with_requested_output_format(&body, requested_output_format), - ), - &url, + let headers = config.headers_for_url( + config.request_headers( + with_python_default_headers(validated.headers, config.default_headers()), + &with_requested_output_format(&body, requested_output_format), ), + &url, + ); + let native_output_fallback = fallback.map(|(input, context)| NativeOutputFallback { + input, + context, + headers: headers.clone(), + tool_custom_field, + drops_unsupported_params: shaping.settings.drops_unsupported_params, + drop_params: shaping.settings.drop_params, + }); + let (body, headers) = config.finalize_request(body, headers, &transform_context)?; + let environment = ValidatedEnvironment { + headers, auth: validated.auth, }; - let body = config.finalize_request(body, &environment.headers, &transform_context); let request_context = RequestContext { model: body.model.clone(), @@ -392,6 +433,11 @@ async fn provider_call( secret_fields: Vec::new(), api_key: api_key.map(SecretValue::new), }; + let serialized = serde_json::to_value(&body).map_err(serialize_failure)?; + let serialized_stream = serialized.get("stream").is_some(); + let wire_body = config.wire_body(serialized); + let stream_moved_to_url = + body.params.streams() && serialized_stream && wire_body.get("stream").is_none(); let authenticated = resolve_auth(credentials.auth, environment, &|key| { std::env::var(key).ok() }) @@ -406,15 +452,15 @@ async fn provider_call( WireRequest { url, headers: joined_header_values(authenticated.headers), - body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?), + body: wire_body, }, request_context, ) .await?; - let stream = wire - .body - .get("stream") - .is_some_and(litellm_llms_types::recognized::python_truthy); + let stream = wire.body.get("stream").map_or( + stream_moved_to_url, + litellm_llms_types::recognized::python_truthy, + ); Ok(ProviderCall { identity, wire, @@ -424,7 +470,9 @@ async fn provider_call( stream, whole_sse_frames: shaping.settings.whole_sse_frames, joined_reads: shaping.settings.joined_reads, + detached_drain: shaping.settings.detached_drain(), transform_context, + native_output_fallback, ascii_response_headers: shaping.settings.ascii_response_headers, unencodable_request_header: shaping.settings.unencodable_request_header, ascii_request_headers: shaping.settings.ascii_request_headers, @@ -498,6 +546,13 @@ fn requested_optional_params( if provider.accepts_service_tier() { return Ok(request); } + if request.params.service_tier.is_some() { + litellm_tracing::warn!( + "Dropping `service_tier` for model={} on provider={}. Only the Anthropic API routes accept it.", + request.model, + provider.as_str() + ); + } Ok(MessagesRequest { params: MessagesOptionalParams { service_tier: None, @@ -560,6 +615,7 @@ async fn call_provider( stream, whole_sse_frames, joined_reads, + detached_drain, .. } = request; let provider_name = provider.as_str(); @@ -581,6 +637,7 @@ async fn call_provider( timeout, joined_reads, ); + let streamed = draining_after_detach(streamed, config.stream_decoder(), detached_drain); return Ok(if whole_sse_frames { framed(streamed) } else { @@ -616,6 +673,7 @@ async fn hosted_response( identity, provider, whole_sse_frames, + detached_drain, .. } = request; let provider_name = provider.as_str(); @@ -633,6 +691,7 @@ async fn hosted_response( .await?; let streamed = hosted_streaming_response(headers, chunks, config.stream_decoder(), provider_name); + let streamed = draining_after_detach(streamed, config.stream_decoder(), detached_drain); return Ok(if whole_sse_frames { framed(streamed) } else { @@ -706,6 +765,7 @@ async fn send_recovering( let config = request.provider.config(); let mut body = std::borrow::Cow::Borrowed(&request.wire.body); let mut attempt = 1; + let mut recovered: Vec<(u16, String)> = Vec::new(); loop { log_request_body(request.provider.as_str(), request.stream, &body); if let Some(value) = &request.unencodable_request_header { @@ -779,6 +839,20 @@ async fn send_recovering( } hosted @ Upstream::Hosted(_) => return Ok((hosted, sent_body)), }; + if let Some(fallback) = request.native_output_fallback.as_ref().filter(|_| { + config.retries_without_native_output( + failure.status, + &failure.text, + &body, + &request.identity.model, + ) + }) { + let rebuilt = fallback.body(config)?; + let replayed = replayed_recoveries(config, rebuilt, &recovered, request); + body = std::borrow::Cow::Owned(replayed.body.clone()); + context.provider_request_retried(replayed).await?; + continue; + } let retried = if attempt == MESSAGES_HTTP_ERROR_ATTEMPTS { None } else { @@ -793,6 +867,7 @@ async fn send_recovering( return Err(failure.into()); }; attempt += 1; + recovered.push((failure.status, failure.text.clone())); tracing::debug!( "Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt {attempt}/{MESSAGES_HTTP_ERROR_ATTEMPTS})." ); @@ -801,6 +876,36 @@ async fn send_recovering( } } +fn replayed_recoveries( + config: &dyn BaseMessagesConfig, + rebuilt: Value, + recovered: &[(u16, String)], + request: &ProviderCall, +) -> litellm_host::interceptors::RetriedRequest { + recovered.iter().fold( + litellm_host::interceptors::RetriedRequest { + reasons: Vec::new(), + body: rebuilt, + }, + |replayed, (status, text)| match config.retry_after_http_error( + *status, + text, + &replayed.body, + &request.transform_context, + ) { + Some(retried) => litellm_host::interceptors::RetriedRequest { + reasons: replayed + .reasons + .into_iter() + .chain(retried.reasons) + .collect(), + body: retried.body, + }, + None => replayed, + }, + ) +} + fn serialize_failure(err: serde_json::Error) -> Error { Error::invalid_request(litellm_llms::ErrorDetail::failed( "Anthropic messages request serialization", @@ -972,16 +1077,23 @@ fn streaming_response( _ => chunks.boxed(), } .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); - let chunks = match decoder { + let unobserved_tail = decoder.map(|_| UnobservedTail::default()); + let chunks = match decoder.zip(unobserved_tail.clone()) { None => chunks.map_err(Error::from).boxed(), - Some(decode) => decoded_chunks( + Some((decode, tail)) => decoded_chunks( chunks.map_err(std::io::Error::other).boxed(), decode, + headers.clone(), provider, + tail, ), }; MessagesCallResponse::Stream { - head: super::route::MessagesStreamHead { headers }, + head: super::route::MessagesStreamHead { + headers, + detached_drain: None, + unobserved_tail, + }, chunks, } } @@ -993,20 +1105,44 @@ fn hosted_streaming_response( provider: &'static str, ) -> MessagesCallResponse { let chunks = chunks.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); - let chunks = match decoder { + let unobserved_tail = decoder.map(|_| UnobservedTail::default()); + let chunks = match decoder.zip(unobserved_tail.clone()) { None => chunks.boxed(), - Some(decode) => decoded_chunks( + Some((decode, tail)) => decoded_chunks( chunks.map_err(std::io::Error::other).boxed(), decode, + headers.clone(), provider, + tail, ), }; MessagesCallResponse::Stream { - head: super::route::MessagesStreamHead { headers }, + head: super::route::MessagesStreamHead { + headers, + detached_drain: None, + unobserved_tail, + }, chunks, } } +fn draining_after_detach( + response: MessagesCallResponse, + decoder: Option, + detached_drain: Option, +) -> MessagesCallResponse { + match response { + MessagesCallResponse::Stream { head, chunks } => MessagesCallResponse::Stream { + head: super::route::MessagesStreamHead { + detached_drain: decoder.and(detached_drain), + ..head + }, + chunks, + }, + complete => complete, + } +} + fn framed(response: MessagesCallResponse) -> MessagesCallResponse { let MessagesCallResponse::Stream { head, chunks } = response else { return response; @@ -1056,17 +1192,14 @@ fn complete_frames_end(pending: &[u8]) -> usize { fn decoded_chunks( bytes: ByteStream, decode: StreamDecoder, + headers: Vec<(String, String)>, provider: &'static str, + tail: UnobservedTail, ) -> BoxStream<'static, Result> { - futures_util::stream::try_unfold(decode(bytes), move |mut events| async move { - let Some(event) = events.try_next().await? else { - return Ok(None); - }; - let chunk = streaming::encode(&event)?; - log_chunk(provider, "client_response", &chunk); - Ok(Some((chunk, events))) - }) - .boxed() + let frames = decode(bytes, headers).map_err(Error::from).boxed(); + super::relay::relayed(frames, tail) + .inspect_ok(move |chunk| log_chunk(provider, "client_response", chunk)) + .boxed() } fn log_request_body(provider: &str, stream: bool, body: &serde_json::Value) { @@ -1112,6 +1245,7 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) { #[cfg(test)] mod tests { + use litellm_llms::base_llm::messages::streaming; use litellm_llms_types::headers::ProviderSpecificHeaders; use rstest::{fixture, rstest}; use serde_json::{Map, Value, json}; @@ -1849,8 +1983,8 @@ mod tests { #[rstest] #[case::event( - "data: {\"type\":\"ping\"}\n\n", - Some("event: ping\ndata: {\"type\":\"ping\"}\n\n") + "data: {\"type\":\"message_stop\"}\n\n", + Some("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") )] #[rstest::rstest] #[case::invalid_event("data: invalid\n\ndata: {\"type\":\"ping\"}\n\n", None)] @@ -1871,7 +2005,11 @@ mod tests { .unwrap(); let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response( response, - Some(streaming::event_stream), + Some(|bytes, _| { + streaming::event_stream(bytes) + .and_then(|event| futures_util::future::ready(streaming::encode(&event))) + .boxed() + }), "test", Duration::from_secs(5), false, @@ -1887,6 +2025,78 @@ mod tests { assert!(chunks.next().await.is_none()); } + async fn upstream_stream(body: &'static str) -> reqwest::Response { + let upstream = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(200).set_body_raw(body, "text/event-stream")) + .mount(&upstream) + .await; + litellm_http::Client::for_test() + .get(upstream.uri()) + .send() + .await + .unwrap() + } + + fn reencoding(bytes: ByteStream, _: Vec<(String, String)>) -> streaming::FrameStream { + streaming::event_stream(bytes) + .and_then(|event| futures_util::future::ready(streaming::encode(&event))) + .boxed() + } + + #[rstest] + #[case::decoded_and_cut_short( + Some(reencoding as StreamDecoder), + "data: {\"type\":\"ping\"}\n\n", + true + )] + #[case::decoded_and_stopped( + Some(reencoding as StreamDecoder), + "data: {\"type\":\"ping\"}\n\ndata: {\"type\":\"message_stop\"}\n\n", + false + )] + #[case::passed_through_and_cut_short(None, "data: {\"type\":\"ping\"}\n\n", false)] + #[tokio::test] + async fn a_decoded_stream_cut_short_ends_with_the_incomplete_stream_error_on_its_tail( + #[case] decoder: Option, + #[case] body: &'static str, + #[case] incomplete: bool, + ) { + let relays = decoder.is_some(); + let MessagesCallResponse::Stream { head, mut chunks } = streaming_response( + upstream_stream(body).await, + decoder, + "test", + Duration::from_secs(5), + false, + ) else { + panic!("a streaming response returns chunks"); + }; + let tail = head.unobserved_tail; + assert_eq!(tail.is_some(), relays); + let mut delivered = Vec::new(); + let mut on_tail = Vec::new(); + while let Some(chunk) = chunks.try_next().await.unwrap() { + match tail.as_ref().is_some_and(UnobservedTail::begun) { + true => on_tail.push(chunk), + false => delivered.push(chunk), + } + } + let error_event = Bytes::from(format!( + "event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{}\"}}}}\n\n", + super::super::relay::INCOMPLETE_STREAM_ERROR_MESSAGE + )); + assert!(!delivered.contains(&error_event)); + assert_eq!( + on_tail, + if incomplete { + vec![error_event] + } else { + Vec::new() + } + ); + } + #[rstest] #[case::no_delimiter(b"data: a".as_slice(), 0)] #[case::lf(b"data: a\n\ndata: b", 9)] diff --git a/litellm/rust/crates/inference-messages/src/lib.rs b/litellm/rust/crates/inference-messages/src/lib.rs index a932fcb3d2d..5a1a725a772 100644 --- a/litellm/rust/crates/inference-messages/src/lib.rs +++ b/litellm/rust/crates/inference-messages/src/lib.rs @@ -5,6 +5,7 @@ mod handler; mod host_answer; mod pre_request; mod reads; +mod relay; pub mod route; mod safeguards; mod transport; diff --git a/litellm/rust/crates/inference-messages/src/relay.rs b/litellm/rust/crates/inference-messages/src/relay.rs new file mode 100644 index 00000000000..1b5022e95d9 --- /dev/null +++ b/litellm/rust/crates/inference-messages/src/relay.rs @@ -0,0 +1,202 @@ +use bytes::Bytes; +use futures_util::{StreamExt, TryStreamExt, stream::BoxStream}; +use litellm_host::protocol::UnobservedTail; +use litellm_python_compat::{Value, json}; + +use super::Error; + +pub(crate) const INCOMPLETE_STREAM_ERROR_MESSAGE: &str = "Provider stream ended before emitting a message_stop event; the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated."; + +type Frames = BoxStream<'static, Result>; + +/// `_incomplete_stream_error_sse_event`. +fn incomplete_stream_error_event() -> Bytes { + Bytes::from(format!( + "event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{INCOMPLETE_STREAM_ERROR_MESSAGE}\"}}}}\n\n" + )) +} + +/// `bytes.splitlines()`. +fn lines(chunk: &[u8]) -> impl Iterator { + let mut rest = Some(chunk); + std::iter::from_fn(move || { + let current = rest.filter(|current| !current.is_empty())?; + match current + .iter() + .position(|byte| matches!(byte, b'\n' | b'\r')) + { + Some(at) => { + let skip = if current[at..].starts_with(b"\r\n") { + 2 + } else { + 1 + }; + rest = Some(¤t[at + skip..]); + Some(¤t[..at]) + } + None => { + rest = None; + Some(current) + } + } + }) +} + +fn field<'a>(fields: &'a [(Value, Value)], name: &str) -> Option<&'a Value> { + fields + .iter() + .find(|(key, _)| matches!(key, Value::Str(key) if key == name)) + .map(|(_, value)| value) +} + +/// `_decoded_sse_data_line`. +fn decoded_data_line(line: &[u8]) -> Option { + let data = line.strip_prefix(b"data:")?.trim_ascii(); + json::loads(std::str::from_utf8(data).ok()?).ok() +} + +/// `is_message_stop_chunk` for a chunk of SSE bytes. +fn is_message_stop_chunk(chunk: &[u8]) -> bool { + lines(chunk).any(|line| line == b"event: message_stop") +} + +/// `is_provider_error_chunk` for a chunk of SSE bytes. +fn is_provider_error_chunk(chunk: &[u8]) -> bool { + lines(chunk) + .filter_map(decoded_data_line) + .find_map(|candidate| match candidate { + Value::Dict(fields) + if matches!(field(&fields, "type"), Some(Value::Str(kind)) if kind == "error") => + { + Some(fields) + } + _ => None, + }) + .is_some_and(|fields| matches!(field(&fields, "error"), Some(Value::Dict(_)))) +} + +/// `_is_terminal_stream_chunk`. +fn is_terminal_stream_chunk(chunk: &[u8]) -> bool { + is_message_stop_chunk(chunk) || is_provider_error_chunk(chunk) +} + +struct Relay { + frames: Option, + terminal: bool, + tail: UnobservedTail, +} + +/// A decoded provider stream as `BaseAnthropicMessagesStreamingIterator.async_sse_wrapper` +/// relays it: one that ends without a `message_stop` or provider error event closes with an +/// `api_error` event. Python adds that event after billing, so it goes out on `tail`, which the +/// stream hooks never observe. +pub(crate) fn relayed(frames: Frames, tail: UnobservedTail) -> Frames { + let relay = Relay { + frames: Some(frames), + terminal: false, + tail, + }; + futures_util::stream::try_unfold(relay, |mut relay| async move { + let Some(frames) = relay.frames.as_mut() else { + return Ok(None); + }; + if let Some(frame) = frames.try_next().await? { + relay.terminal |= is_terminal_stream_chunk(&frame); + return Ok(Some((frame, relay))); + } + relay.frames = None; + if relay.terminal { + return Ok(None); + } + relay.tail.begin(); + Ok(Some((incomplete_stream_error_event(), relay))) + }) + .boxed() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const START: &str = "event: message_start\ndata: {\"type\": \"message_start\"}\n\n"; + const STOP: &str = "event: message_stop\ndata: {\"type\": \"message_stop\"}\n\n"; + const PROVIDER_ERROR: &str = "event: error\ndata: {\"type\": \"error\", \"error\": {\"type\": \"overloaded_error\"}}\n\n"; + const UNSHAPED_ERROR: &str = + "event: error\ndata: {\"type\": \"error\", \"error\": \"overloaded\"}\n\n"; + + fn frames(items: Vec>) -> Frames { + futures_util::stream::iter(items.into_iter().map(|item| item.map(Bytes::from))).boxed() + } + + async fn relayed_frames( + items: Vec>, + ) -> (Vec>, UnobservedTail) { + let tail = UnobservedTail::default(); + let relayed = relayed(frames(items), tail.clone()).collect().await; + (relayed, tail) + } + + #[rstest] + #[case::message_stop(vec![START, STOP])] + #[case::provider_error(vec![START, PROVIDER_ERROR])] + #[case::stop_then_more(vec![START, STOP, START])] + #[tokio::test] + async fn a_terminated_stream_is_relayed_as_it_is(#[case] sent: Vec<&'static str>) { + let (relayed, tail) = relayed_frames(sent.iter().copied().map(Ok).collect()).await; + let relayed: Vec = relayed.into_iter().map(Result::unwrap).collect(); + assert_eq!( + relayed, + sent.into_iter().map(Bytes::from).collect::>() + ); + assert!(!tail.begun()); + } + + #[rstest] + #[case::empty(vec![])] + #[case::cut_short(vec![START])] + #[case::error_without_an_error_object(vec![START, UNSHAPED_ERROR])] + #[tokio::test] + async fn an_unterminated_stream_ends_with_the_incomplete_stream_error_on_the_tail( + #[case] sent: Vec<&'static str>, + ) { + let (relayed, tail) = relayed_frames(sent.iter().copied().map(Ok).collect()).await; + let mut relayed: Vec = relayed.into_iter().map(Result::unwrap).collect(); + assert_eq!( + relayed.pop().unwrap(), + Bytes::from(format!( + "event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{INCOMPLETE_STREAM_ERROR_MESSAGE}\"}}}}\n\n" + )) + ); + assert_eq!( + relayed, + sent.into_iter().map(Bytes::from).collect::>() + ); + assert!(tail.begun()); + } + + #[tokio::test] + async fn a_failed_stream_ends_with_its_failure() { + let failure = Error::from(litellm_http::Error::upstream(500, "cut".to_owned())); + let (relayed, tail) = relayed_frames(vec![Ok(START), Err(failure)]).await; + assert_eq!(relayed.len(), 2); + assert_eq!(relayed[0].as_ref().unwrap(), START.as_bytes()); + assert!(relayed[1].is_err()); + assert!(!tail.begun()); + } + + #[rstest] + #[case::crlf("event: message_stop\r\ndata: {}\r\n\r\n", true)] + #[case::cr("event: ping\revent: message_stop\r", true)] + #[case::unspaced_event("event:message_stop\n\n", false)] + #[case::error_data_padded("data: {\"type\": \"error\", \"error\": {}} \n\n", true)] + #[case::first_error_payload_decides( + "data: {\"type\": \"error\", \"error\": 1}\ndata: {\"type\": \"error\", \"error\": {}}\n\n", + false + )] + #[case::undecodable_data("data: {\n\n", false)] + fn terminal_chunks_are_python_s(#[case] chunk: &str, #[case] terminal: bool) { + assert_eq!(is_terminal_stream_chunk(chunk.as_bytes()), terminal); + } +} diff --git a/litellm/rust/crates/inference-messages/src/route.rs b/litellm/rust/crates/inference-messages/src/route.rs index fa850b23bf2..054bd0585ab 100644 --- a/litellm/rust/crates/inference-messages/src/route.rs +++ b/litellm/rust/crates/inference-messages/src/route.rs @@ -13,6 +13,8 @@ pub type MessagesOutput = HostedCompletion; /// The upstream response as the caller sees it at stream hand-off, before any chunk. pub struct MessagesStreamHead { pub headers: Vec<(String, String)>, + pub detached_drain: Option, + pub unobserved_tail: Option, } pub struct Messages; @@ -24,6 +26,16 @@ impl Protocol for Messages { type HostCall = Infallible; type Chunk = Bytes; type StreamHead = MessagesStreamHead; + + fn detached_drain(head: &MessagesStreamHead) -> Option { + head.detached_drain + } + + fn unobserved_tail( + head: &MessagesStreamHead, + ) -> Option { + head.unobserved_tail.clone() + } } pub type MessagesMachine = HostedMachine; @@ -66,6 +78,8 @@ impl litellm_inference::caching::StreamCachable for Messages { Some(litellm_host::call::CallOutput::Stream { head: MessagesStreamHead { headers: Vec::new(), + detached_drain: None, + unobserved_tail: None, }, chunks: Box::pin(futures_util::stream::iter([Ok(data)])), }) diff --git a/litellm/rust/crates/inference-messages/src/safeguards.rs b/litellm/rust/crates/inference-messages/src/safeguards.rs index 78e4a752536..4511fa7a241 100644 --- a/litellm/rust/crates/inference-messages/src/safeguards.rs +++ b/litellm/rust/crates/inference-messages/src/safeguards.rs @@ -168,6 +168,8 @@ mod tests { MessagesCallResponse::Stream { head: MessagesStreamHead { headers: Vec::new(), + detached_drain: None, + unobserved_tail: None, }, chunks: futures_util::stream::iter(chunks).boxed(), } diff --git a/litellm/rust/crates/inference-messages/src/types.rs b/litellm/rust/crates/inference-messages/src/types.rs index f265a9edc2b..a5b392a2c72 100644 --- a/litellm/rust/crates/inference-messages/src/types.rs +++ b/litellm/rust/crates/inference-messages/src/types.rs @@ -1,15 +1,13 @@ use std::time::Duration; use bytes::Bytes; -use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_host::call::CallOutput; -use litellm_inference::provider::resolve_llm_provider; use litellm_llms::{ ErrorDetail, base_llm::messages::context::{MessagesModelCapabilities, MessagesShapeOptions}, }; use litellm_llms_types::{ - formats::messages::{MessagesRequest, MessagesResponse, SystemPrompt}, + formats::messages::{MessagesRequest, MessagesResponse}, headers::ProviderSpecificHeaders, }; use litellm_router_types::LitellmParams; @@ -31,32 +29,10 @@ pub struct MessagesCall { pub shaping: MessagesShaping, } -/// The request body. `custom_llm_provider` is the call's, as the provider is resolved from it -/// and the model. -pub fn messages_body( - body: Map, - custom_llm_provider: Option<&str>, -) -> Result { - if !reads_unparsed_system(&body, custom_llm_provider) - && let Some(system) = body.get(SYSTEM).filter(|system| !system.is_null()) - && let Err(err) = SystemPrompt::deserialize(system) - { - return Err(invalid_request(err)); - } +pub fn messages_body(body: Map) -> Result { serde_json::from_value(Value::Object(body)).map_err(invalid_request) } -const SYSTEM: &str = "system"; - -/// Only `azure_ai` keeps a system prompt Rust could not type as sent, as Python does. By the -/// owner's ruling every other provider, `anthropic` included, keeps main's request error for it. -fn reads_unparsed_system(body: &Map, custom_llm_provider: Option<&str>) -> bool { - body.get("model") - .and_then(Value::as_str) - .and_then(|model| resolve_llm_provider(model, custom_llm_provider, "messages").ok()) - .is_some_and(|resolved| resolved.provider == LlmProviders::AzureAi) -} - /// The caller's litellm params, projected by a host from the keys [`LitellmParams::fields`] /// names. A key present with a value of the wrong type is a request error, as it is for /// Python's `GenericLiteLLMParams(**kwargs)`. @@ -110,6 +86,8 @@ pub struct MessagesSettings { #[serde(default)] pub drops_unsupported_params: bool, #[serde(default)] + pub global_drop_params: bool, + #[serde(default)] pub intercepted: bool, #[serde(default)] pub mcp_gateway: bool, @@ -126,6 +104,10 @@ pub struct MessagesSettings { /// settings dict. #[serde(skip)] pub module_api_key: Option, + #[serde(default)] + pub detached_drain_timeout: Option, + #[serde(default)] + pub detached_drain_limit: Option, } impl MessagesSettings { @@ -133,10 +115,18 @@ impl MessagesSettings { MessagesShapeOptions { reasoning_auto_summary: self.reasoning_auto_summary, presanitized: self.presanitized, - ..MessagesShapeOptions::default() } } + /// Python's Bedrock Messages relay keeps reading the provider stream after its client leaves, + /// for at most this long and in at most this many calls at once. + pub fn detached_drain(&self) -> Option { + Some(litellm_host::protocol::DetachedDrain { + timeout: Duration::try_from_secs_f64(self.detached_drain_timeout?).ok()?, + limit: self.detached_drain_limit?, + }) + } + pub fn request_timeout(&self) -> Option { self.request_timeout .and_then(|secs| Duration::try_from_secs_f64(secs).ok()) @@ -225,6 +215,7 @@ mod tests { supports_sampling_params: false, supports_speed: true, supports_mid_conversation_system: false, + bedrock: Default::default(), effort_tiers: SupportedEffortTiers { minimal: false, low: true, diff --git a/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs b/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs index 5207f4e9d58..c19ffc5a47c 100644 --- a/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs +++ b/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs @@ -227,6 +227,57 @@ async fn managed_aws_credentials_sign_the_exact_native_request( assert_eq!(request.json()["model"], "claude-test"); } +#[rstest] +#[tokio::test] +async fn bedrock_invoke_keeps_the_region_path_and_signs_caller_headers_as_python_does( + call: MessagesCall, +) { + let upstream = upstream([message_response()]).await; + let secrets = Arc::new(RecordingSecrets::new([("AWS_REGION_NAME", "us-east-1")])); + run_with( + secrets, + MessagesCall { + api_base: Some(upstream.uri()), + litellm_params: litellm_router_types::LitellmParams { + aws: litellm_auth::AwsParams { + aws_access_key_id: Some("test-access".into()), + aws_secret_access_key: Some("test-secret".into()), + ..Default::default() + }, + ..Default::default() + }, + extra_headers: Some(serde_json::Map::from_iter([ + ("x-amz-date".to_string(), json!("20200101T000000Z")), + ("x-amz-security-token".to_string(), json!("caller-token")), + ("date".to_string(), json!("Wed, 01 Jan 2020 00:00:00 GMT")), + ])), + ..with_model( + call, + "bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0", + ) + }, + ) + .await + .unwrap(); + let request = only_request(&upstream).await; + let auth = request.header("authorization").unwrap(); + assert_eq!( + request.url.path(), + "/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke" + ); + assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=test-access/")); + assert!(auth.contains( + "/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;date;host;x-amz-security-token, " + )); + assert_eq!(request.header("x-amz-security-token"), Some("caller-token")); + assert_eq!(request.header("x-amz-date"), None); + assert!( + request + .header("date") + .is_some_and(|date| date.ends_with(" -0000") && !date.contains("2020")) + ); +} + #[rstest] #[tokio::test] async fn private_mantle_endpoint_is_used_and_signed(call: MessagesCall) { diff --git a/litellm/rust/crates/inference-messages/tests/messages/request.rs b/litellm/rust/crates/inference-messages/tests/messages/request.rs index 74bc3388045..bc2e108d57c 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/request.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/request.rs @@ -920,13 +920,12 @@ impl litellm_tracing::Sink for Warnings { } #[rstest] -#[case::anthropic_logs_as_main("anthropic", false)] -#[case::azure_ai_logs_python_s_warning("azure_ai", true)] +#[case::anthropic("anthropic")] +#[case::azure_ai("azure_ai")] #[tokio::test] -async fn a_speed_dropped_under_drop_params_is_logged_only_on_azure_ai( +async fn a_speed_dropped_under_drop_params_logs_python_s_warning( call: MessagesCall, #[case] provider: &str, - #[case] logged: bool, ) { let upstream = upstream([message_response()]).await; let warnings = Warnings::default(); @@ -956,11 +955,7 @@ async fn a_speed_dropped_under_drop_params_is_logged_only_on_azure_ai( .filter(|message| message.starts_with("Dropping unsupported `speed`")) .cloned() .collect(); - assert_eq!( - speed_warnings.len(), - usize::from(logged), - "{speed_warnings:?}" - ); + assert_eq!(speed_warnings.len(), 1, "{speed_warnings:?}"); } #[rstest] diff --git a/litellm/rust/crates/inference-messages/tests/messages/response.rs b/litellm/rust/crates/inference-messages/tests/messages/response.rs index 365d6ed711c..0f4809b72fc 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/response.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/response.rs @@ -483,7 +483,7 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes #[case::mistyped_param(json!({"model": MODEL, "messages": "hello", "max_tokens": 16}))] #[case::missing_messages(json!({"model": MODEL, "max_tokens": 16}))] fn a_body_that_does_not_parse_is_an_invalid_request(#[case] raw: Value) { - let error = messages_body(object(raw), None).expect_err("the body is rejected"); + let error = messages_body(object(raw)).expect_err("the body is rejected"); assert!(error.is_request(), "{error:?}"); assert!( @@ -502,38 +502,27 @@ const UNPARSED_SYSTEMS: [&str; 4] = [ ]; #[rstest] -fn a_system_prompt_that_does_not_parse_is_main_s_request_error_except_on_azure_ai( +fn a_system_prompt_that_does_not_parse_is_kept_as_python_keeps_it( #[values(0, 1, 2, 3)] system: usize, #[values( - ("anthropic/claude-sonnet-4-5", None), - ("claude-sonnet-4-5", Some("anthropic")), - ("vertex_ai/claude-sonnet-4-5", None), - ("azure_ai/claude-sonnet-4-5", None), - ("claude-sonnet-4-5", Some("azure_ai")) + "anthropic/claude-sonnet-4-5", + "vertex_ai/claude-sonnet-4-5", + "azure_ai/claude-sonnet-4-5" )] - call: (&str, Option<&str>), + model: &str, ) { - let (model, custom_llm_provider) = call; let system: Value = serde_json::from_str(UNPARSED_SYSTEMS[system]).unwrap(); let body = object(json!({"model": model, "messages": [], "max_tokens": 16, "system": system})); - let parsed = messages_body(body, custom_llm_provider); - let azure_ai = model.starts_with("azure_ai/") || custom_llm_provider == Some("azure_ai"); - match parsed { + match messages_body(body) { Err(error) => { - assert!( - !azure_ai || system.is_number() || system.is_boolean(), - "{error:?}" - ); + assert!(system.is_number() || system.is_boolean(), "{error:?}"); let text = litellm_error::Report::new(&error).to_string(); - let expected = if azure_ai { - "system must be a string or a list of content blocks" - } else { - "invalid Anthropic messages request: data did not match any variant of untagged enum SystemPrompt" - }; - assert!(text.contains(expected), "{text}"); + assert!( + text.contains("system must be a string or a list of content blocks"), + "{text}" + ); } Ok(request) => { - assert!(azure_ai, "{request:?}"); assert!( request .params diff --git a/litellm/rust/crates/inference/src/error.rs b/litellm/rust/crates/inference/src/error.rs index 5f6e1b3f70d..0f5eacf26b5 100644 --- a/litellm/rust/crates/inference/src/error.rs +++ b/litellm/rust/crates/inference/src/error.rs @@ -215,11 +215,6 @@ impl Error { impl From for Error { fn from(error: litellm_http::Error) -> Self { - // Python drops the caller's copy and prefers a forwarded Authorization - // over the signature, so leave the request to it. - if error.is_computed_header() { - return Self::Declined(Decline::new("request forwards a header AWS SigV4 computes")); - } Self::Http(error) } } @@ -330,16 +325,6 @@ mod tests { assert!(error.is_missing_credential()); } - #[rstest] - #[case::direct(litellm_http::Error::computed_header("authorization").into())] - #[case::through_a_transformation( - LlmError::Http(litellm_http::Error::computed_header("authorization")).into() - )] - fn a_header_the_signer_computes_is_left_to_the_host_on_every_route(#[case] error: Error) { - assert!(error.declined().is_some(), "{error:?}"); - assert!(!error.dispatched()); - } - #[test] fn a_transformation_decline_survives_into_the_route_error() { let decline = Decline::new("unrecognized request parameter"); diff --git a/litellm/rust/crates/llms-types/src/formats/messages/streaming.rs b/litellm/rust/crates/llms-types/src/formats/messages/streaming.rs index b2f70ee3753..58f2db287aa 100644 --- a/litellm/rust/crates/llms-types/src/formats/messages/streaming.rs +++ b/litellm/rust/crates/llms-types/src/formats/messages/streaming.rs @@ -137,6 +137,8 @@ pub enum MessagesStreamEvent { MessageStop { #[serde(default, skip_serializing_if = "Option::is_none")] usage: Option, + #[serde(flatten)] + extra: Map, }, Ping, Error { diff --git a/litellm/rust/crates/llms/Cargo.toml b/litellm/rust/crates/llms/Cargo.toml index f779d4edf66..f537a26bd73 100644 --- a/litellm/rust/crates/llms/Cargo.toml +++ b/litellm/rust/crates/llms/Cargo.toml @@ -27,6 +27,7 @@ litellm-model-catalog.workspace = true litellm-tracing.workspace = true base64.workspace = true bytes.workspace = true +crc32fast = "1.5" data-url = "0.3.2" fancy-regex.workspace = true futures-util.workspace = true @@ -39,6 +40,7 @@ strum.workspace = true thiserror.workspace = true time.workspace = true tokio = { workspace = true, features = ["sync"] } +tokio-util = { version = "0.7", features = ["codec"] } url.workspace = true [dev-dependencies] diff --git a/litellm/rust/crates/llms/src/anthropic/beta_gated_fields.rs b/litellm/rust/crates/llms/src/anthropic/beta_gated_fields.rs index 4445fb49a2b..b06e78f4cf0 100644 --- a/litellm/rust/crates/llms/src/anthropic/beta_gated_fields.rs +++ b/litellm/rust/crates/llms/src/anthropic/beta_gated_fields.rs @@ -1,8 +1,13 @@ use std::collections::BTreeSet; +use std::fmt::Display; + use litellm_llms_types::{ - formats::messages::{ContentBlock, ContentBlockType, Message, MessageContent}, - providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet}, + formats::messages::{ + ContentBlock, ContentBlockType, Message, MessageContent, MessagesOptionalParams, + MessagesRequest, ThinkingConfig, + }, + providers::anthropic::{AnthropicBeta, BetaSet}, recognized::Recognized, }; use serde_json::{Map, Value}; @@ -16,15 +21,20 @@ const CONTENT: &str = "content"; const TYPE: &str = "type"; const OUTPUT_CONFIG: &str = "output_config"; const TOOL_CHANGE_BLOCK_TYPES: [&str; 2] = ["tool_addition", "tool_removal"]; +const BLOCK_BINDING: &str = "block_binding"; +const DISPLAY: &str = "display"; +const DISPLAY_UPDATES: &str = "updates"; -/// The mid-conversation system fields of Python's `BETA_GATED_FIELDS`, named as Python's warning -/// names them and declared in the order it sorts those names. +/// The fields of Python's `BETA_GATED_FIELDS`, named as Python's warning names them and declared +/// in the order it sorts those names. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] enum GatedField { Emptied, ClearAt, ToolChanges, OutputConfig, + BlockBinding, + DisplayUpdates, } impl GatedField { @@ -34,6 +44,8 @@ impl GatedField { Self::ClearAt => "messages[].system.clear_at", Self::ToolChanges => "messages[].system.content[].tool_addition|tool_removal", Self::OutputConfig => "messages[].system.output_config", + Self::BlockBinding => "thinking.block_binding", + Self::DisplayUpdates => "thinking.display=updates", } } @@ -49,6 +61,8 @@ impl GatedField { AnthropicBeta::MidConversationOutputConfig20260701, AnthropicBeta::PerTurnControl20260701, ], + Self::BlockBinding => &[AnthropicBeta::ThinkingBindingControls20260801], + Self::DisplayUpdates => &[AnthropicBeta::ThinkingDisplayUpdates20260818], } } @@ -82,11 +96,98 @@ pub fn strip_beta_gated_system_messages( active: &BetaSet, drops_unsupported_params: bool, model: &str, - provider: BetaProvider, + provider: impl Display, ) -> Vec> { if !drops_unsupported_params { return messages; } + let (messages, gated) = stripped_system_messages(messages, active); + log_beta_gated_strips(&gated, model, provider); + messages +} + +/// Python's `strip_beta_gated_native_request` and `log_beta_gated_strips`, run once the final +/// `anthropic-beta` values are known: under drop_params the `thinking` fields and the system +/// message fields no beta in `active` unlocks are dropped, and one warning names them all. +pub fn strip_beta_gated_request( + request: MessagesRequest, + active: &BetaSet, + drops_unsupported_params: bool, + provider: impl Display, +) -> MessagesRequest { + if !drops_unsupported_params { + return request; + } + let (thinking, thinking_gated) = stripped_thinking(request.params.thinking, active); + let (messages, messages_gated) = stripped_system_messages(request.messages, active); + log_beta_gated_strips( + &thinking_gated.union(&messages_gated).copied().collect(), + &request.model, + provider, + ); + MessagesRequest { + messages, + params: MessagesOptionalParams { + thinking, + ..request.params + }, + ..request + } +} + +fn log_beta_gated_strips(gated: &BTreeSet, model: &str, provider: impl Display) { + if gated.is_empty() { + return; + } + litellm_tracing::warn!( + "Dropped {} for model={model} provider={provider} because drop_params is on and the provider would reject them on this request, either since this provider leg does not send that anthropic-beta (see anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user messages.", + gated + .iter() + .copied() + .map(GatedField::described) + .collect::>() + .join(", ") + ); +} + +/// Python's `strip_beta_gated_thinking`, which reads `thinking` as any mapping. +fn stripped_thinking( + thinking: Option>, + active: &BetaSet, +) -> (Option>, BTreeSet) { + let Some(Value::Object(fields)) = thinking.as_ref().and_then(|t| serde_json::to_value(t).ok()) + else { + return (thinking, BTreeSet::new()); + }; + let binding_gated = + fields.contains_key(BLOCK_BINDING) && GatedField::BlockBinding.gated(active); + let display_gated = fields.get(DISPLAY).and_then(Value::as_str) == Some(DISPLAY_UPDATES) + && GatedField::DisplayUpdates.gated(active); + if !binding_gated && !display_gated { + return (thinking, BTreeSet::new()); + } + let kept: Map = fields + .into_iter() + .filter(|(key, _)| { + !((binding_gated && key == BLOCK_BINDING) || (display_gated && key == DISPLAY)) + }) + .collect(); + let gated = [ + (binding_gated, GatedField::BlockBinding), + (display_gated, GatedField::DisplayUpdates), + ] + .into_iter() + .filter_map(|(gated, field)| gated.then_some(field)) + .collect(); + let thinking = serde_json::from_value(Value::Object(kept.clone())) + .unwrap_or(Recognized::Unrecognized(Value::Object(kept))); + (Some(thinking), gated) +} + +fn stripped_system_messages( + messages: Vec>, + active: &BetaSet, +) -> (Vec>, BTreeSet) { let stripped: Vec<(Option>, Vec)> = messages .iter() .cloned() @@ -112,24 +213,16 @@ pub fn strip_beta_gated_system_messages( .flat_map(|(_, gated)| gated.iter().copied()) .collect(); if gated.is_empty() { - return messages; + return (messages, gated); } let kept: Vec> = stripped .into_iter() .filter_map(|(message, _)| message) .collect(); if ends_on_assistant(&kept) && !ends_on_assistant(&messages) { - return messages; + return (messages, BTreeSet::new()); } - litellm_tracing::warn!( - "Dropped {} for model={model} provider={provider} because drop_params is on and the provider would reject them on this request, either since this provider leg does not send that anthropic-beta (see anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user messages.", - gated - .into_iter() - .map(GatedField::described) - .collect::>() - .join(", ") - ); - kept + (kept, gated) } fn strip_system_message(message: Message, active: &BetaSet) -> (Option, Vec) { @@ -263,14 +356,14 @@ fn ends_on_assistant(messages: &[Recognized]) -> bool { #[cfg(test)] mod tests { use litellm_llms_types::{ - formats::messages::Message, + formats::messages::{Message, MessagesRequest}, providers::anthropic::{BetaProvider, BetaSet}, recognized::Recognized, }; use rstest::rstest; use serde_json::{Value, json}; - use super::strip_beta_gated_system_messages; + use super::{strip_beta_gated_request, strip_beta_gated_system_messages}; fn strip( messages: Vec>, @@ -445,6 +538,99 @@ mod tests { assert_eq!(strip(sent.clone(), &BetaSet::default(), true), sent); } + fn thinking_request(thinking: Value, system: Value) -> MessagesRequest { + serde_json::from_value(json!({ + "model": "test-model", + "max_tokens": 16, + "messages": conversation(system), + "thinking": thinking, + })) + .unwrap() + } + + #[rstest] + #[case::block_binding_without_its_beta( + json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}), + "", + json!({"type": "adaptive"}) + )] + #[case::block_binding_with_its_beta( + json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}), + "thinking-binding-controls-2026-08-01", + json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}) + )] + #[case::display_updates_without_its_beta( + json!({"type": "adaptive", "display": "updates"}), + "", + json!({"type": "adaptive"}) + )] + #[case::display_updates_with_its_beta( + json!({"type": "adaptive", "display": "updates"}), + "thinking-display-updates-2026-08-18", + json!({"type": "adaptive", "display": "updates"}) + )] + #[case::another_display( + json!({"type": "adaptive", "display": "summarized"}), + "", + json!({"type": "adaptive", "display": "summarized"}) + )] + #[case::both_on_enabled_thinking( + json!({"type": "enabled", "budget_tokens": 1024, "display": "updates", "block_binding": {}}), + "", + json!({"type": "enabled", "budget_tokens": 1024}) + )] + fn gated_thinking_fields_are_dropped( + #[case] thinking: Value, + #[case] active: &str, + #[case] expected: Value, + ) { + let stripped = strip_beta_gated_request( + thinking_request(thinking, json!({"role": "user", "content": "more"})), + &betas(active), + true, + "test", + ); + assert_eq!( + serde_json::to_value(stripped.params.thinking).unwrap(), + expected + ); + } + + #[rstest] + fn the_request_strip_drops_gated_thinking_and_system_fields_together() { + let stripped = strip_beta_gated_request( + thinking_request( + json!({"type": "adaptive", "display": "updates"}), + json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}), + ), + &BetaSet::default(), + true, + "test", + ); + assert_eq!( + ( + serde_json::to_value(stripped.params.thinking).unwrap(), + stripped.messages + ), + ( + json!({"type": "adaptive"}), + messages(conversation(json!({"role": "system", "content": "note"}))) + ) + ); + } + + #[rstest] + fn without_drop_params_the_request_is_sent_as_is() { + let sent = thinking_request( + json!({"type": "adaptive", "display": "updates", "block_binding": {}}), + json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}), + ); + assert_eq!( + strip_beta_gated_request(sent.clone(), &BetaSet::default(), false, "test"), + sent + ); + } + #[rstest] fn non_system_messages_keep_their_fields() { let sent = messages(json!([ diff --git a/litellm/rust/crates/llms/src/anthropic/beta_headers.rs b/litellm/rust/crates/llms/src/anthropic/beta_headers.rs index 4a3575c9bbc..6f78a71950c 100644 --- a/litellm/rust/crates/llms/src/anthropic/beta_headers.rs +++ b/litellm/rust/crates/llms/src/anthropic/beta_headers.rs @@ -16,20 +16,38 @@ pub enum BetaPolicy { impl BetaPolicy { pub fn apply(self, headers: Headers) -> Headers { + self.apply_keeping(headers, &BetaSet::default()) + } + + /// Python's `update_headers_with_filtered_beta` with `keep`: a beta the routed model + /// accepts is sent as the caller sent it, after the betas the filter kept. + pub fn apply_keeping(self, headers: Headers, keep: &BetaSet) -> Headers { let provider = match self { Self::Forward => return headers, Self::Drop => return without_headers(headers, &[BETA_HEADER]), Self::Filter(provider) => provider, }; - let accepted: BetaSet = existing_betas(&headers) + let sent = existing_betas(&headers); + let accepted: BetaSet = sent .iter() + .filter(|beta| !keep.contains(beta)) .filter_map(|beta| beta.on(provider)) .collect(); + let kept: Vec<&AnthropicBeta> = sent + .iter() + .filter(|beta| keep.contains(beta) && !accepted.contains(beta)) + .collect(); let headers = without_headers(headers, &[BETA_HEADER]); - if accepted.is_empty() { + if accepted.is_empty() && kept.is_empty() { return headers; } - with_header(headers, BETA_HEADER, accepted.to_string()) + let value = accepted + .iter() + .chain(kept) + .map(AnthropicBeta::as_str) + .collect::>() + .join(","); + with_header(headers, BETA_HEADER, value) } } @@ -46,7 +64,7 @@ pub fn accepted_or_unlisted(betas: BetaSet, provider: BetaProvider) -> BetaSet { #[cfg(test)] mod tests { use crate::anthropic::beta_headers::BetaPolicy; - use litellm_llms_types::providers::anthropic::BetaProvider; + use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet}; use rstest::rstest; fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> { @@ -125,6 +143,37 @@ mod tests { ); } + #[rstest] + #[case::kept_after_the_filtered_betas( + "per-turn-control-2026-07-01,advanced-tool-use-2025-11-20,example-beta-2099-01-01", + Some("tool-search-tool-2025-10-19,per-turn-control-2026-07-01") + )] + #[case::kept_alone( + "per-turn-control-2026-07-01,example-beta-2099-01-01", + Some("per-turn-control-2026-07-01") + )] + #[case::not_sent("advanced-tool-use-2025-11-20", Some("tool-search-tool-2025-10-19"))] + #[case::nothing_survives("example-beta-2099-01-01", None)] + fn filter_sends_a_kept_beta_as_sent_after_the_filtered_ones( + #[case] sent: &str, + #[case] expected: Option<&str>, + ) { + let keep: BetaSet = "per-turn-control-2026-07-01" + .parse() + .unwrap_or_else(|never| match never {}); + assert_eq!( + BetaPolicy::Filter(BetaProvider::Bedrock).apply_keeping( + headers(&[("x-api-key", "k"), ("anthropic-beta", sent)]), + &keep + ), + [("x-api-key", "k")] + .into_iter() + .chain(expected.map(|beta| ("anthropic-beta", beta))) + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>() + ); + } + #[rstest] #[case::anthropic( BetaProvider::Anthropic, diff --git a/litellm/rust/crates/llms/src/anthropic/common_utils.rs b/litellm/rust/crates/llms/src/anthropic/common_utils.rs index 5674fbd1aef..681ae22150c 100644 --- a/litellm/rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm/rust/crates/llms/src/anthropic/common_utils.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; -use crate::base_llm::messages::context::{MessagesModelCapabilities, UnparsedMessages}; +use crate::base_llm::messages::context::MessagesModelCapabilities; use fancy_regex::Regex; use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; @@ -322,26 +322,19 @@ pub fn has_advisor_tool(tools: Option<&[Recognized]>) -> bool { pub fn requires_native_compaction_beta( compaction: Option<&Value>, messages: &[Recognized], - unparsed: UnparsedMessages, ) -> bool { - compaction.is_some() - || messages - .iter() - .any(|message| carries_signed_compaction_block(message, unparsed)) + compaction.is_some() || messages.iter().any(carries_signed_compaction_block) } -/// Python's `carries_signed_compaction_block` reads any mapping, so where unparsed messages are -/// read a message that did not parse is scanned through its raw content too. -fn carries_signed_compaction_block( - message: &Recognized, - unparsed: UnparsedMessages, -) -> bool { +/// Python's `carries_signed_compaction_block` reads any mapping, so a message that did not parse +/// is scanned through its raw content too. +fn carries_signed_compaction_block(message: &Recognized) -> bool { match message { Recognized::Known(message) => message.blocks().iter().any(|block| { block.is_type(ContentBlockType::Compaction) && block.signature.as_deref().is_some_and(|s| !s.is_empty()) }), - Recognized::Unrecognized(Value::Object(fields)) if unparsed.read() => matches!( + Recognized::Unrecognized(Value::Object(fields)) => matches!( fields.get("content"), Some(Value::Array(blocks)) if blocks.iter().any(|block| { block.get("type").and_then(Value::as_str) == Some("compaction") @@ -2773,33 +2766,19 @@ mod tests { #[case] messages: Value, #[case] expected: bool, ) { - for unparsed in [UnparsedMessages::Read, UnparsedMessages::Skipped] { - assert_eq!( - requires_native_compaction_beta( - compaction.as_ref(), - &history(messages.clone()), - unparsed - ), - expected - ); - } + assert_eq!( + requires_native_compaction_beta(compaction.as_ref(), &history(messages)), + expected + ); } - #[rstest] - #[case::read(UnparsedMessages::Read, true)] - #[case::skipped(UnparsedMessages::Skipped, false)] - fn a_signed_compaction_block_in_an_unparsed_message_counts_only_where_unparsed_messages_are_read( - #[case] unparsed: UnparsedMessages, - #[case] expected: bool, - ) { + #[test] + fn a_signed_compaction_block_in_an_unparsed_message_counts() { let messages = history(json!([{"role": "assistant", "content": [ {"type": "text", "text": 5}, {"type": "compaction", "content": "c", "signature": "s"} ]}])); - assert_eq!( - requires_native_compaction_beta(None, &messages, unparsed), - expected - ); + assert!(requires_native_compaction_beta(None, &messages)); } #[rstest] @@ -2975,6 +2954,7 @@ mod tests { supports_speed: false, supports_mid_conversation_system: false, effort_tiers: tiers(false, false, false, false, false, false), + bedrock: Default::default(), } ); assert_eq!( diff --git a/litellm/rust/crates/llms/src/anthropic/messages/handler.rs b/litellm/rust/crates/llms/src/anthropic/messages/handler.rs index 2709e729d05..4bd598e8a18 100644 --- a/litellm/rust/crates/llms/src/anthropic/messages/handler.rs +++ b/litellm/rust/crates/llms/src/anthropic/messages/handler.rs @@ -3,7 +3,7 @@ use litellm_llms_types::{ AdaptiveThinking, EnabledThinking, Message, MessagesOptionalParams, MessagesRequest, ThinkingConfig, ThinkingDisplay, }, - recognized::{Recognized, map_known_runs}, + recognized::Recognized, }; use serde_json::{Value, json}; @@ -15,30 +15,25 @@ use crate::{ strip_empty_raw_content_blocks, strip_provider_specific_fields, strip_raw_provider_specific_fields, }, - base_llm::messages::context::{MessagesShapeOptions, UnparsedMessages}, + base_llm::messages::context::MessagesShapeOptions, }; pub fn shape_anthropic_messages_request( request: MessagesRequest, options: MessagesShapeOptions, ) -> Result { - let messages = match (options.presanitized, options.unparsed) { - (true, UnparsedMessages::Read) => map_messages( + let messages = if options.presanitized { + map_messages( request.messages, strip_provider_specific_fields, strip_raw_provider_specific_fields, - ), - (false, UnparsedMessages::Read) => map_messages( + ) + } else { + map_messages( request.messages, sanitize_anthropic_messages, sanitize_raw_anthropic_message, - ), - (true, UnparsedMessages::Skipped) => { - map_known_runs(request.messages, strip_provider_specific_fields) - } - (false, UnparsedMessages::Skipped) => { - map_known_runs(request.messages, sanitize_anthropic_messages) - } + ) }; Ok(MessagesRequest { messages, @@ -334,7 +329,6 @@ mod tests { MessagesShapeOptions { reasoning_auto_summary: true, presanitized: false, - unparsed: UnparsedMessages::Skipped, }, ) .unwrap(); @@ -355,7 +349,6 @@ mod tests { #[rstest] #[case::sanitized( false, - UnparsedMessages::Read, json!([ {"role": "user", "content": "hi"}, {"role": "assistant", "content": [ @@ -365,7 +358,6 @@ mod tests { )] #[case::presanitized( true, - UnparsedMessages::Read, json!([ {"role": "user", "content": "hi"}, {"role": "user", "content": [{"type": "text", "text": 5}]}, @@ -375,33 +367,8 @@ mod tests { ]} ]) )] - #[case::skipped( - false, - UnparsedMessages::Skipped, - json!([ - {"role": "user", "content": "hi"}, - {"role": "user", "content": [{"type": "text", "text": 5}]}, - {"role": "assistant", "content": [ - {"type": "text", "text": 5}, - {"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}, "provider_specific_fields": {"x": 1}} - ]} - ]) - )] - #[case::skipped_presanitized( - true, - UnparsedMessages::Skipped, - json!([ - {"role": "user", "content": "hi"}, - {"role": "user", "content": [{"type": "text", "text": 5}]}, - {"role": "assistant", "content": [ - {"type": "text", "text": 5}, - {"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}, "provider_specific_fields": {"x": 1}} - ]} - ]) - )] - fn shaping_rewrites_a_message_that_did_not_parse_only_where_unparsed_messages_are_read( + fn shaping_rewrites_a_message_that_did_not_parse( #[case] presanitized: bool, - #[case] unparsed: UnparsedMessages, #[case] expected: Value, ) { let shaped = shape_anthropic_messages_request( @@ -416,7 +383,6 @@ mod tests { MessagesShapeOptions { reasoning_auto_summary: false, presanitized, - unparsed, }, ) .unwrap(); @@ -434,7 +400,6 @@ mod tests { MessagesShapeOptions { reasoning_auto_summary: false, presanitized: true, - unparsed: UnparsedMessages::Skipped, }, ) .unwrap(); diff --git a/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs index f8d2ad88d7d..3998454b1e0 100644 --- a/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs @@ -7,7 +7,7 @@ use litellm_llms_types::{ MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay, }, providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet, DEFAULT_HEADERS}, - recognized::{Recognized, map_known_runs}, + recognized::Recognized, }; use litellm_router_types::LitellmParams; use serde_json::{Map, Value, json}; @@ -17,9 +17,11 @@ use super::{ scalar::{is_one, python_str, python_value}, thinking::{drop_unsupported_effort, translate_thinking}, }; -use crate::anthropic::beta_headers::accepted_or_unlisted; +use crate::anthropic::{ + beta_gated_fields::strip_beta_gated_request, beta_headers::accepted_or_unlisted, +}; use crate::base_llm::messages::{ - context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages}, + context::{MessagesShapeOptions, MessagesTransformContext}, tools::ToolCustomField, }; use crate::{ @@ -32,7 +34,7 @@ use crate::{ is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error, is_claude_strict_tools_rejection, is_tool_search_used, map_messages, merge_beta_headers, minted_oauth_environment, optionally_handle_anthropic_oauth, - requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks, + requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks_restoring_redacted_data, strip_raw_advisor_blocks, strip_raw_encrypted_reasoning_blocks, strip_thinking_blocks_from_anthropic_messages_request, tools_without_strict, @@ -132,6 +134,23 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { ) } + /// Python's transform strips the thinking and system fields the final header does not + /// unlock. + fn finalize_request( + &self, + request: MessagesRequest, + headers: Headers, + context: &MessagesTransformContext, + ) -> Result<(MessagesRequest, Headers), Error> { + let stripped = strip_beta_gated_request( + request, + &existing_betas(&headers), + context.drops_unsupported_params, + BetaProvider::Anthropic, + ); + Ok((stripped, headers)) + } + fn retry_after_http_error( &self, status: u16, @@ -301,19 +320,16 @@ pub(crate) fn transform_messages_request( .context_management .clone() .map(map_openai_context_management_to_anthropic); - let reads_unparsed = context.unparsed.read(); let messages = if has_advisor_tool(request.params.tools.as_deref()) || continues_message_thread(&request.params) { request.messages - } else if reads_unparsed { + } else { map_messages( request.messages, strip_advisor_blocks, strip_raw_advisor_blocks, ) - } else { - map_known_runs(request.messages, strip_advisor_blocks) }; let messages = if context.drops_unsupported_params { drop_non_string_tool_result_texts(messages) @@ -321,15 +337,11 @@ pub(crate) fn transform_messages_request( messages }; Ok(MessagesRequest { - messages: if reads_unparsed { - map_messages( - messages, - strip_encrypted_reasoning_blocks_restoring_redacted_data, - strip_raw_encrypted_reasoning_blocks, - ) - } else { - map_known_runs(messages, strip_encrypted_reasoning_blocks) - }, + messages: map_messages( + messages, + strip_encrypted_reasoning_blocks_restoring_redacted_data, + strip_raw_encrypted_reasoning_blocks, + ), params: MessagesOptionalParams { context_management, extra: without_null_safeguards(request.params.extra), @@ -428,21 +440,17 @@ pub(crate) fn update_headers_with_anthropic_beta( headers: Headers, request: &MessagesRequest, ) -> Headers { - merge_beta_headers(headers, feature_betas(request, UnparsedMessages::Skipped)) + merge_beta_headers(headers, feature_betas(request)) } -/// `unparsed` says whether a beta trigger in a message that did not parse counts, as it does -/// for Python, which reads any mapping. pub(crate) fn update_headers_with_safeguards_beta( headers: Headers, request: &MessagesRequest, provider: BetaProvider, - unparsed: UnparsedMessages, ) -> Headers { merge_beta_headers( headers, - feature_betas(request, unparsed) - .union(on_provider(mapped_host_betas(&request.params), provider)), + feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)), ) } @@ -450,7 +458,7 @@ pub(crate) fn update_headers_with_safeguards_beta( /// through that column. pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet { on_provider( - feature_betas(request, UnparsedMessages::Skipped).union(mapped_host_betas(&request.params)), + feature_betas(request).union(mapped_host_betas(&request.params)), provider, ) } @@ -483,7 +491,7 @@ fn is_threaded(params: &MessagesOptionalParams) -> bool { .is_some_and(|thread| !thread.is_null()) } -fn feature_betas(request: &MessagesRequest, unparsed: UnparsedMessages) -> BetaSet { +fn feature_betas(request: &MessagesRequest) -> BetaSet { let params = &request.params; let tools = params.tools.as_deref(); let scanned_messages: &[Recognized] = if continues_message_thread(params) { @@ -492,19 +500,19 @@ fn feature_betas(request: &MessagesRequest, unparsed: UnparsedMessages) -> BetaS &request.messages }; [ - requires_native_compaction_beta(params.compaction.as_ref(), scanned_messages, unparsed) + requires_native_compaction_beta(params.compaction.as_ref(), scanned_messages) .then_some(AnthropicBeta::Compact20260904), uses_structured_output(params).then_some(AnthropicBeta::StructuredOutputs20251113), (params.speed == Some(Recognized::Known(Speed::Fast))) .then_some(AnthropicBeta::FastMode20260201), - messages_carry_output_config(scanned_messages, unparsed) + messages_carry_output_config(scanned_messages) .then_some(AnthropicBeta::PerTurnControl20260701), is_threaded(params).then_some(AnthropicBeta::MessageThreads20260812), uses_thinking_display_updates(params) .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), uses_thinking_block_binding(params) .then_some(AnthropicBeta::ThinkingBindingControls20260801), - messages_carry_tool_changes(scanned_messages, unparsed) + messages_carry_tool_changes(scanned_messages) .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120), @@ -550,17 +558,12 @@ fn carries_safeguards(params: &MessagesOptionalParams) -> bool { .is_some_and(|value| !value.is_null()) } -/// Python's `messages_carry_output_config` reads any mapping, so where unparsed messages are read -/// a message that did not parse, such as a system turn sent without content, counts too. -fn messages_carry_output_config( - messages: &[Recognized], - unparsed: UnparsedMessages, -) -> bool { +/// Python's `messages_carry_output_config` reads any mapping, so a message that did not parse, +/// such as a system turn sent without content, counts too. +fn messages_carry_output_config(messages: &[Recognized]) -> bool { messages.iter().any(|message| match message { Recognized::Known(message) => message.extra.contains_key(OUTPUT_CONFIG), - Recognized::Unrecognized(Value::Object(fields)) => { - unparsed.read() && fields.contains_key(OUTPUT_CONFIG) - } + Recognized::Unrecognized(Value::Object(fields)) => fields.contains_key(OUTPUT_CONFIG), Recognized::Unrecognized(_) => false, }) } @@ -618,19 +621,15 @@ fn is_raw_tool_reference_change(block: &Value) -> bool { == Some("tool_reference") } -/// `AnthropicModelInfo.is_mid_conversation_tool_change_used` reads any mapping, so where unparsed -/// messages are read a system message that did not parse is scanned through its raw content too. -fn messages_carry_tool_changes( - messages: &[Recognized], - unparsed: UnparsedMessages, -) -> bool { +/// `AnthropicModelInfo.is_mid_conversation_tool_change_used` reads any mapping, so a system +/// message that did not parse is scanned through its raw content too. +fn messages_carry_tool_changes(messages: &[Recognized]) -> bool { messages.iter().any(|message| match message { Recognized::Known(message) => { message.role == "system" && message.blocks().iter().any(is_tool_reference_change) } Recognized::Unrecognized(Value::Object(fields)) => { - unparsed.read() - && fields.get("role").and_then(Value::as_str) == Some("system") + fields.get("role").and_then(Value::as_str) == Some("system") && matches!( fields.get("content"), Some(Value::Array(blocks)) if blocks.iter().any(is_raw_tool_reference_change) @@ -671,11 +670,9 @@ pub fn drop_unsupported_params( let speed = match ¶ms.speed { Some(speed) if !capabilities.supports_speed => { reject("speed", speed_repr(speed), "")?; - if context.logs_dropped_params { - litellm_tracing::warn!( - "Dropping unsupported `speed` for model={model} (drop_params=True). Fast mode is only supported on select Opus models." - ); - } + litellm_tracing::warn!( + "Dropping unsupported `speed` for model={model} (drop_params=True). Fast mode is only supported on select Opus models." + ); None } _ => params.speed.clone(), @@ -1346,8 +1343,7 @@ mod tests { }, drop_params: true, drops_unsupported_params: false, - unparsed: UnparsedMessages::Skipped, - logs_dropped_params: false, + global_drop_params: false, } ); return; @@ -1736,10 +1732,7 @@ mod tests { &["message-threads-2026-08-12", "per-turn-control-2026-07-01"] )] fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) { - assert_eq!( - feature_betas(&request(fields), UnparsedMessages::Read), - betas(expected) - ); + assert_eq!(feature_betas(&request(fields)), betas(expected)); } #[rstest] @@ -1758,21 +1751,16 @@ mod tests { {"role": "user", "content": "Use the ping tool"}, {"role": "system", "content": [{"type": "text", "text": 5}, {"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, ]}))] - fn a_beta_trigger_in_an_unparsed_message_counts_only_where_unparsed_messages_are_read( - #[case] fields: Value, - ) { + fn a_beta_trigger_in_an_unparsed_message_counts_on_every_provider(#[case] fields: Value) { let request = request(fields); + assert_ne!(feature_betas(&request), BetaSet::default()); assert_eq!( - feature_betas(&request, UnparsedMessages::Skipped), - BetaSet::default() + existing_betas(&update_headers_with_anthropic_beta(Vec::new(), &request)), + feature_betas(&request) ); assert_eq!( provider_feature_betas(&request, BetaProvider::Anthropic), - BetaSet::default() - ); - assert_ne!( - feature_betas(&request, UnparsedMessages::Read), - BetaSet::default() + feature_betas(&request) ); } diff --git a/litellm/rust/crates/llms/src/anthropic/model_capabilities.rs b/litellm/rust/crates/llms/src/anthropic/model_capabilities.rs index 1f31473db45..7aff234f778 100644 --- a/litellm/rust/crates/llms/src/anthropic/model_capabilities.rs +++ b/litellm/rust/crates/llms/src/anthropic/model_capabilities.rs @@ -1,19 +1,32 @@ use std::sync::LazyLock; -use crate::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers}; +use crate::base_llm::messages::context::{ + BedrockCapabilities, MessagesModelCapabilities, NativeOutputSupport, SupportedEffortTiers, +}; use crate::bedrock::common_utils::{ - bedrock_base_model, is_bedrock_application_inference_profile_arn, without_routing_prefixes, + bedrock_base_model, is_bedrock_application_inference_profile_arn, strip_bedrock_routing_prefix, }; use fancy_regex::Regex; use litellm_auth_aws::is_bedrock_region; use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; +use litellm_llms_types::formats::messages::EffortLevel; use litellm_model_catalog::{Catalog, wire::Fields}; +use litellm_python_compat::{json::from_json, truthy::truthy}; +use serde::Deserialize; +use serde_json::Value; const SAMPLING_RESTRICTED_NAMES: [&str; 9] = [ "fable", "opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7", "opus-4-8", "opus_4_8", "opus-4.8", "opus_4.8", ]; const SAMPLING_TOLERANT_PROVIDERS: [&str; 1] = ["openrouter"]; +const BEDROCK: &str = "bedrock"; +const NATIVE_OUTPUT: &str = "supports_native_structured_output"; +const BEDROCK_NATIVE_OUTPUT: &str = "supports_bedrock_native_structured_output"; +const NATIVE_OUTPUT_BY_DEFAULT: &str = "supports_bedrock_native_structured_output_by_default"; +const OUTPUT_CONFIG: &str = "supports_output_config"; +const PER_TURN_CONTROL: &str = "supports_per_turn_control"; +const PROMPT_CACHE_TTL_PRICE: &str = "cache_creation_input_token_cost_above_1hr"; const BACKSTOP_PREFIXES: [&str; 4] = [ "bedrock/converse/", "bedrock/invoke/", @@ -45,7 +58,8 @@ pub fn anthropic_model_capabilities( .map_or((model, "anthropic"), |resolved| { (resolved.model, resolved.custom_llm_provider) }); - let (model, name) = match base_model.map(without_routing_prefixes) { + let requested = model; + let (model, name) = match base_model.map(strip_bedrock_routing_prefix) { Some(base) if is_bedrock(provider) && is_bedrock_application_inference_profile_arn(resolved) => { @@ -53,18 +67,7 @@ pub fn anthropic_model_capabilities( } _ => (model, resolved), }; - let probe = Probe { - catalog, - declared, - fallbacks: backstop_candidates(model), - info: catalog.model_info( - &candidate_names(name, provider) - .each_ref() - .map(String::as_str), - provider, - ), - bare: catalog.lookup(name).map(|found| found.entry.fields()), - }; + let probe = Probe::new(catalog, model, name, provider, declared); let thinking_always_on = probe.supports("thinking_always_on"); MessagesModelCapabilities { supports_reasoning: probe.supports("supports_reasoning"), @@ -98,9 +101,73 @@ pub fn anthropic_model_capabilities( xhigh: probe.supports("supports_xhigh_reasoning_effort"), max: probe.supports("supports_max_reasoning_effort"), }, + bedrock: if provider == BEDROCK { + bedrock_model_capabilities(catalog, requested, declared) + } else { + BedrockCapabilities::default() + }, } } +pub fn bedrock_model_capabilities( + catalog: &Catalog, + model: &str, + declared: &dyn Fn(&str) -> Option, +) -> BedrockCapabilities { + let model = bedrock_model(model); + let probe = Probe::new(catalog, model, bedrock_model(model), BEDROCK, &|_| None); + let mapped = probe.factory(NATIVE_OUTPUT); + let opted_out = declared(BEDROCK_NATIVE_OUTPUT).map_or_else( + || mapped_native_output_flag(catalog, model) == Some(false), + |opted_in| !opted_in, + ); + BedrockCapabilities { + output_config_effort_ceiling: probe + .info + .as_ref() + .and_then(|info| info.get("bedrock_output_config_effort_ceiling")) + .and_then(|ceiling| EffortLevel::deserialize(ceiling).ok()), + supports_output_config: probe.factory(OUTPUT_CONFIG), + supports_prompt_cache_ttl: [model, bedrock_base_model(model)].iter().any(|name| { + catalog + .get(name) + .and_then(|fields| fields.get(PROMPT_CACHE_TTL_PRICE)) + .is_some_and(|price| !price.is_null()) + }), + supports_tool_search: probe.supports("supports_tool_search"), + supports_per_turn_control: declared(PER_TURN_CONTROL) + .unwrap_or_else(|| probe.supports(PER_TURN_CONTROL)), + native_output: NativeOutputSupport { + enabled: declared(NATIVE_OUTPUT) + .unwrap_or_else(|| declared(BEDROCK_NATIVE_OUTPUT) != Some(false) && mapped), + opt_in: declared(BEDROCK_NATIVE_OUTPUT) + .unwrap_or_else(|| probe.factory(BEDROCK_NATIVE_OUTPUT)), + mapped, + replaces_unforceable_tool: backstop( + catalog, + &probe.fallbacks, + "supports_forced_tool_use", + ) == Some(false) + && !opted_out, + }, + } +} + +fn bedrock_model(model: &str) -> &str { + get_custom_llm_provider(model, Some(BEDROCK)).map_or(model, |found| found.model) +} + +fn mapped_native_output_flag(catalog: &Catalog, model: &str) -> Option { + let routed = strip_bedrock_routing_prefix(model); + [ + format!("{BEDROCK}/{routed}"), + routed.to_owned(), + without_region(routed).to_owned(), + ] + .iter() + .find_map(|name| flag(catalog.get(name), BEDROCK_NATIVE_OUTPUT)) +} + struct Probe<'a> { catalog: &'a Catalog, declared: &'a dyn Fn(&str) -> Option, @@ -109,7 +176,28 @@ struct Probe<'a> { bare: Option<&'a Fields>, } -impl Probe<'_> { +impl<'a> Probe<'a> { + fn new( + catalog: &'a Catalog, + model: &str, + name: &str, + provider: &str, + declared: &'a dyn Fn(&str) -> Option, + ) -> Self { + Self { + catalog, + declared, + fallbacks: backstop_candidates(model), + info: catalog.model_info( + &candidate_names(name, provider) + .each_ref() + .map(String::as_str), + provider, + ), + bare: catalog.lookup(name).map(|found| found.entry.fields()), + } + } + fn supports(&self, capability: &str) -> bool { if let Some(supported) = (self.declared)(capability) { return supported; @@ -117,10 +205,35 @@ impl Probe<'_> { if let Some(supported) = flag(self.info.as_ref(), capability) { return supported; } - if self.info.is_some() && flag(self.bare, capability) == Some(true) { - return true; + self.bare_supports(capability) + || backstop(self.catalog, &self.fallbacks, capability) == Some(true) + } + + fn factory(&self, capability: &str) -> bool { + match self.info.as_ref() { + Some(info) => { + model_info_flag(info, capability).unwrap_or_else(|| self.bare_supports(capability)) + } + None => false, } - backstop(self.catalog, &self.fallbacks, capability) == Some(true) + } + + fn bare_supports(&self, capability: &str) -> bool { + self.info.is_some() && flag(self.bare, capability) == Some(true) + } +} + +fn model_info_flag(info: &Fields, capability: &str) -> Option { + match capability { + NATIVE_OUTPUT + if info + .get(NATIVE_OUTPUT_BY_DEFAULT) + .is_some_and(|value| truthy(&from_json(value.clone()))) => + { + Some(info.get(BEDROCK_NATIVE_OUTPUT) != Some(&Value::Bool(false))) + } + OUTPUT_CONFIG => None, + _ => flag(Some(info), capability), } } @@ -170,7 +283,7 @@ fn split_model<'a>(model: &'a str, provider: &str) -> &'a str { if !is_bedrock(provider) { return model; } - let routed = without_routing_prefixes(model); + let routed = strip_bedrock_routing_prefix(model); let region_free = without_region(routed); if !region_free.starts_with("arn:") { return routed; @@ -239,7 +352,9 @@ mod tests { use rstest::{fixture, rstest}; use serde_json::Value; - use super::anthropic_model_capabilities; + use super::{anthropic_model_capabilities, bedrock_model_capabilities}; + use crate::base_llm::messages::context::{BedrockCapabilities, NativeOutputSupport}; + use litellm_llms_types::formats::messages::EffortLevel; const CATALOG: &str = r#"{ "claude-test-5": { @@ -460,6 +575,137 @@ mod tests { ); } + #[fixture] + fn bundled() -> Catalog { + Catalog::parse(BUNDLED_REGISTRY).unwrap().value + } + + type Expected = (Option, [bool; 8]); + + fn projected(capabilities: BedrockCapabilities) -> Expected { + let BedrockCapabilities { + output_config_effort_ceiling, + supports_output_config, + supports_prompt_cache_ttl, + supports_tool_search, + supports_per_turn_control, + native_output: + NativeOutputSupport { + enabled, + opt_in, + mapped, + replaces_unforceable_tool, + }, + } = capabilities; + ( + output_config_effort_ceiling, + [ + supports_output_config, + supports_prompt_cache_ttl, + supports_tool_search, + supports_per_turn_control, + enabled, + opt_in, + mapped, + replaces_unforceable_tool, + ], + ) + } + + const XHIGH: Option = Some(EffortLevel::Xhigh); + const PROFILE: &str = "bedrock/arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-opus-4-8"; + const FOUNDATION: &str = "bedrock/invoke/arn:aws:bedrock:us-east-1:123456789012:foundation-model/anthropic.claude-opus-4-8"; + const APPLICATION: &str = + "bedrock/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123"; + + #[rstest] + #[case::us("bedrock/us.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))] + #[case::eu("bedrock/eu.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))] + #[case::global("bedrock/global.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))] + #[case::invoke_prefix("bedrock/invoke/us.anthropic.claude-opus-4-8", (XHIGH, [false, true, true, true, true, true, true, false]))] + #[case::region_path("bedrock/us-east-1/anthropic.claude-opus-4-8", (XHIGH, [false, true, true, true, false, true, false, false]))] + #[case::inference_profile_arn(PROFILE, (XHIGH, [false, true, true, true, true, true, true, false]))] + #[case::invoke_foundation_arn(FOUNDATION, (XHIGH, [false, true, true, true, false, true, false, false]))] + #[case::unprefixed("anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, false, true, false, false]))] + #[case::other_casing("bedrock/ANTHROPIC.CLAUDE-OPUS-4-8", (XHIGH, [true, false, true, true, false, true, false, false]))] + #[case::sonnet_4_5("bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", (None, [false, true, true, true, true, false, true, false]))] + #[case::converse_prefix("bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0", (None, [false, true, true, true, true, false, true, false]))] + #[case::throughput_suffix("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0:1000k", (None, [false, true, true, true, true, false, true, false]))] + #[case::claude_3_5("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", (None, [false; 8]))] + #[case::opus_4_7_without_version("bedrock/global.anthropic.claude-opus-4-7", (XHIGH, [true, true, true, true, false, false, false, false]))] + #[case::opus_4_7_invoke("bedrock/invoke/global.anthropic.claude-opus-4-7-v1:0", (None, [false, false, true, true, false, false, false, false]))] + #[case::unforceable_tool("bedrock/anthropic.claude-fable-5-1", (XHIGH, [true, true, true, true, false, false, false, true]))] + #[case::native_by_default("bedrock/us.anthropic.claude-mythos-5-2", (XHIGH, [true, true, true, true, true, false, true, false]))] + #[case::opt_in_only("bedrock/anthropic.claude-sonnet-5", (XHIGH, [true, true, true, true, false, true, false, false]))] + #[case::unmapped_claude_version("bedrock/anthropic.claude-opus-9-9-v1:0", (None, [false, false, true, true, false, false, false, false]))] + #[case::unmapped_claude("bedrock/anthropic.claude-newthing", (None, [false; 8]))] + #[case::application_profile(APPLICATION, (None, [false; 8]))] + #[case::other_family("bedrock/amazon.nova-pro-v1:0", (None, [false; 8]))] + #[case::unknown("bedrock/unknown-model", (None, [false; 8]))] + fn bedrock_capabilities_match_python( + bundled: Catalog, + #[case] model: &str, + #[case] expected: Expected, + ) { + assert_eq!( + projected(bedrock_model_capabilities(&bundled, model, &|_| None)), + expected + ); + } + + #[rstest] + #[case::native_on("bedrock/us.anthropic.claude-opus-4-8", &[("supports_native_structured_output", true)], (XHIGH, [true, true, true, true, true, true, true, false]))] + #[case::native_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_native_structured_output", false)], (XHIGH, [true, true, true, true, false, true, true, false]))] + #[case::bedrock_native_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_bedrock_native_structured_output", false)], (XHIGH, [true, true, true, true, false, false, true, false]))] + #[case::per_turn_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_per_turn_control", false)], (XHIGH, [true, true, true, false, true, true, true, false]))] + #[case::native_off_opt_in( + "bedrock/us.anthropic.claude-opus-4-8", + &[("supports_native_structured_output", false), ("supports_bedrock_native_structured_output", true)], + (XHIGH, [true, true, true, true, false, true, true, false]) + )] + #[case::unforceable_native_on("bedrock/anthropic.claude-fable-5-1", &[("supports_native_structured_output", true)], (XHIGH, [true, true, true, true, true, false, false, true]))] + #[case::unforceable_opt_in("bedrock/anthropic.claude-fable-5-1", &[("supports_bedrock_native_structured_output", true)], (XHIGH, [true, true, true, true, false, true, false, true]))] + #[case::unforceable_opted_out("bedrock/anthropic.claude-fable-5-1", &[("supports_bedrock_native_structured_output", false)], (XHIGH, [true, true, true, true, false, false, false, false]))] + #[case::bedrock_native_on("bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", &[("supports_bedrock_native_structured_output", true)], (None, [false, true, true, true, true, true, true, false]))] + #[case::per_turn_off_haiku("bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", &[("supports_per_turn_control", false)], (None, [false, true, true, false, true, false, true, false]))] + fn deployment_model_info_overrides_the_catalog_as_python_does( + bundled: Catalog, + #[case] model: &str, + #[case] declared: &[(&str, bool)], + #[case] expected: Expected, + ) { + let lookup = |capability: &str| { + declared + .iter() + .find(|(name, _)| *name == capability) + .map(|(_, supported)| *supported) + }; + + assert_eq!( + projected(bedrock_model_capabilities(&bundled, model, &lookup)), + expected + ); + } + + #[rstest] + #[case::bedrock("bedrock/us.anthropic.claude-opus-4-8", None, true)] + #[case::configured_bedrock("us.anthropic.claude-opus-4-8", Some("bedrock"), true)] + #[case::anthropic("anthropic/claude-opus-4-8", None, false)] + fn only_bedrock_deployments_carry_bedrock_capabilities( + bundled: Catalog, + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, + #[case] carried: bool, + ) { + let capabilities = + anthropic_model_capabilities(&bundled, model, custom_llm_provider, None, &|_| None); + + assert_eq!( + capabilities.bedrock != BedrockCapabilities::default(), + carried + ); + } + #[rstest] fn every_bundled_anthropic_row_shapes_with_its_own_flags() { let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value; diff --git a/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs index 5673b3825ee..4663a92b290 100644 --- a/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -16,7 +16,7 @@ use serde_json::Value; use crate::{ Error, anthropic::{ - beta_gated_fields::strip_beta_gated_system_messages, + beta_gated_fields::strip_beta_gated_request, beta_headers::BetaPolicy, common_utils::{API_KEY_PLACEMENT, existing_betas, strip_trailing_slashes}, messages::{ @@ -34,7 +34,7 @@ use crate::{ base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, messages::{ - context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages}, + context::{MessagesShapeOptions, MessagesTransformContext}, normalization::{normalize_system_role_messages, strip_cache_control_scope}, transformation::BaseMessagesConfig, }, @@ -59,10 +59,6 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { } } - fn logs_dropped_params(&self) -> bool { - true - } - fn retry_after_http_error( &self, status: u16, @@ -78,7 +74,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { request: MessagesRequest, options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, options.reading_unparsed_messages()) + shape_anthropic_messages_request(request, options) } fn get_complete_url( @@ -97,12 +93,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { request: MessagesRequest, context: &MessagesTransformContext, ) -> Result { - transform_messages_request( - request, - &context - .keeping_display_updates() - .reading_unparsed_messages(), - ) + transform_messages_request(request, &context.keeping_display_updates()) } fn secret_names(&self) -> &'static [&'static str] { @@ -211,30 +202,25 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { headers, request, BetaProvider::AzureAi, - UnparsedMessages::Read, )) } - /// Python's transform strips the system fields the final header does not unlock, then - /// normalizes system roles and drops `cache_control.scope`, all after the betas were chosen - /// from the request as sent. + /// Python's transform strips the thinking and system fields the final header does not + /// unlock, then normalizes system roles and drops `cache_control.scope`, all after the betas + /// were chosen from the request as sent. fn finalize_request( &self, request: MessagesRequest, - headers: &Headers, + headers: Headers, context: &MessagesTransformContext, - ) -> MessagesRequest { - let stripped = MessagesRequest { - messages: strip_beta_gated_system_messages( - request.messages, - &existing_betas(headers), - context.drops_unsupported_params, - &request.model, - BetaProvider::AzureAi, - ), - ..request - }; - strip_cache_control_scope(normalize_system_role_messages( + ) -> Result<(MessagesRequest, Headers), Error> { + let stripped = strip_beta_gated_request( + request, + &existing_betas(&headers), + context.drops_unsupported_params, + BetaProvider::AzureAi, + ); + let normalized = strip_cache_control_scope(normalize_system_role_messages( stripped, context .thinking @@ -242,7 +228,8 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { .supports_mid_conversation_system, context.drops_unsupported_params, BetaProvider::AzureAi, - )) + )); + Ok((normalized, headers)) } } @@ -430,7 +417,12 @@ mod tests { .iter() .map(|(name, value)| (name.to_string(), value.to_string())) .collect(); - to_value(AZURE_ANTHROPIC_MESSAGES_CONFIG.finalize_request(request, &headers, context)) + to_value( + AZURE_ANTHROPIC_MESSAGES_CONFIG + .finalize_request(request, headers, context) + .expect("finalized request") + .0, + ) } #[rstest] diff --git a/litellm/rust/crates/llms/src/base_llm/auth.rs b/litellm/rust/crates/llms/src/base_llm/auth.rs index 1d9444a2ac4..7ec1d2fbf9c 100644 --- a/litellm/rust/crates/llms/src/base_llm/auth.rs +++ b/litellm/rust/crates/llms/src/base_llm/auth.rs @@ -9,7 +9,7 @@ use litellm_auth::{AuthServices, CredentialPlacement, TokenProviderHandle}; use litellm_auth_azure::AzureAuthInputs; use litellm_secret_value::SecretValue; -use litellm_auth_aws::{AwsCredentialSource, SigV4Signer}; +use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders, SigV4Signer}; use litellm_auth_gcp::VertexConfig; use litellm_http::request::with_header; @@ -57,6 +57,7 @@ pub enum AuthScheme { region: String, service: &'static str, credentials: Box, + caller_headers: CallerSigningHeaders, }, } @@ -191,11 +192,13 @@ pub async fn resolve_auth( region, service, credentials, + caller_headers, } => Ok(Authenticated { headers, signer: Some( SigV4Signer::resolve(&services.aws, region, service, *credentials, env_lookup) - .await?, + .await? + .with_caller_headers(caller_headers), ), }), } @@ -445,6 +448,7 @@ mod tests { None, "test", ))), + caller_headers: CallerSigningHeaders::Dropped, }, ) .await; diff --git a/litellm/rust/crates/llms/src/base_llm/messages/context.rs b/litellm/rust/crates/llms/src/base_llm/messages/context.rs index 9b65620bf7e..b3dd3b53aeb 100644 --- a/litellm/rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm/rust/crates/llms/src/base_llm/messages/context.rs @@ -1,4 +1,5 @@ use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_llms_types::formats::messages::EffortLevel; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -23,39 +24,10 @@ impl SupportedEffortTiers { } } -/// What the shared Anthropic shaping does with a message, or a beta trigger in one, that Rust -/// could not type. `Read` reads it as Python does, off any mapping. `Skipped` leaves it as sent -/// and finds no beta in it, as main's native route does. Only `azure_ai` reads them: by the -/// owner's ruling `anthropic` keeps main's behaviour. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum UnparsedMessages { - #[default] - Skipped, - Read, -} - -impl UnparsedMessages { - pub fn read(self) -> bool { - self == Self::Read - } -} - #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct MessagesShapeOptions { pub reasoning_auto_summary: bool, pub presanitized: bool, - pub unparsed: UnparsedMessages, -} - -impl MessagesShapeOptions { - /// The same options for a provider whose shaping reads unparsed messages. - #[must_use] - pub fn reading_unparsed_messages(self) -> Self { - Self { - unparsed: UnparsedMessages::Read, - ..self - } - } } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -80,6 +52,46 @@ pub struct MessagesModelCapabilities { pub supports_mid_conversation_system: bool, #[serde(default)] pub effort_tiers: SupportedEffortTiers, + #[serde(default)] + pub bedrock: BedrockCapabilities, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct BedrockCapabilities { + #[serde(default)] + pub output_config_effort_ceiling: Option, + #[serde(default)] + pub supports_output_config: bool, + #[serde(default)] + pub supports_prompt_cache_ttl: bool, + #[serde(default)] + pub supports_tool_search: bool, + #[serde(default)] + pub supports_per_turn_control: bool, + #[serde(default)] + pub native_output: NativeOutputSupport, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct NativeOutputSupport { + #[serde(default)] + pub enabled: bool, + #[serde(default)] + pub opt_in: bool, + #[serde(default)] + pub mapped: bool, + #[serde(default)] + pub replaces_unforceable_tool: bool, +} + +impl NativeOutputSupport { + pub fn is_opt_in(self) -> bool { + !self.enabled && (self.opt_in || self.replaces_unforceable_tool) + } + + pub fn retries(self) -> bool { + self.enabled || self.is_opt_in() + } } fn default_true() -> bool { @@ -99,6 +111,7 @@ impl Default for MessagesModelCapabilities { supports_speed: false, supports_mid_conversation_system: false, effort_tiers: SupportedEffortTiers::default(), + bedrock: BedrockCapabilities::default(), } } } @@ -111,6 +124,7 @@ pub struct ThinkingBudgets { pub high: u64, pub xhigh: u64, pub max: u64, + pub bedrock_min: u64, } impl Default for ThinkingBudgets { @@ -122,6 +136,7 @@ impl Default for ThinkingBudgets { high: 4096, xhigh: 8192, max: 16384, + bedrock_min: 1024, } } } @@ -140,6 +155,9 @@ impl ThinkingBudgets { high: tier("HIGH", defaults.high), xhigh: tier("XHIGH", defaults.xhigh), max: tier("MAX", defaults.max), + bedrock_min: env + .parsed::("BEDROCK_MIN_THINKING_BUDGET_TOKENS") + .unwrap_or(defaults.bedrock_min), } } } @@ -159,9 +177,7 @@ pub struct MessagesTransformContext { pub thinking: ThinkingContext, pub drop_params: bool, pub drops_unsupported_params: bool, - pub unparsed: UnparsedMessages, - /// Logs Python's warning for a param `drop_params` drops because the model lacks it. - pub logs_dropped_params: bool, + pub global_drop_params: bool, } impl MessagesTransformContext { pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self { @@ -182,8 +198,7 @@ impl MessagesTransformContext { }, drop_params, drops_unsupported_params: false, - unparsed: UnparsedMessages::Skipped, - logs_dropped_params: false, + global_drop_params: false, } } @@ -198,13 +213,4 @@ impl MessagesTransformContext { ..*self } } - - /// The same context for a provider whose transform reads unparsed messages. - #[must_use] - pub fn reading_unparsed_messages(&self) -> Self { - Self { - unparsed: UnparsedMessages::Read, - ..*self - } - } } diff --git a/litellm/rust/crates/llms/src/base_llm/messages/streaming.rs b/litellm/rust/crates/llms/src/base_llm/messages/streaming.rs index 8c301e67de3..42cafb23805 100644 --- a/litellm/rust/crates/llms/src/base_llm/messages/streaming.rs +++ b/litellm/rust/crates/llms/src/base_llm/messages/streaming.rs @@ -5,7 +5,8 @@ pub use crate::base_llm::base_model_iterator::ByteStream; use crate::{Error, base_llm::sse}; pub type EventStream = sse::EventStream; -pub type StreamDecoder = fn(ByteStream) -> EventStream; +pub type FrameStream = futures_util::stream::BoxStream<'static, Result>; +pub type StreamDecoder = fn(ByteStream, Vec<(String, String)>) -> FrameStream; fn no_control_frames(_: &litellm_framer::sse::SseEvent) -> bool { false @@ -36,7 +37,7 @@ mod tests { MessagesContentBlockDelta, MessagesStreamUsage, }; use rstest::rstest; - use serde_json::json; + use serde_json::{Map, json}; use super::*; @@ -61,6 +62,7 @@ mod tests { output_tokens: Some(7), ..MessagesStreamUsage::default() }), + extra: Map::new(), }, ] } @@ -88,7 +90,10 @@ mod tests { text: "hello".into(), }, }, - MessagesStreamEvent::MessageStop { usage: None }, + MessagesStreamEvent::MessageStop { + usage: None, + extra: Map::new(), + }, ]; let streamed = event_stream(in_pieces(wire.as_bytes(), size)) .try_collect::>() @@ -138,7 +143,11 @@ mod tests { #[test] fn an_event_is_named_by_its_type() { - let encoded = encode(&MessagesStreamEvent::MessageStop { usage: None }).unwrap(); + let encoded = encode(&MessagesStreamEvent::MessageStop { + usage: None, + extra: Map::new(), + }) + .unwrap(); assert_eq!( encoded, Bytes::from(format!( diff --git a/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs index d76d660f9ea..74fa13d293c 100644 --- a/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs @@ -49,15 +49,28 @@ pub trait BaseMessagesConfig: Sync { Ok(request) } - fn tool_custom_field(&self, _model: &str) -> ToolCustomField { - ToolCustomField::Keep + fn native_output_fallback( + &self, + _request: &MessagesRequest, + _context: &MessagesTransformContext, + ) -> Option { + None } - /// Whether a param `drop_params` drops because the model lacks it logs Python's warning. - fn logs_dropped_params(&self) -> bool { + fn retries_without_native_output( + &self, + _status: u16, + _error_text: &str, + _request_body: &Value, + _model: &str, + ) -> bool { false } + fn tool_custom_field(&self, _model: &str) -> ToolCustomField { + ToolCustomField::Keep + } + fn transform_anthropic_messages_response( &self, _model: &str, @@ -149,14 +162,15 @@ pub trait BaseMessagesConfig: Sync { } /// The request steps Python runs once the final `anthropic-beta` header is known, which - /// read that header or must not change which betas the request asks for. + /// read that header, must not change which betas the request asks for, or rewrite the + /// headers that go out with the body. fn finalize_request( &self, request: MessagesRequest, - _headers: &Headers, + headers: Headers, _context: &MessagesTransformContext, - ) -> MessagesRequest { - request + ) -> Result<(MessagesRequest, Headers), Error> { + Ok((request, headers)) } /// The JSON that goes on the wire, for a host whose body differs from the typed request: @@ -249,7 +263,6 @@ mod tests { MessagesShapeOptions { reasoning_auto_summary, presanitized, - ..MessagesShapeOptions::default() } ), Ok(request) diff --git a/litellm/rust/crates/llms/src/bedrock/audio_transcription/mod.rs b/litellm/rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 1c976c1f73d..d3a14b98a26 100644 --- a/litellm/rust/crates/llms/src/bedrock/audio_transcription/mod.rs +++ b/litellm/rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -1,8 +1,7 @@ use litellm_auth::AwsParams; use litellm_auth_aws::{ - AwsCredentialSource, bedrock_model_id_and_region, - constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, - resolve_bedrock_region, + AwsCredentialSource, CallerSigningHeaders, bedrock_model_id_and_region, + bedrock_runtime_endpoint, constants::BEDROCK_SERVICE, resolve_bedrock_region, }; use litellm_core_utils::core_helpers::json_type_name; use litellm_llms_types::{ @@ -161,7 +160,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { .map(str::trim) .filter(|value| !value.is_empty()) .map(str::to_string) - .unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion)); + .unwrap_or_else(|| bedrock_runtime_endpoint(®ion)); Ok(format!( "{}/model/{model_id}/converse", endpoint.trim_end_matches('/') @@ -187,6 +186,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { region: resolve_bedrock_region(model_region.as_deref(), ¶ms, env_lookup), service: BEDROCK_SERVICE, credentials: Box::new(AwsCredentialSource::from_params(¶ms, env_lookup)), + caller_headers: CallerSigningHeaders::Dropped, }, }) } diff --git a/litellm/rust/crates/llms/src/bedrock/chat/AGENTS.md b/litellm/rust/crates/llms/src/bedrock/chat/AGENTS.md index 8627e3f73ef..605c2b33698 100644 --- a/litellm/rust/crates/llms/src/bedrock/chat/AGENTS.md +++ b/litellm/rust/crates/llms/src/bedrock/chat/AGENTS.md @@ -2,4 +2,5 @@ - Chat Completions over Converse. Converse wire docs live in `llms-types/src/providers/bedrock/AGENTS.md` - `topK` is not forwarded, because Python's placement depends on the model catalog this crate cannot see -- `invoke_handler.rs` decodes InvokeModel event streams for Claude, reusing `anthropic/chat` response handling +- `invoke_handler.rs` ports Python's `AWSEventStreamDecoder`: it decodes InvokeModel event streams the way botocore's `EventStreamBuffer` does +- `invoke_transformations/base_invoke_transformation.rs` ports the InvokeModel helpers of Python's `base_invoke_transformation.py` diff --git a/litellm/rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm/rust/crates/llms/src/bedrock/chat/converse_transformation.rs index e02e7b9a155..cdc3581a4ce 100644 --- a/litellm/rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -3,8 +3,9 @@ use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; use litellm_auth_aws::{ - AwsCredentialSource, bedrock_model_id_and_region, - constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, + AwsCredentialSource, CallerSigningHeaders, bedrock_model_id_and_region, + bedrock_runtime_endpoint, + constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_SERVICE}, resolve_bedrock_region, }; use litellm_core_utils::decline::Decline; @@ -128,7 +129,7 @@ impl BaseConfig for AmazonConverseConfig { .map(str::trim) .filter(|value| !value.is_empty()) .map(str::to_string) - .unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion)); + .unwrap_or_else(|| bedrock_runtime_endpoint(®ion)); let endpoint = endpoint.trim_end_matches('/'); // A host that already built the full Converse URL (LiteLLM's Python // path encodes the model id itself) passes it through untouched, the @@ -253,6 +254,7 @@ impl BaseConfig for AmazonConverseConfig { region: resolve_bedrock_region(model_region.as_deref(), ¶ms, env_lookup), service: BEDROCK_SERVICE, credentials: Box::new(AwsCredentialSource::from_params(¶ms, env_lookup)), + caller_headers: CallerSigningHeaders::Botocore, }, }) } @@ -582,16 +584,25 @@ mod tests { ); } - #[test] - fn builds_the_converse_url_from_the_region_in_the_model_id() { + #[rstest] + #[case::commercial( + "us-east-1/anthropic.claude-v2", + "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse" + )] + #[case::china_partition( + "cn-north-1/anthropic.claude-v2", + "https://bedrock-runtime.cn-north-1.amazonaws.com.cn/model/anthropic.claude-v2/converse" + )] + fn builds_the_converse_url_from_the_region_in_the_model_id( + #[case] model: &str, + #[case] expected: &str, + ) { let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; assert_eq!( config - .get_complete_url(None, "us-east-1/anthropic.claude-v2", &Map::new(), &|_| { - None - }) + .get_complete_url(None, model, &Map::new(), &|_| None) .expect("url builds"), - "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse" + expected ); } diff --git a/litellm/rust/crates/llms/src/bedrock/chat/invoke_handler.rs b/litellm/rust/crates/llms/src/bedrock/chat/invoke_handler.rs index 15a6ff38822..626c6faceaf 100644 --- a/litellm/rust/crates/llms/src/bedrock/chat/invoke_handler.rs +++ b/litellm/rust/crates/llms/src/bedrock/chat/invoke_handler.rs @@ -1,132 +1,685 @@ +use std::sync::{Arc, Mutex, PoisonError}; + use base64::Engine; -use bytes::Buf; -use futures_util::{Stream, StreamExt}; +use bytes::BytesMut; +use futures_util::{StreamExt, stream}; use litellm_framer::{ EventStreamError, - aws_event_stream::{AwsEventStreamCodec, Message}, + aws_event_stream::{AwsEventStreamCodec, HeaderValue, Message}, frames, }; -use litellm_llms_types::formats::messages::streaming::MessagesStreamEvent; -use serde::Deserialize; +use litellm_python_compat::{Value as PythonValue, json as python_json, repr}; use serde_json::Value; +use tokio_util::codec::Decoder; -use crate::{ - Error, - base_llm::{ - base_model_iterator::tunnelled_http, - messages::streaming::{ByteStream, EventStream}, - }, -}; +use crate::{Error, ErrorDetail, base_llm::base_model_iterator::tunnelled_http}; -#[derive(Deserialize)] -struct InvokeChunkPayload { - bytes: String, +const STREAM_HEAD_BYTES: usize = 200; +const PRELUDE_BYTES: usize = 12; +const MAX_HEADERS_BYTES: u32 = 128 * 1024; +const MAX_PAYLOAD_BYTES: i64 = 24 * 1024 * 1024; + +const PROVIDER_REQUEST_ID_HEADERS: [&str; 7] = [ + "x-amzn-requestid", + "x-request-id", + "request-id", + "x-ms-request-id", + "apim-request-id", + "x-goog-request-id", + "cf-ray", +]; + +const STREAM_EVENT_STATUSES: [(&str, Option); 13] = [ + ("chunk", None), + ("messageStart", None), + ("contentBlockStart", None), + ("contentBlockDelta", None), + ("contentBlockStop", None), + ("messageStop", None), + ("metadata", None), + ("internalServerException", Some(500)), + ("modelStreamErrorException", Some(424)), + ("validationException", Some(400)), + ("throttlingException", Some(429)), + ("modelTimeoutException", Some(408)), + ("serviceUnavailableException", Some(503)), +]; + +const RESPONSE_STREAM_EXCEPTION_STATUSES: [(&str, u16); 6] = [ + ("internalServerException", 500), + ("modelStreamErrorException", 424), + ("validationException", 400), + ("throttlingException", 429), + ("modelTimeoutException", 408), + ("serviceUnavailableException", 503), +]; + +/// An exception Python's Bedrock stream decoder raises as it is, unmapped: botocore's +/// `EventStreamBuffer` parser errors, `json.loads` failing on an event payload, and the +/// chunk parser's `pop` on a payload that is not a JSON object. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum BedrockStreamRaise { + #[error("Checksum mismatch: expected 0x{expected:08x}, calculated 0x{calculated:08x}")] + ChecksumMismatch { expected: u32, calculated: u32 }, + #[error("Header length of {0} exceeded the maximum of {MAX_HEADERS_BYTES}")] + InvalidHeadersLength(u32), + #[error("Payload length of {0} exceeded the maximum of {MAX_PAYLOAD_BYTES}")] + InvalidPayloadLength(i64), + #[error("Duplicate header present: \"{0}\"")] + DuplicateHeader(String), + #[error("Bedrock event payload is not JSON: {0}")] + InvalidJson(String), + #[error("Bedrock chunk is not a JSON object: {0}")] + NotAnObject(String), } -pub fn decode_invoke_chunk(message: Message) -> Result { - let payload: InvokeChunkPayload = - serde_json::from_slice(message.payload()).map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid("Bedrock event payload", error)) - })?; - let chunk = base64::engine::general_purpose::STANDARD - .decode(payload.bytes) - .map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid( - "Bedrock event payload base64", - error, - )) - })?; - serde_json::from_slice(&chunk).map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid("Anthropic stream event", error)) +pub(crate) fn raised(raise: BedrockStreamRaise) -> Error { + Error::InvalidResponse(ErrorDetail::failed("Bedrock event stream", raise)) +} + +/// Port of Python's `_EventStreamTally`. +#[derive(Default)] +struct EventStreamTally { + bytes_received: usize, + bytes_decoded: usize, + events: usize, + recognized_events: usize, + unrecognized_event_types: std::collections::BTreeSet, + unrecognized_head: Vec, + head: Vec, +} + +type SharedEventStreamTally = Arc>; + +fn with_tally( + tally: &SharedEventStreamTally, + update: impl FnOnce(&mut EventStreamTally) -> T, +) -> T { + update(&mut tally.lock().unwrap_or_else(PoisonError::into_inner)) +} + +#[derive(Debug, thiserror::Error)] +enum FrameError { + #[error(transparent)] + Raise(#[from] BedrockStreamRaise), + #[error(transparent)] + Framing(#[from] EventStreamError), +} + +fn be_u32(bytes: &[u8], at: usize) -> u32 { + u32::from_be_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]]) +} + +fn checked(data: &[u8], expected: u32, seed: u32) -> Result<(), BedrockStreamRaise> { + let mut hasher = crc32fast::Hasher::new_with_initial(seed); + hasher.update(data); + let calculated = hasher.finalize(); + if calculated == expected { + return Ok(()); + } + Err(BedrockStreamRaise::ChecksumMismatch { + expected, + calculated, }) } -pub fn invoke_chunk_stream(input: S) -> impl Stream> + Send -where - S: Stream> + Send, - B: Buf + Send, - E: std::error::Error + Send + Sync + 'static, -{ - frames(input, AwsEventStreamCodec).map(|message| decode_invoke_chunk(message.map_err(framing)?)) +/// Validates each frame in the order botocore's `EventStreamBuffer` does, so a body that is not +/// an event stream fails with the parser error Python raises. +struct BotocoreCodec { + tally: SharedEventStreamTally, } -fn framing(error: EventStreamError) -> Error { +impl BotocoreCodec { + fn prelude(src: &[u8]) -> Result { + let total = be_u32(src, 0); + let headers = be_u32(src, 4); + checked(&src[..PRELUDE_BYTES - 4], be_u32(src, 8), 0)?; + if headers > MAX_HEADERS_BYTES { + return Err(BedrockStreamRaise::InvalidHeadersLength(headers)); + } + let payload = i64::from(total) - i64::from(headers) - 16; + if payload > MAX_PAYLOAD_BYTES { + return Err(BedrockStreamRaise::InvalidPayloadLength(payload)); + } + Ok(total as usize) + } + + fn unique_headers(message: &Message) -> Result<(), BedrockStreamRaise> { + let mut seen = std::collections::HashSet::new(); + message + .headers() + .iter() + .map(|header| header.name().as_str()) + .find(|name| !seen.insert(*name)) + .map_or(Ok(()), |name| { + Err(BedrockStreamRaise::DuplicateHeader(name.to_string())) + }) + } +} + +impl Decoder for BotocoreCodec { + type Item = Message; + type Error = FrameError; + + fn decode(&mut self, src: &mut BytesMut) -> Result, FrameError> { + if src.len() < PRELUDE_BYTES { + return Ok(None); + } + let length = Self::prelude(src)?; + if src.len() < length { + return Ok(None); + } + if length >= 16 { + checked( + &src[PRELUDE_BYTES - 4..length - 4], + be_u32(src, length - 4), + be_u32(src, 8), + )?; + } + let Some(message) = AwsEventStreamCodec.decode(src)? else { + return Ok(None); + }; + Self::unique_headers(&message)?; + with_tally(&self.tally, |tally| tally.bytes_decoded += length); + Ok(Some(message)) + } + + fn decode_eof(&mut self, src: &mut BytesMut) -> Result, FrameError> { + self.decode(src) + } +} + +impl From for FrameError { + fn from(error: std::io::Error) -> Self { + Self::Framing(EventStreamError::Body(error)) + } +} + +fn header_text<'a>(message: &'a Message, name: &str) -> Option<&'a str> { + message + .headers() + .iter() + .find(|header| header.name().as_str() == name) + .and_then(|header| match header.value() { + HeaderValue::String(value) => Some(value.as_str()), + _ => None, + }) +} + +fn event_status(event_type: &str) -> Option> { + STREAM_EVENT_STATUSES + .iter() + .find(|(known, _)| *known == event_type) + .map(|(_, status)| *status) +} + +fn python_bytes_repr(bytes: &[u8]) -> String { + repr::repr(&PythonValue::Bytes(bytes.to_vec())) +} + +fn python_optional_str_repr(value: Option<&str>) -> String { + repr::repr(&value.map_or(PythonValue::None, |text| PythonValue::Str(text.to_string()))) +} + +fn response_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) +} + +fn request_id_headers(headers: &[(String, String)]) -> Vec<(String, String)> { + headers + .iter() + .filter(|(name, value)| { + PROVIDER_REQUEST_ID_HEADERS.contains(&name.to_lowercase().as_str()) && value.is_ascii() + }) + .cloned() + .collect() +} + +fn bedrock_error(status: u16, message: String, headers: &[(String, String)]) -> Error { + Error::Http(litellm_http::Error::upstream_with_headers( + status, + message, + request_id_headers(headers), + )) +} + +/// Port of Python's `_EventStreamTally.add_event`. +fn add_event(tally: &SharedEventStreamTally, message: &Message) { + let message_type = header_text(message, ":message-type"); + let event_type = header_text(message, ":event-type"); + with_tally(tally, |tally| { + tally.events += 1; + if message_type != Some("event") || event_type.and_then(event_status).is_some() { + tally.recognized_events += 1; + return; + } + tally + .unrecognized_event_types + .insert(event_type.unwrap_or("").to_string()); + if tally.unrecognized_head.is_empty() { + let payload = message.payload(); + tally.unrecognized_head = payload[..payload.len().min(STREAM_HEAD_BYTES)].to_vec(); + } + }); +} + +/// Port of Python's `_EventStreamTally.undecoded_stream_error`. +fn undecoded_stream_error(tally: &EventStreamTally, headers: &[(String, String)]) -> Option { + let undecoded = tally.bytes_received.saturating_sub(tally.bytes_decoded); + if tally.recognized_events > 0 && undecoded == 0 { + return None; + } + let detail = format!( + "content-type={}, x-amzn-requestid={}, {} bytes received", + python_optional_str_repr(response_header(headers, "content-type")), + python_optional_str_repr(response_header(headers, "x-amzn-requestid")), + tally.bytes_received + ); + let message = if tally.events == 0 { + format!( + "Bedrock answered the stream with HTTP 200 but its body decoded to no events ({detail}, first bytes={})", + python_bytes_repr(&tally.head) + ) + } else if tally.recognized_events == 0 { + let event_types = PythonValue::List( + tally + .unrecognized_event_types + .iter() + .cloned() + .map(PythonValue::Str) + .collect(), + ); + format!( + "Bedrock answered the stream with HTTP 200 but none of its {} events carried a known event type (event types={}, {detail}, first payload={})", + tally.events, + repr::repr(&event_types), + python_bytes_repr(&tally.unrecognized_head) + ) + } else { + format!( + "Bedrock stream ended with {undecoded} undecoded bytes after {} events ({detail})", + tally.events + ) + }; + Some(bedrock_error(502, message, headers)) +} + +fn framing(error: FrameError) -> Error { + let error = match error { + FrameError::Raise(raise) => return raised(raise), + FrameError::Framing(error) => error, + }; match &error { EventStreamError::Body(body) if let Some(http) = tunnelled_http(body) => { Error::Http(http.clone()) } - EventStreamError::Body(_) - | EventStreamError::InvalidLength(_) - | EventStreamError::Truncated - | EventStreamError::Malformed(_) => { - Error::InvalidResponse(crate::ErrorDetail::failed("stream framing", error)) - } + _ => Error::InvalidResponse(ErrorDetail::failed("stream framing", error)), } } -pub fn decode_invoke_anthropic_chunk(chunk: Value) -> Result { - serde_json::from_value(chunk).map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid("Anthropic stream event", error)) - }) +fn invalid_payload(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::InvalidResponse(ErrorDetail::invalid("Bedrock event payload", error)) } -pub fn invoke_anthropic_event_stream(bytes: ByteStream) -> EventStream { - Box::pin(invoke_chunk_stream(bytes).map(|chunk| decode_invoke_anthropic_chunk(chunk?))) +/// Port of Python's `bedrock_event_chunk_text`. +fn bedrock_event_chunk_text(payload: &[u8]) -> Result>, Error> { + let parsed: Value = serde_json::from_slice(payload).map_err(invalid_payload)?; + let Some(chunk) = parsed.as_object().filter(|chunk| !chunk.is_empty()) else { + return Ok(None); + }; + let encoded = chunk + .get("bytes") + .and_then(Value::as_str) + .ok_or(Error::InvalidResponse(ErrorDetail::MissingField { + subject: "Bedrock chunk event", + field: "bytes", + }))?; + base64::engine::general_purpose::STANDARD + .decode(encoded) + .map(Some) + .map_err(invalid_payload) +} + +/// Port of Python's `AWSEventStreamDecoder._parse_message_from_response`, parsed with `json.loads`. +fn parse_message_from_response( + message: &Message, + headers: &[(String, String)], +) -> Result, Error> { + let body = String::from_utf8_lossy(message.payload()); + if header_text(message, ":message-type") != Some("event") { + let error = match header_text(message, ":exception-type") { + Some(exception_type) => { + let status = RESPONSE_STREAM_EXCEPTION_STATUSES + .iter() + .find(|(known, _)| *known == exception_type) + .map_or(400, |(_, status)| *status); + bedrock_error(status, format!("{exception_type} {body}"), headers) + } + None => bedrock_error(400, body.into_owned(), headers), + }; + return Err(error); + } + let event_type = header_text(message, ":event-type"); + if let Some((event_type, status)) = + event_type.and_then(|event_type| Some((event_type, event_status(event_type).flatten()?))) + { + return Err(bedrock_error( + status, + format!("{event_type} {body}"), + headers, + )); + } + let text = if event_type == Some("chunk") { + bedrock_event_chunk_text(message.payload())? + } else { + Some(message.payload().to_vec()).filter(|payload| !payload.is_empty()) + }; + text.filter(|text| !text.is_empty()) + .map(|text| { + let text = String::from_utf8_lossy(&text); + python_json::loads(&text) + .map_err(|_| raised(BedrockStreamRaise::InvalidJson(text.into_owned()))) + }) + .transpose() +} + +/// Port of Python's `AWSEventStreamDecoder.aiter_bytes`, yielding the decoded JSON of each event. +pub fn aiter_bytes( + bytes: crate::base_llm::messages::streaming::ByteStream, + headers: Vec<(String, String)>, +) -> impl futures_util::Stream> + Send { + let tally = SharedEventStreamTally::default(); + let counted = { + let tally = tally.clone(); + bytes.inspect(move |chunk| { + if let Ok(chunk) = chunk { + with_tally(&tally, |tally| { + tally.bytes_received += chunk.len(); + let room = STREAM_HEAD_BYTES.saturating_sub(tally.head.len()); + tally + .head + .extend_from_slice(&chunk[..chunk.len().min(room)]); + }); + } + }) + }; + let headers = Arc::new(headers); + let events = { + let tally = tally.clone(); + let headers = headers.clone(); + frames( + counted, + BotocoreCodec { + tally: tally.clone(), + }, + ) + .map(move |message| { + let message = message.map_err(framing)?; + add_event(&tally, &message); + parse_message_from_response(&message, &headers) + }) + }; + let end = stream::once(async move { + with_tally(&tally, |tally| undecoded_stream_error(tally, &headers)).map_or(Ok(None), Err) + }); + events + .chain(end) + .scan(false, |failed, item| { + if *failed { + return futures_util::future::ready(None); + } + *failed = item.is_err(); + futures_util::future::ready(Some(item)) + }) + .filter_map(|item| futures_util::future::ready(item.transpose())) } #[cfg(test)] mod tests { use aws_smithy_eventstream::frame::write_message_to; - use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; + use aws_smithy_types::event_stream::{ + Header, HeaderValue as WireHeaderValue, Message as WireMessage, + }; use base64::engine::general_purpose::STANDARD; use bytes::Bytes; - use futures_util::TryStreamExt; - use litellm_llms_types::formats::messages::streaming::MessagesContentBlockDelta; + use futures_util::StreamExt; + use rstest::rstest; + use serde_json::json; use super::*; - use crate::base_llm::messages::streaming; - fn in_pieces(wire: &[u8]) -> ByteStream { - let pieces: Vec = wire.chunks(3).map(Bytes::copy_from_slice).collect(); - futures_util::stream::iter(pieces.into_iter().map(Ok)).boxed() - } - - const TEXT_DELTA: &str = - r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}"#; - - fn aws_wire(chunk: &str) -> Vec { - let payload = serde_json::json!({"bytes": STANDARD.encode(chunk)}); - let message = Message::new(Bytes::from(serde_json::to_vec(&payload).unwrap())).add_header( - Header::new(":event-type", HeaderValue::String("chunk".into())), + fn frame(headers: &[(&str, &str)], payload: &[u8]) -> Vec { + let message = headers.iter().fold( + WireMessage::new(Bytes::copy_from_slice(payload)), + |message, (name, value)| { + message.add_header(Header::new( + name.to_string(), + WireHeaderValue::String(value.to_string().into()), + )) + }, ); let mut wire = Vec::new(); write_message_to(&message, &mut wire).unwrap(); wire } + fn chunk(event: &Value) -> Vec { + let payload = json!({"bytes": STANDARD.encode(event.to_string())}); + frame( + &[(":event-type", "chunk"), (":message-type", "event")], + payload.to_string().as_bytes(), + ) + } + + fn response_headers() -> Vec<(String, String)> { + vec![ + ( + "content-type".into(), + "application/vnd.amazon.eventstream".into(), + ), + ("x-amzn-requestid".into(), "rid-1".into()), + ("x-other".into(), "dropped".into()), + ] + } + + async fn decoded(wire: Vec) -> (Vec, Option) { + let bytes = futures_util::stream::iter( + wire.chunks(5) + .map(|piece| Ok(Bytes::copy_from_slice(piece))) + .collect::>(), + ) + .boxed(); + let items: Vec> = + aiter_bytes(bytes, response_headers()).collect().await; + let (events, errors): (Vec<_>, Vec<_>) = items.into_iter().partition(Result::is_ok); + ( + events.into_iter().filter_map(Result::ok).collect(), + errors.into_iter().find_map(Result::err), + ) + } + + fn upstream(error: Option) -> (u16, String, Vec<(String, String)>) { + let Some(Error::Http(error)) = error else { + panic!("expected an upstream error, got {error:?}"); + }; + let (status, body) = error.upstream_response().unwrap(); + (status, body.to_string(), error.upstream_headers().to_vec()) + } + #[tokio::test] - async fn aws_and_sse_framing_decode_to_the_same_anthropic_events() { - let aws = aws_wire(TEXT_DELTA); - let sse = format!("event: content_block_delta\ndata: {TEXT_DELTA}\n\n"); - - let from_aws = invoke_anthropic_event_stream(in_pieces(&aws)) - .try_collect::>() - .await - .unwrap(); - let from_sse = streaming::event_stream(in_pieces(sse.as_bytes())) - .try_collect::>() - .await - .unwrap(); - + async fn chunk_events_decode_to_anthropic_events() { + let event = json!({"type": "message_stop"}); + let (events, error) = decoded(chunk(&event)).await; assert_eq!( - from_aws, - vec![MessagesStreamEvent::ContentBlockDelta { - index: 0, - delta: MessagesContentBlockDelta::TextDelta { - text: "hello".into(), - }, - }] + (events, error.is_none()), + (vec![python_json::loads(&event.to_string()).unwrap()], true) ); - assert_eq!(from_aws, from_sse); + } + + #[rstest] + #[case::throttling("throttlingException", 429)] + #[case::model_stream_error("modelStreamErrorException", 424)] + #[case::validation("validationException", 400)] + #[case::unmodeled("somethingNewException", 400)] + #[tokio::test] + async fn exception_frames_raise_their_modeled_status( + #[case] exception_type: &str, + #[case] status: u16, + ) { + let wire = [ + chunk(&json!({"type": "ping"})), + frame( + &[ + (":message-type", "exception"), + (":exception-type", exception_type), + ], + br#"{"message":"slow down"}"#, + ), + ] + .concat(); + let (events, error) = decoded(wire).await; + assert_eq!(events.len(), 1); + assert_eq!( + upstream(error), + ( + status, + format!(r#"{exception_type} {{"message":"slow down"}}"#), + vec![("x-amzn-requestid".to_string(), "rid-1".to_string())] + ) + ); + } + + #[tokio::test] + async fn an_error_event_type_raises_its_status() { + let wire = frame( + &[ + (":message-type", "event"), + (":event-type", "internalServerException"), + ], + b"boom", + ); + assert_eq!(upstream(decoded(wire).await.1).0, 500); + } + + #[tokio::test] + async fn an_empty_body_is_a_502() { + let (_, body, _) = upstream(decoded(Vec::new()).await.1); + assert_eq!( + body, + "Bedrock answered the stream with HTTP 200 but its body decoded to no events (content-type='application/vnd.amazon.eventstream', x-amzn-requestid='rid-1', 0 bytes received, first bytes=b'')" + ); + } + + #[tokio::test] + async fn unknown_event_types_are_a_502() { + let wire = frame( + &[(":message-type", "event"), (":event-type", "mystery")], + b"{}", + ); + let (status, body, _) = upstream(decoded(wire.clone()).await.1); + assert_eq!( + (status, body), + ( + 502, + format!( + "Bedrock answered the stream with HTTP 200 but none of its 1 events carried a known event type (event types=['mystery'], content-type='application/vnd.amazon.eventstream', x-amzn-requestid='rid-1', {} bytes received, first payload=b'{{}}')", + wire.len() + ) + ) + ); + } + + #[tokio::test] + async fn trailing_undecoded_bytes_are_a_502() { + let complete = chunk(&json!({"type": "ping"})); + let truncated = chunk(&json!({"type": "message_stop"})); + let wire = [complete, truncated[..20].to_vec()].concat(); + let (events, error) = decoded(wire).await; + let (status, body, _) = upstream(error); + assert_eq!(events.len(), 1); + assert_eq!(status, 502); + assert!( + body.starts_with("Bedrock stream ended with 20 undecoded bytes after 1 events ("), + "{body}" + ); + } + + #[tokio::test] + async fn decoding_stops_at_the_first_error() { + let wire = [ + frame( + &[ + (":message-type", "exception"), + (":exception-type", "throttlingException"), + ], + b"x", + ), + chunk(&json!({"type": "ping"})), + ] + .concat(); + let bytes = futures_util::stream::iter([Ok(Bytes::from(wire))]).boxed(); + let items: Vec> = aiter_bytes(bytes, Vec::new()).collect().await; + assert_eq!(items.len(), 1); + assert!(items[0].is_err()); + } + + fn raise_of(error: Option) -> BedrockStreamRaise { + litellm_error::find::(&error.expect("an error")) + .cloned() + .expect("a botocore parser error") + } + + #[tokio::test] + async fn a_json_body_fails_the_prelude_checksum_as_botocore_does() { + let body = br#"{"id":"msg_1","type":"message","content":[]}"#; + assert_eq!( + raise_of(decoded(body.to_vec()).await.1), + BedrockStreamRaise::ChecksumMismatch { + expected: be_u32(body, 8), + calculated: crc32fast::hash(&body[..8]), + } + ); + } + + fn with_prelude(total: u32, headers: u32) -> Vec { + let mut prelude = [total.to_be_bytes(), headers.to_be_bytes()].concat(); + let crc = crc32fast::hash(&prelude); + prelude.extend_from_slice(&crc.to_be_bytes()); + prelude + } + + #[rstest] + #[case::headers_too_long( + with_prelude(200_000, 140_000), + BedrockStreamRaise::InvalidHeadersLength(140_000) + )] + #[case::payload_too_long( + with_prelude(30_000_000, 0), + BedrockStreamRaise::InvalidPayloadLength(30_000_000 - 16) + )] + #[case::duplicate_header( + frame(&[(":message-type", "event"), (":message-type", "event")], b"{}"), + BedrockStreamRaise::DuplicateHeader(":message-type".into()) + )] + #[tokio::test] + async fn a_malformed_frame_raises_the_botocore_parser_error( + #[case] wire: Vec, + #[case] expected: BedrockStreamRaise, + ) { + assert_eq!(raise_of(decoded(wire).await.1), expected); + } + + #[tokio::test] + async fn a_corrupt_message_checksum_raises_a_checksum_mismatch() { + let mut wire = chunk(&json!({"type": "ping"})); + let last = wire.len() - 1; + wire[last] ^= 0xff; + assert!(matches!( + raise_of(decoded(wire).await.1), + BedrockStreamRaise::ChecksumMismatch { .. } + )); } } diff --git a/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/base_invoke_transformation.rs b/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/base_invoke_transformation.rs new file mode 100644 index 00000000000..8a221943b1f --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/base_invoke_transformation.rs @@ -0,0 +1,186 @@ +use litellm_python_compat::{json as python_json, repr}; +use serde_json::{Map, Value}; + +use crate::Error; + +pub const GUARDRAIL_CONFIG_PARAM: &str = "guardrailConfig"; + +pub(crate) const GUARDRAIL_IDENTIFIER_HEADER: &str = "X-Amzn-Bedrock-GuardrailIdentifier"; +pub(crate) const GUARDRAIL_VERSION_HEADER: &str = "X-Amzn-Bedrock-GuardrailVersion"; +const GUARDRAIL_TRACE_HEADER: &str = "X-Amzn-Bedrock-Trace"; +const GUARDRAIL_CONFIG_EXPECTED_FORMAT: &str = "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"; +const GUARDRAIL_TRACE_VALUES: [&str; 3] = ["enabled", "disabled", "enabled_full"]; + +fn python_repr(value: &Value) -> String { + repr::to_str(&python_json::from_json(value.clone())) +} + +fn python_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) => "dict", + } +} + +fn validation_error(location: &str, message: &str, kind: &str, input: &Value) -> String { + let location = if location.is_empty() { + String::new() + } else { + format!("\n{location}") + }; + format!( + "1 validation error for GuardrailConfigBlock{location}\n {message} [type={kind}, input_value={}, input_type={}]", + python_repr(input), + python_type_name(input) + ) +} + +fn guardrail_violation(config: &Map) -> Option { + ["guardrailIdentifier", "guardrailVersion"] + .into_iter() + .find_map(|field| { + config + .get(field) + .filter(|value| !value.is_string()) + .map(|value| { + validation_error( + field, + "Input should be a valid string", + "string_type", + value, + ) + }) + }) + .or_else(|| { + config + .get("trace") + .filter(|value| { + !value + .as_str() + .is_some_and(|trace| GUARDRAIL_TRACE_VALUES.contains(&trace)) + }) + .map(|value| { + validation_error( + "trace", + "Input should be 'enabled', 'disabled' or 'enabled_full'", + "literal_error", + value, + ) + }) + }) +} + +fn guardrail_error(message: String) -> Error { + Error::Http(litellm_http::Error::upstream(400, message)) +} + +/// Port of Python's `bedrock_invoke_guardrail_headers`. +pub(crate) fn bedrock_invoke_guardrail_headers( + raw: &Value, +) -> Result, Error> { + let Value::Object(config) = raw else { + return Err(guardrail_error(format!( + "Invalid guardrailConfig={}. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}. Error: {}", + python_repr(raw), + validation_error("", "Input should be a valid dictionary", "dict_type", raw) + ))); + }; + if let Some(violation) = guardrail_violation(config) { + return Err(guardrail_error(format!( + "Invalid guardrailConfig={}. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}. Error: {violation}", + python_repr(raw) + ))); + } + let Some(identifier) = config.get("guardrailIdentifier").and_then(Value::as_str) else { + return Err(guardrail_error(format!( + "guardrailConfig={} is missing 'guardrailIdentifier'. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}", + python_repr(raw) + ))); + }; + Ok([ + (GUARDRAIL_IDENTIFIER_HEADER, Some(identifier.to_string())), + ( + GUARDRAIL_VERSION_HEADER, + config + .get("guardrailVersion") + .and_then(Value::as_str) + .map(str::to_string), + ), + ( + GUARDRAIL_TRACE_HEADER, + config + .get("trace") + .and_then(Value::as_str) + .map(str::to_uppercase), + ), + ] + .into_iter() + .filter_map(|(name, value)| Some((name.to_string(), value?))) + .collect()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn rejection(error: Error) -> (u16, String) { + let Error::Http(error) = error else { + panic!("expected a 400, got {error:?}"); + }; + let (status, body) = error.upstream_response().unwrap(); + (status, body.to_string()) + } + + #[rstest] + #[case::enabled("enabled", "ENABLED")] + #[case::disabled("disabled", "DISABLED")] + #[case::enabled_full("enabled_full", "ENABLED_FULL")] + fn guardrail_config_becomes_invoke_headers(#[case] trace: &str, #[case] expected: &str) { + assert_eq!( + bedrock_invoke_guardrail_headers( + &json!({"guardrailIdentifier": "gid", "guardrailVersion": "2", "trace": trace}) + ) + .unwrap(), + vec![ + (GUARDRAIL_IDENTIFIER_HEADER.to_string(), "gid".to_string()), + (GUARDRAIL_VERSION_HEADER.to_string(), "2".to_string()), + (GUARDRAIL_TRACE_HEADER.to_string(), expected.to_string()), + ] + ); + } + + #[test] + fn a_guardrail_config_without_an_identifier_is_a_400() { + assert_eq!( + rejection(bedrock_invoke_guardrail_headers(&json!({"guardrailVersion": "2"})).unwrap_err()), + ( + 400, + "guardrailConfig={'guardrailVersion': '2'} is missing 'guardrailIdentifier'. Expected format: {'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}".to_string() + ) + ); + } + + #[rstest] + #[case::bad_trace(json!({"guardrailIdentifier": "gid", "trace": "loud"}))] + #[case::not_a_dict(json!("gid"))] + #[case::non_string_version(json!({"guardrailIdentifier": "gid", "guardrailVersion": 2}))] + fn an_invalid_guardrail_config_is_a_400(#[case] raw: Value) { + let (status, body) = rejection(bedrock_invoke_guardrail_headers(&raw).unwrap_err()); + assert_eq!(status, 400); + assert!( + body.starts_with(&format!( + "Invalid guardrailConfig={}. Expected format: ", + python_repr(&raw) + )), + "{body}" + ); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/mod.rs b/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/mod.rs new file mode 100644 index 00000000000..418ceff691e --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/mod.rs @@ -0,0 +1 @@ +pub mod base_invoke_transformation; diff --git a/litellm/rust/crates/llms/src/bedrock/chat/mod.rs b/litellm/rust/crates/llms/src/bedrock/chat/mod.rs index a46514aa697..0dd26c8cfc5 100644 --- a/litellm/rust/crates/llms/src/bedrock/chat/mod.rs +++ b/litellm/rust/crates/llms/src/bedrock/chat/mod.rs @@ -1,2 +1,3 @@ pub mod converse_transformation; pub mod invoke_handler; +pub mod invoke_transformations; diff --git a/litellm/rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs b/litellm/rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs index 52c93d097f5..e4deb3adb86 100644 --- a/litellm/rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs @@ -5,7 +5,7 @@ use std::sync::LazyLock; use litellm_auth::{AwsParams, CredentialPlacement, ParamKind, ParamSpec}; use litellm_secret_value::SecretValue; -use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region}; +use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders, resolve_aws_region}; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{header_value, with_header, without_headers}; use litellm_llms_types::{ @@ -24,7 +24,7 @@ use crate::{ base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, messages::{ - context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages}, + context::{MessagesShapeOptions, MessagesTransformContext}, transformation::BaseMessagesConfig, }, }, @@ -123,6 +123,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { region: platform_region(base.as_deref(), params, env)?, service: SERVICE, credentials: Box::new(AwsCredentialSource::from_params(¶ms.aws, env)), + caller_headers: CallerSigningHeaders::Botocore, }, }) } @@ -196,12 +197,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()), None => headers, }; - update_headers_with_safeguards_beta( - headers, - request, - BetaProvider::Anthropic, - UnparsedMessages::Skipped, - ) + update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic) } fn wire_body(&self, body: Value) -> Value { diff --git a/litellm/rust/crates/llms/src/bedrock/common_utils.rs b/litellm/rust/crates/llms/src/bedrock/common_utils.rs index 05f6871529a..295641c5b20 100644 --- a/litellm/rust/crates/llms/src/bedrock/common_utils.rs +++ b/litellm/rust/crates/llms/src/bedrock/common_utils.rs @@ -1,7 +1,10 @@ use litellm_auth_aws::is_bedrock_region; +use serde_json::{Map, Value, json}; use crate::{Error, ErrorDetail}; +pub mod structured_output; + const ROUTE_PREFIXES: [&str; 3] = ["bedrock/converse/", "bedrock/", "converse/"]; const ROUTING_PREFIXES: [&str; 8] = [ "bedrock/", @@ -27,7 +30,7 @@ pub fn bedrock_base_model(model: &str) -> &str { if unrouted.starts_with("nova/") { return "amazon.nova-custom"; } - let named = without_throughput_suffix(model_name_from_arn(without_routing_prefixes(model))); + let named = without_throughput_suffix(model_name_from_arn(strip_bedrock_routing_prefix(model))); if let Some((region, rest)) = named.split_once('.') && CROSS_REGION_INFERENCE_PREFIXES.contains(®ion) { @@ -43,7 +46,8 @@ pub fn is_bedrock_application_inference_profile_arn(model: &str) -> bool { model.contains(":application-inference-profile/") } -pub fn without_routing_prefixes(model: &str) -> &str { +/// Port of Python's `strip_bedrock_routing_prefix`. +pub fn strip_bedrock_routing_prefix(model: &str) -> &str { ROUTING_PREFIXES.into_iter().fold(model, |model, prefix| { model.strip_prefix(prefix).unwrap_or(model) }) @@ -83,6 +87,181 @@ fn is_digits(value: &str) -> bool { !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) } +const UNNAMED_TOOL_PREFIX: &str = "litellm_unnamed_tool_"; + +pub(crate) fn is_int(value: &Value) -> bool { + value.is_i64() || value.is_u64() +} + +fn effort_rank(effort: &str) -> Option { + match effort { + "low" => Some(0), + "medium" => Some(1), + "high" => Some(2), + "max" => Some(3), + "xhigh" => Some(4), + _ => None, + } +} + +pub(crate) fn clamp_effort(effort: &str, ceiling: Option<&str>) -> Option { + let ceiling = ceiling?; + (effort_rank(effort)? > effort_rank(ceiling)?).then(|| ceiling.to_string()) +} + +/// Port of Python's `normalize_bedrock_opus_output_config_effort`. +pub(crate) fn normalize_bedrock_opus_output_config_effort( + output_config: Option<&mut Value>, + ceiling: Option<&str>, +) { + let Some(Value::Object(output_config)) = output_config else { + return; + }; + if let Some(clamped) = output_config + .get("effort") + .and_then(Value::as_str) + .and_then(|effort| clamp_effort(effort, ceiling)) + { + output_config.insert("effort".into(), Value::String(clamped)); + } +} + +/// Port of Python's `strip_unsupported_bedrock_invoke_output_config_keys`. +pub(crate) fn strip_unsupported_bedrock_invoke_output_config_keys( + model: &str, + request_body: &mut Map, + accepts_output_config: bool, +) { + let Some(Value::Object(output_config)) = request_body.get("output_config") else { + return; + }; + if output_config.keys().all(|key| key == "format") || accepts_output_config { + return; + } + litellm_tracing::warn!( + "Bedrock Invoke: stripping unsupported `output_config` keys for model={model}: neither `supports_output_config` nor any `supports_*_reasoning_effort` flag is set in model_prices_and_context_window.json. Add the capability flag to the model JSON entry if this model accepts `output_config`." + ); + match output_config + .get("format") + .filter(|format| !format.is_null()) + .cloned() + { + Some(format) => { + request_body.insert("output_config".into(), json!({"format": format})); + } + None => { + request_body.shift_remove("output_config"); + } + } +} + +fn tools_mut(request_body: &mut Map) -> Option<&mut Vec> { + match request_body.get_mut("tools") { + Some(Value::Array(tools)) => Some(tools), + _ => None, + } +} + +/// Port of Python's `_normalize_custom_field_on_tool`. +fn normalize_custom_field_on_tool(tool: &mut Value) { + let Value::Object(tool) = tool else { + return; + }; + if let Some(Value::Object(custom)) = tool.shift_remove("custom") + && !tool.contains_key("defer_loading") + && let Some(deferred @ Value::Bool(_)) = custom.get("defer_loading") + { + tool.insert("defer_loading".into(), deferred.clone()); + } +} + +/// Port of Python's `normalize_custom_field_on_tools`. +pub(crate) fn normalize_custom_field_on_tools(request_body: &mut Map) { + if let Some(tools) = tools_mut(request_body) { + tools.iter_mut().for_each(normalize_custom_field_on_tool); + } +} + +/// Port of Python's `normalize_json_schema_custom_types_to_object`. +fn normalize_json_schema_custom_types_to_object(schema: &mut Value) { + let Value::Object(node) = schema else { + return; + }; + if node.get("type").and_then(Value::as_str) == Some("custom") { + node.insert("type".into(), Value::String("object".into())); + } + for key in ["items", "additionalProperties"] { + if let Some(child) = node.get_mut(key) { + normalize_json_schema_custom_types_to_object(child); + } + } + if let Some(Value::Object(properties)) = node.get_mut("properties") { + properties + .values_mut() + .for_each(normalize_json_schema_custom_types_to_object); + } + for combiner in ["allOf", "anyOf", "oneOf"] { + if let Some(Value::Array(branches)) = node.get_mut(combiner) { + branches + .iter_mut() + .for_each(normalize_json_schema_custom_types_to_object); + } + } +} + +/// Port of Python's `normalize_tool_input_schema_types_for_bedrock_invoke`. +pub(crate) fn normalize_tool_input_schema_types_for_bedrock_invoke( + request_body: &mut Map, +) { + let Some(tools) = tools_mut(request_body) else { + return; + }; + tools + .iter_mut() + .filter_map(|tool| tool.get_mut("input_schema")) + .filter(|input_schema| input_schema.is_object()) + .for_each(normalize_json_schema_custom_types_to_object); +} + +/// Port of Python's `ensure_bedrock_anthropic_messages_tool_names`. +pub(crate) fn ensure_bedrock_anthropic_messages_tool_names(request_body: &mut Map) { + let Some(tools) = tools_mut(request_body) else { + return; + }; + for (index, tool) in tools.iter_mut().enumerate() { + let Value::Object(tool) = tool else { + continue; + }; + let unnamed = match tool.get("name") { + None | Some(Value::Null) => true, + Some(Value::String(name)) => name.trim().is_empty(), + Some(_) => false, + }; + if unnamed { + tool.insert( + "name".into(), + Value::String(format!("{UNNAMED_TOOL_PREFIX}{index}")), + ); + } + } +} + +/// Port of Python's `tools_without_eager_input_streaming`. +pub(crate) fn tools_without_eager_input_streaming(request_body: &mut Map) { + let Some(tools) = tools_mut(request_body) else { + return; + }; + if !tools.iter().all(Value::is_object) { + return; + } + tools + .iter_mut() + .filter_map(Value::as_object_mut) + .for_each(|tool| { + tool.shift_remove("eager_input_streaming"); + }); +} + pub(crate) fn validated_region(region: String) -> Result { if !region.is_empty() && region @@ -101,8 +280,42 @@ pub(crate) fn validated_region(region: String) -> Result { #[cfg(test)] mod tests { use rstest::rstest; + use serde_json::{Value, json}; - use super::bedrock_base_model; + use super::*; + use crate::{ + anthropic::common_utils::supports_effort_param, + base_llm::messages::context::{ + BedrockCapabilities, MessagesModelCapabilities, SupportedEffortTiers, + }, + }; + + fn adaptive() -> MessagesModelCapabilities { + MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + low: true, + medium: true, + high: true, + max: true, + xhigh: true, + ..SupportedEffortTiers::default() + }, + ..MessagesModelCapabilities::default() + } + } + + fn with_bedrock( + capabilities: MessagesModelCapabilities, + bedrock: BedrockCapabilities, + ) -> MessagesModelCapabilities { + MessagesModelCapabilities { + bedrock, + ..capabilities + } + } #[rstest] #[case::cross_region("us.anthropic.claude-x-v1:0", "anthropic.claude-x-v1:0")] @@ -132,4 +345,66 @@ mod tests { fn base_model_matches_the_python_resolution(#[case] model: &str, #[case] expected: &str) { assert_eq!(bedrock_base_model(model), expected); } + + #[rstest] + #[case::hoists_bool_defer_loading( + json!([{"name": "Read", "input_schema": {"type": "object"}, "custom": {"defer_loading": true}}, {"name": "Write", "input_schema": {"type": "object"}}]), + json!([{"name": "Read", "input_schema": {"type": "object"}, "defer_loading": true}, {"name": "Write", "input_schema": {"type": "object"}}]), + )] + #[case::keeps_an_existing_defer_loading( + json!([{"name": "Read", "defer_loading": false, "custom": {"defer_loading": true}}]), + json!([{"name": "Read", "defer_loading": false}]), + )] + #[case::drops_a_non_bool_defer_loading( + json!([{"name": "Read", "custom": {"defer_loading": "yes", "other": 1}}]), + json!([{"name": "Read"}]), + )] + #[case::custom_schema_types_become_object( + json!([{"name": "t", "input_schema": {"type": "custom", "properties": {"a": {"type": "custom", "items": {"type": "custom"}}}, "anyOf": [{"type": "custom"}]}}]), + json!([{"name": "t", "input_schema": {"type": "object", "properties": {"a": {"type": "object", "items": {"type": "object"}}}, "anyOf": [{"type": "object"}]}}]), + )] + #[case::unnamed_tools_get_their_index( + json!([{"name": "first"}, {"input_schema": {"type": "object"}}, {"name": " "}, {"name": null}]), + json!([{"name": "first"}, {"input_schema": {"type": "object"}, "name": "litellm_unnamed_tool_1"}, {"name": "litellm_unnamed_tool_2"}, {"name": "litellm_unnamed_tool_3"}]), + )] + fn tools_are_normalized_for_bedrock(#[case] tools: Value, #[case] expected: Value) { + let mut request = json!({"tools": tools}).as_object().unwrap().clone(); + normalize_custom_field_on_tools(&mut request); + normalize_tool_input_schema_types_for_bedrock_invoke(&mut request); + ensure_bedrock_anthropic_messages_tool_names(&mut request); + assert_eq!(request["tools"], expected); + } + + #[rstest] + #[case::stripped_without_support( + MessagesModelCapabilities::default(), + json!({"effort": "high"}), + None, + )] + #[case::kept_with_output_config_support( + adaptive(), + json!({"effort": "high"}), + Some(json!({"effort": "high"})), + )] + #[case::bedrock_flag_keeps_it( + with_bedrock(MessagesModelCapabilities::default(), BedrockCapabilities { supports_output_config: true, ..BedrockCapabilities::default() }), + json!({"effort": "high"}), + Some(json!({"effort": "high"})), + )] + fn output_config_keys_follow_model_support( + #[case] capabilities: MessagesModelCapabilities, + #[case] output_config: Value, + #[case] expected: Option, + ) { + let mut request = json!({"output_config": output_config}) + .as_object() + .unwrap() + .clone(); + strip_unsupported_bedrock_invoke_output_config_keys( + "model", + &mut request, + capabilities.bedrock.supports_output_config || supports_effort_param(&capabilities), + ); + assert_eq!(request.get("output_config").cloned(), expected); + } } diff --git a/litellm/rust/crates/llms/src/bedrock/common_utils/structured_output.rs b/litellm/rust/crates/llms/src/bedrock/common_utils/structured_output.rs new file mode 100644 index 00000000000..9056daa6da2 --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/common_utils/structured_output.rs @@ -0,0 +1,904 @@ +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use litellm_python_compat::{json as python_json, repr}; +use serde_json::{Map, Value, json}; + +use crate::{Error, ErrorDetail, base_llm::messages::context::NativeOutputSupport}; + +use litellm_llms_types::recognized::python_truthy; + +use super::is_int; + +type Body = Map; + +const NATIVE_OUTPUT_MAX_PROPERTIES: usize = 64; +const NATIVE_OUTPUT_MAX_UNION_PARAMETERS: usize = 16; +const NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: usize = 24; +const NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: u128 = 2048; +const MAX_SCHEMA_NESTING: usize = 1024; +const NATIVE_OUTPUT_STRING_FORMATS: [&str; 10] = [ + "date-time", + "time", + "date", + "duration", + "email", + "hostname", + "uri", + "ipv4", + "ipv6", + "uuid", +]; +const SUBSCHEMA_KEYWORDS: [&str; 11] = [ + "additionalItems", + "additionalProperties", + "contains", + "else", + "if", + "items", + "not", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", +]; +const SUBSCHEMA_LIST_KEYWORDS: [&str; 5] = ["allOf", "anyOf", "items", "oneOf", "prefixItems"]; +const SUBSCHEMA_MAP_KEYWORDS: [&str; 5] = [ + "$defs", + "definitions", + "dependentSchemas", + "patternProperties", + "properties", +]; +const CONSTRAINT_LABELS: [(&str, &str); 25] = [ + ("minItems", "minimum number of items: {}"), + ("maxItems", "maximum number of items: {}"), + ("uniqueItems", "all array items must be unique"), + ("contains", "array must contain an item matching: {}"), + ("minContains", "minimum number of matching items: {}"), + ("maxContains", "maximum number of matching items: {}"), + ("prefixItems", "leading items must match, in order: {}"), + ("minimum", "minimum value: {}"), + ("maximum", "maximum value: {}"), + ("exclusiveMinimum", "exclusive minimum value: {}"), + ("exclusiveMaximum", "exclusive maximum value: {}"), + ("multipleOf", "must be a multiple of {}"), + ("minLength", "minimum length: {}"), + ("maxLength", "maximum length: {}"), + ("minProperties", "minimum number of properties: {}"), + ("maxProperties", "maximum number of properties: {}"), + ( + "patternProperties", + "properties whose names match each pattern must satisfy: {}", + ), + ("propertyNames", "property names must satisfy: {}"), + ("dependentRequired", "dependent required properties: {}"), + ("dependentSchemas", "dependent schemas: {}"), + ( + "unevaluatedProperties", + "unevaluated properties must satisfy: {}", + ), + ("if", "conditional (if): {}"), + ("then", "conditional (then): {}"), + ("else", "conditional (else): {}"), + ("not", "must not match: {}"), +]; + +type Schema = Map; + +fn subschemas(node: &Schema) -> impl Iterator { + node.iter().flat_map(|(key, value)| { + let key = key.as_str(); + let children: Vec<&Schema> = match value { + Value::Object(entries) if SUBSCHEMA_MAP_KEYWORDS.contains(&key) => { + entries.values().filter_map(Value::as_object).collect() + } + Value::Array(entries) if SUBSCHEMA_LIST_KEYWORDS.contains(&key) => { + entries.iter().filter_map(Value::as_object).collect() + } + Value::Object(child) if SUBSCHEMA_KEYWORDS.contains(&key) => vec![child], + _ => Vec::new(), + }; + children + }) +} + +fn schema_nodes(schema: &Schema) -> Vec<&Schema> { + let mut nodes = Vec::new(); + let mut frontier = vec![schema]; + let mut depth = 0; + while !frontier.is_empty() && depth < MAX_SCHEMA_NESTING { + let next: Vec<&Schema> = frontier.iter().flat_map(|node| subschemas(node)).collect(); + nodes.extend(frontier); + frontier = next; + depth += 1; + } + nodes +} + +fn schema_at<'a>(node: &'a Schema, key: &str) -> Option<&'a Schema> { + node.get(key).and_then(Value::as_object) +} + +fn list_at<'a>(node: &'a Schema, key: &str) -> &'a [Value] { + match node.get(key) { + Some(Value::Array(values)) => values, + _ => &[], + } +} + +fn schema_fragment_ref(reference: &str) -> String { + if reference.starts_with("/$defs/") || reference.starts_with("/definitions/") { + format!("#{reference}") + } else { + reference.to_string() + } +} + +fn refs_in(schema: &Schema) -> BTreeSet { + schema_nodes(schema) + .into_iter() + .filter_map(|node| node.get("$ref").and_then(Value::as_str)) + .map(schema_fragment_ref) + .collect() +} + +fn root_definitions(schema: &Schema) -> Vec<(String, &Schema)> { + ["$defs", "definitions"] + .into_iter() + .filter_map(|defs_key| schema_at(schema, defs_key).map(|defs| (defs_key, defs))) + .flat_map(|(defs_key, defs)| { + defs.iter().filter_map(move |(name, body)| { + body.as_object() + .map(|body| (format!("#/{defs_key}/{name}"), body)) + }) + }) + .collect() +} + +fn has_cycle(graph: &BTreeMap>) -> bool { + let mut unresolved: BTreeMap<&str, usize> = BTreeMap::new(); + let mut dependents: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + graph.iter().for_each(|(node, edges)| { + let known = edges.iter().filter(|edge| graph.contains_key(*edge)); + unresolved.insert(node, known.clone().count()); + known.for_each(|edge| dependents.entry(edge).or_default().push(node)); + }); + let mut ready: VecDeque<&str> = unresolved + .iter() + .filter(|(_, count)| **count == 0) + .map(|(node, _)| *node) + .collect(); + let mut resolved = 0; + while let Some(node) = ready.pop_front() { + resolved += 1; + dependents + .get(node) + .into_iter() + .flatten() + .for_each(|dependent| { + if let Some(count) = unresolved.get_mut(dependent) { + *count -= 1; + if *count == 0 { + ready.push_back(*dependent); + } + } + }); + } + resolved != graph.len() +} + +fn schema_has_unresolvable_ref(schema: &Schema) -> bool { + let definitions: BTreeMap = root_definitions(schema).into_iter().collect(); + if !refs_in(schema) + .iter() + .all(|reference| definitions.contains_key(reference)) + { + return true; + } + has_cycle( + &definitions + .iter() + .map(|(pointer, body)| (pointer.clone(), refs_in(body))) + .collect(), + ) +} + +fn is_object_schema(node: &Schema) -> bool { + match node.get("type") { + Some(Value::String(kind)) => kind == "object", + Some(Value::Array(kinds)) => kinds.iter().any(|kind| kind.as_str() == Some("object")), + _ => false, + } +} + +fn is_open_object(node: &Schema) -> bool { + let additional = node.get("additionalProperties"); + if additional == Some(&Value::Bool(true)) + || additional.is_some_and(Value::is_object) + || node.contains_key("patternProperties") + { + return true; + } + is_object_schema(node) + && !node.contains_key("properties") + && additional != Some(&Value::Bool(false)) +} + +fn uses_regex_lookaround(pattern: &str) -> bool { + ["(?=", "(?!", "(?<=", "(? bool { + match node.get("type") { + None => true, + Some(Value::String(kind)) => kind == "string", + Some(_) => list_at(node, "type") + .iter() + .any(|kind| kind.as_str() == Some("string")), + } +} + +fn optional_property_names(node: &Schema) -> BTreeSet<&str> { + let Some(properties) = schema_at(node, "properties") else { + return BTreeSet::new(); + }; + let required: BTreeSet<&str> = list_at(node, "required") + .iter() + .filter_map(Value::as_str) + .collect(); + properties + .keys() + .map(String::as_str) + .filter(|name| !required.contains(name)) + .collect() +} + +fn is_union(node: &Schema) -> bool { + node.get("anyOf").is_some_and(Value::is_array) + || node.get("oneOf").is_some_and(Value::is_array) + || list_at(node, "type").len() > 1 +} + +fn grammar_branches(node: &Schema) -> u128 { + let Some(properties) = schema_at(node, "properties") else { + return 0; + }; + let optional = optional_property_names(node).len(); + let required = (properties.len() - optional.min(properties.len())) as u128 + 1; + u32::try_from(optional) + .ok() + .and_then(|shift| 1u128.checked_shl(shift)) + .and_then(|branches| branches.checked_mul(required)) + .unwrap_or(u128::MAX) +} + +/// Port of Python's `schema_property_count`. +pub(crate) fn schema_property_count(schema: &Schema) -> usize { + schema_nodes(schema) + .into_iter() + .filter_map(|node| schema_at(node, "properties")) + .map(Map::len) + .sum() +} + +fn native_output_rejects(schema: &Schema) -> bool { + let nodes = schema_nodes(schema); + let string_formats_supported = nodes + .iter() + .filter(|node| may_be_string(node)) + .filter_map(|node| node.get("format").and_then(Value::as_str)) + .all(|format| NATIVE_OUTPUT_STRING_FORMATS.contains(&format)); + let union_properties = nodes + .iter() + .filter_map(|node| schema_at(node, "properties")) + .flat_map(|properties| properties.values().filter_map(Value::as_object)) + .filter(|property| is_union(property)) + .count(); + let optional_properties: usize = nodes + .iter() + .map(|node| optional_property_names(node).len()) + .sum(); + let grammar_branches = nodes + .iter() + .map(|node| grammar_branches(node)) + .fold(0u128, u128::saturating_add); + schema_has_unresolvable_ref(schema) + || nodes.iter().any(|node| is_open_object(node)) + || nodes.iter().any(|node| { + node.get("pattern") + .and_then(Value::as_str) + .is_some_and(uses_regex_lookaround) + }) + || !string_formats_supported + || union_properties > NATIVE_OUTPUT_MAX_UNION_PARAMETERS + || optional_properties > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS + || grammar_branches > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES +} + +fn enum_conflicts_with_declared_type(schema: &Schema) -> bool { + let Some(Value::Array(values)) = schema.get("enum") else { + return false; + }; + let declared = match schema.get("type") { + None | Some(Value::Null) => return false, + Some(Value::Array(_)) => return true, + Some(Value::String(kind)) => kind.as_str(), + Some(_) => return false, + }; + let check: fn(&Value) -> bool = match declared { + "null" => Value::is_null, + "boolean" => Value::is_boolean, + "integer" => is_int, + "number" => Value::is_number, + "string" => Value::is_string, + "array" => Value::is_array, + "object" => Value::is_object, + _ => return false, + }; + !values.iter().all(check) +} + +fn note_value(value: &Value) -> Result { + let python = python_json::from_json(value.clone()); + match value { + Value::Object(_) | Value::Array(_) => python_json::dumps(&python).map_err(|error| { + Error::InvalidRequest(ErrorDetail::failed( + "output schema note serialization", + error, + )) + }), + _ => Ok(repr::to_str(&python)), + } +} + +fn described_with_note(description: Option<&Value>, note: String) -> Result { + match description { + Some(Value::String(text)) if !text.is_empty() => { + Ok(Value::String(format!("{text} {note}"))) + } + Some(other) if python_truthy(other) => { + Err(Error::Declined(litellm_core_utils::decline::Decline::new( + "a non-string output schema description next to a dropped constraint", + ))) + } + _ => Ok(Value::String(note)), + } +} + +fn filtered_subschema(value: &Value) -> Result { + match value { + Value::Object(schema) => filter_output_schema(schema).map(Value::Object), + other => Ok(other.clone()), + } +} + +fn filtered_subschemas(value: &Value) -> Result { + match value { + Value::Object(entries) => entries + .iter() + .map(|(name, schema)| Ok((name.clone(), filtered_subschema(schema)?))) + .collect::, Error>>() + .map(Value::Object), + other => Ok(other.clone()), + } +} + +fn filtered_branches(branches: &[Value]) -> Result, Error> { + branches.iter().map(filtered_subschema).collect() +} + +fn extended_any_of(result: &Schema, branches: &[Value]) -> Result { + let filtered = filtered_branches(branches)?; + match result.get("anyOf") { + Some(Value::Array(existing)) => Ok(Value::Array( + existing.iter().cloned().chain(filtered).collect(), + )), + Some(_) => Err(Error::Declined(litellm_core_utils::decline::Decline::new( + "an output schema whose anyOf cannot be extended", + ))), + None => Ok(Value::Array(filtered)), + } +} + +pub(crate) fn filter_output_schema(schema: &Schema) -> Result { + let enforced_min_items = schema + .get("minItems") + .filter(|value| is_int(value)) + .and_then(Value::as_i64) + .is_some_and(|count| count == 0 || count == 1); + let unsupported = |field: &str| { + CONSTRAINT_LABELS.iter().any(|(label, _)| *label == field) + && !(field == "minItems" && enforced_min_items) + }; + let notes = CONSTRAINT_LABELS + .iter() + .filter(|(field, _)| unsupported(field)) + .filter_map(|(field, label)| schema.get(*field).map(|value| (label, value))) + .filter(|(_, value)| **value != Value::Bool(false)) + .map(|(label, value)| Ok(label.replacen("{}", ¬e_value(value)?, 1))) + .collect::, Error>>()?; + let mut result = Map::new(); + if !notes.is_empty() { + let note = format!("Note: {}.", notes.join(", ")); + result.insert( + "description".into(), + described_with_note(schema.get("description"), note)?, + ); + } + let drops_conflicting_type = enum_conflicts_with_declared_type(schema); + for (key, value) in schema { + let key = key.as_str(); + if unsupported(key) + || (key == "type" && drops_conflicting_type) + || (key == "description" && result.contains_key("description")) + { + continue; + } + let filtered = match (key, value) { + ("properties" | "$defs" | "definitions", _) => filtered_subschemas(value)?, + ("items", _) => filtered_subschema(value)?, + ("anyOf" | "oneOf", Value::Array(branches)) => { + let extended = extended_any_of(&result, branches)?; + result.insert("anyOf".into(), extended); + continue; + } + ("allOf", Value::Array(branches)) => Value::Array(filtered_branches(branches)?), + ("$ref", Value::String(reference)) => Value::String(schema_fragment_ref(reference)), + _ => value.clone(), + }; + result.insert(key.to_string(), filtered); + } + if is_object_schema(&result) { + result.insert("additionalProperties".into(), Value::Bool(false)); + } + Ok(result) +} + +/// Port of Python's `AnthropicConfig.native_output_schema`. +pub(crate) fn native_output_schema(schema: &Schema) -> Result, Error> { + if native_output_rejects(schema) { + return Ok(None); + } + filter_output_schema(schema).map(Some) +} + +/// Port of Python's `bedrock_native_output_format`, with `bedrock_native_output_schema` and +/// `bedrock_cleaned_output_schema` inlined. +pub(crate) fn bedrock_native_output_format( + output_format: &Schema, + native: NativeOutputSupport, +) -> Result, Error> { + let Some(Value::Object(schema)) = output_format.get("schema") else { + return Ok((native.enabled || native.opt_in).then(|| output_format.clone())); + }; + let cleaned = if !native.mapped && schema_property_count(schema) > NATIVE_OUTPUT_MAX_PROPERTIES + { + None + } else { + native_output_schema(schema)? + }; + let native_schema = if native.enabled { + Some(cleaned.unwrap_or_else(|| schema.clone())) + } else if native.opt_in { + cleaned + } else { + None + }; + Ok(native_schema.map(|native_schema| { + let mut format = output_format.clone(); + format.insert("schema".into(), Value::Object(native_schema)); + format + })) +} + +/// Port of Python's `pop_bedrock_invoke_output_config_format`. +fn pop_bedrock_invoke_output_config_format(body: &mut Body) -> Option { + let Some(Value::Object(output_config)) = body.get_mut("output_config") else { + return None; + }; + let format = output_config.shift_remove("format"); + if output_config.is_empty() { + body.shift_remove("output_config"); + } + match format { + Some(Value::Object(format)) => Some(format), + _ => None, + } +} + +/// Port of Python's `convert_bedrock_invoke_output_format_to_inline_schema`. +fn convert_bedrock_invoke_output_format_to_inline_schema( + output_format: &Schema, + body: &mut Body, +) -> Result<(), Error> { + let Some(schema) = output_format + .get("schema") + .filter(|schema| python_truthy(schema)) + else { + return Ok(()); + }; + let text = python_json::dumps(&python_json::from_json(schema.clone())).map_err(|error| { + Error::InvalidRequest(ErrorDetail::failed("output schema serialization", error)) + })?; + let schema_block = json!({"type": "text", "text": text}); + let Some(Value::Array(messages)) = body.get_mut("messages") else { + return Ok(()); + }; + let Some(last_user) = messages + .iter_mut() + .rev() + .find(|message| message.get("role").and_then(Value::as_str) == Some("user")) + .and_then(Value::as_object_mut) + else { + return Ok(()); + }; + let content = match last_user.get("content") { + None => Value::Array(vec![schema_block]), + Some(Value::String(text)) => json!([{"type": "text", "text": text}, schema_block]), + Some(Value::Array(blocks)) => { + Value::Array(blocks.iter().cloned().chain([schema_block]).collect()) + } + Some(_) => return Ok(()), + }; + last_user.insert("content".into(), content); + Ok(()) +} + +/// Port of Python's `apply_bedrock_invoke_structured_output`. +pub(crate) fn apply_bedrock_invoke_structured_output( + body: &mut Body, + model: &str, + native: NativeOutputSupport, +) -> Result<(), Error> { + let legacy = body.shift_remove("output_format"); + let output_config_format = pop_bedrock_invoke_output_config_format(body); + let schema_format = match legacy { + Some(Value::Object(legacy)) => Some(legacy), + _ => output_config_format, + }; + let Some(schema_format) = schema_format else { + return Ok(()); + }; + if let Some(native_format) = bedrock_native_output_format(&schema_format, native)? { + match body.get_mut("output_config") { + Some(Value::Object(output_config)) => { + output_config.insert("format".into(), Value::Object(native_format)); + } + _ => { + body.insert("output_config".into(), json!({"format": native_format})); + } + } + return Ok(()); + } + litellm_tracing::warn!( + "Bedrock Invoke: model={model} does not take native structured output for this schema per model_prices_and_context_window.json, so the JSON schema was inlined into the last user message and is NOT enforced by the model." + ); + convert_bedrock_invoke_output_format_to_inline_schema(&schema_format, body) +} + +/// Port of Python's `requests_output_config_format`. +fn requests_output_config_format(params: &Map) -> bool { + matches!(params.get("output_config"), Some(Value::Object(output_config)) if output_config.contains_key("format")) +} + +pub(crate) fn requested_output_format(params: &Map) -> Option<&Schema> { + match params.get("output_format") { + Some(Value::Object(legacy)) => Some(legacy), + _ => params + .get("output_config") + .and_then(Value::as_object) + .and_then(|output_config| output_config.get("format")) + .and_then(Value::as_object), + } +} + +/// Port of Python's `bedrock_native_output_retry_requested`. +pub(crate) fn bedrock_native_output_retry_requested( + params: &Map, + native: NativeOutputSupport, +) -> bool { + (params.get("output_format").is_some_and(Value::is_object) + || requests_output_config_format(params)) + && native.retries() +} + +pub(crate) fn sent_output_config_format(body: &Value) -> bool { + body.as_object().is_some_and(requests_output_config_format) +} + +pub(crate) const NATIVE_OUTPUT_UNSUPPORTED: &str = + "output_config.format: Extra inputs are not permitted"; + +const NATIVE_OUTPUT_REJECTIONS: [&str; 4] = [ + NATIVE_OUTPUT_UNSUPPORTED, + "compilation exceeded limits: Schema is too complex", + "The compiled grammar is too large", + "output_config.format.schema:", +]; + +/// Port of Python's `is_bedrock_native_output_rejection`. +pub(crate) fn is_bedrock_native_output_rejection(status: u16, error_text: &str) -> bool { + status == 400 + && NATIVE_OUTPUT_REJECTIONS + .iter() + .any(|rejection| error_text.contains(rejection)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn schema(value: Value) -> Schema { + value.as_object().unwrap().clone() + } + + fn legacy() -> NativeOutputSupport { + NativeOutputSupport { + enabled: true, + mapped: true, + ..NativeOutputSupport::default() + } + } + + fn bedrock_opt_in() -> NativeOutputSupport { + NativeOutputSupport { + opt_in: true, + ..NativeOutputSupport::default() + } + } + + fn tree() -> Value { + json!({ + "type": "object", + "properties": {"root": {"$ref": "#/$defs/Node"}}, + "$defs": {"Node": {"type": "object", "properties": {"children": {"type": "array", "items": {"$ref": "#/$defs/Node"}}}}}, + }) + } + + fn open_object() -> Value { + json!({ + "type": "object", + "properties": {"counts": {"type": "object", "additionalProperties": {"type": "integer"}}}, + }) + } + + fn many_properties(count: usize) -> Value { + let properties: Map = (0..count) + .map(|index| (format!("p{index}"), json!({"type": "string"}))) + .collect(); + let required: Vec = (0..count).map(|index| json!(format!("p{index}"))).collect(); + json!({"type": "object", "properties": properties, "required": required, "additionalProperties": false}) + } + + fn sent(native: NativeOutputSupport, raw: Value) -> Body { + let mut body = schema(json!({ + "messages": [{"role": "user", "content": "Hello"}], + "output_config": {"format": {"type": "json_schema", "schema": raw}}, + })); + apply_bedrock_invoke_structured_output(&mut body, "model", native).unwrap(); + body + } + + #[test] + fn native_output_drops_schema_keywords_bedrock_rejects() { + let body = sent( + bedrock_opt_in(), + json!({ + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + }, + "required": ["count", "ids"], + }), + ); + assert_eq!( + body["output_config"]["format"]["schema"], + json!({ + "type": "object", + "properties": { + "count": {"description": "Note: minimum value: 1.", "type": "integer"}, + "ids": {"description": "Note: all array items must be unique.", "type": "array", "items": {"type": "string"}}, + }, + "required": ["count", "ids"], + "additionalProperties": false, + }) + ); + } + + #[rstest] + #[case::recursive_on_a_bedrock_profile(bedrock_opt_in(), tree(), false)] + #[case::open_object_on_a_bedrock_profile(bedrock_opt_in(), open_object(), false)] + #[case::recursive_on_a_legacy_profile(legacy(), tree(), true)] + #[case::open_object_on_a_legacy_profile(legacy(), open_object(), true)] + fn schemas_native_output_cannot_hold( + #[case] native: NativeOutputSupport, + #[case] raw: Value, + #[case] forwarded: bool, + ) { + let body = sent(native, raw.clone()); + if forwarded { + assert_eq!(body["output_config"]["format"]["schema"], raw); + } else { + assert_eq!(body.get("output_config"), None); + let text = body["messages"][0]["content"][1]["text"].as_str().unwrap(); + assert_eq!(serde_json::from_str::(text).unwrap(), raw); + } + } + + #[test] + fn shared_defs_reach_native_output_once() { + let party = json!({"$ref": "#/$defs/Party"}); + let body = sent( + legacy(), + json!({ + "type": "object", + "properties": {"buyer": party, "seller": party}, + "required": ["buyer", "seller"], + "additionalProperties": false, + "$defs": {"Party": {"type": "object", "properties": {"email": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, "required": ["email"], "additionalProperties": false}}, + }), + ); + let sent_schema = &body["output_config"]["format"]["schema"]; + assert_eq!(sent_schema["properties"]["buyer"], party); + assert_eq!(sent_schema.to_string().matches("\"anyOf\"").count(), 1); + } + + #[rstest] + #[case::unmapped_over_the_limit(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, 65, false)] + #[case::unmapped_at_the_limit(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, 24, true)] + #[case::mapped_over_the_limit(NativeOutputSupport { opt_in: true, mapped: true, ..NativeOutputSupport::default() }, 65, true)] + fn property_count_rule_applies_to_unmapped_models( + #[case] native: NativeOutputSupport, + #[case] properties: usize, + #[case] native_sent: bool, + ) { + assert_eq!( + sent(native, many_properties(properties)).contains_key("output_config"), + native_sent + ); + } + + #[test] + fn legacy_output_format_wins() { + let mut body = schema(json!({ + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "output_format": {"type": "json_schema", "schema": {"type": "object", "properties": {"legacy": {"type": "string"}}}}, + "output_config": {"effort": "high", "format": {"type": "json_schema", "schema": {"type": "object", "properties": {"new": {"type": "string"}}}}}, + })); + apply_bedrock_invoke_structured_output(&mut body, "model", NativeOutputSupport::default()) + .unwrap(); + assert_eq!(body["output_config"], json!({"effort": "high"})); + assert_eq!( + body["messages"][0]["content"][1]["text"], + json!(r#"{"type": "object", "properties": {"legacy": {"type": "string"}}}"#) + ); + } + + #[rstest] + #[case::one_of_becomes_any_of( + json!({"oneOf": [{"type": "string"}, {"type": "integer", "maximum": 3}]}), + json!({"anyOf": [{"type": "string"}, {"description": "Note: maximum value: 3.", "type": "integer"}]}), + )] + #[case::small_min_items_is_kept( + json!({"type": "array", "minItems": 1, "maxItems": 4}), + json!({"description": "Note: maximum number of items: 4.", "type": "array", "minItems": 1}), + )] + #[case::false_constraints_add_no_note( + json!({"type": "array", "uniqueItems": false}), + json!({"type": "array"}), + )] + #[case::conflicting_enum_type_is_dropped( + json!({"type": "string", "enum": [1, 2]}), + json!({"enum": [1, 2]}), + )] + #[case::sub_schema_constraints_note_their_json( + json!({"type": "array", "contains": {"type": "string"}, "description": "Items"}), + json!({"description": "Items Note: array must contain an item matching: {\"type\": \"string\"}.", "type": "array"}), + )] + #[case::ref_becomes_a_fragment( + json!({"$ref": "/$defs/Node"}), + json!({"$ref": "#/$defs/Node"}), + )] + fn output_schema_filter_matches_the_anthropic_sdk(#[case] raw: Value, #[case] expected: Value) { + assert_eq!( + Value::Object(filter_output_schema(&schema(raw)).unwrap()), + expected + ); + } + + #[rstest] + #[case::lookaround(json!({"type": "string", "pattern": "^(?!x)"}), true)] + #[case::unsupported_format(json!({"type": "string", "format": "regex"}), true)] + #[case::supported_format(json!({"type": "string", "format": "email"}), false)] + #[case::unresolvable_ref(json!({"$ref": "#/$defs/Missing"}), true)] + #[case::too_many_optional(json!({"type": "object", "properties": (0..25).map(|i| (format!("p{i}"), json!({"type": "string"}))).collect::>(), "additionalProperties": false}), true)] + fn native_output_rejects_what_the_grammar_cannot_compile( + #[case] raw: Value, + #[case] rejected: bool, + ) { + assert_eq!(native_output_rejects(&schema(raw)), rejected); + } + + fn graph(edges: Vec<(&str, Vec<&str>)>) -> BTreeMap> { + edges + .into_iter() + .map(|(node, refs)| { + ( + node.to_string(), + refs.into_iter().map(str::to_string).collect(), + ) + }) + .collect() + } + + #[rstest] + #[case::none(vec![], false)] + #[case::chain(vec![("a", vec!["b"]), ("b", vec!["c"]), ("c", vec![])], false)] + #[case::diamond(vec![("a", vec!["b", "c"]), ("b", vec!["d"]), ("c", vec!["d"]), ("d", vec![])], false)] + #[case::unknown_edges_are_ignored(vec![("a", vec!["missing"])], false)] + #[case::self_reference(vec![("a", vec!["a"])], true)] + #[case::two_node_cycle(vec![("a", vec!["b"]), ("b", vec!["a"])], true)] + #[case::cycle_behind_a_resolved_node(vec![("a", vec![]), ("b", vec!["a", "c"]), ("c", vec!["b"])], true)] + fn has_cycle_finds_only_reference_loops( + #[case] edges: Vec<(&str, Vec<&str>)>, + #[case] expected: bool, + ) { + assert_eq!(has_cycle(&graph(edges)), expected); + } + + #[test] + fn has_cycle_stays_linear_on_a_long_reference_chain() { + let size = 50_000; + let chain: BTreeMap> = (0..size) + .map(|i| { + ( + format!("n{i}"), + (i + 1 < size) + .then(|| format!("n{}", i + 1)) + .into_iter() + .collect(), + ) + }) + .collect(); + let started = std::time::Instant::now(); + assert!(!has_cycle(&chain)); + assert!(started.elapsed() < std::time::Duration::from_secs(5)); + } + + #[rstest] + #[case::extra_inputs(400, "output_config.format: Extra inputs are not permitted", true)] + #[case::too_complex(400, "compilation exceeded limits: Schema is too complex", true)] + #[case::grammar_too_large(400, "The compiled grammar is too large", true)] + #[case::schema_error(400, "output_config.format.schema: bad", true)] + #[case::other_400(400, "messages: bad", false)] + #[case::not_a_400(500, "output_config.format: Extra inputs are not permitted", false)] + fn native_output_rejections_are_recognized( + #[case] status: u16, + #[case] text: &str, + #[case] expected: bool, + ) { + assert_eq!(is_bedrock_native_output_rejection(status, text), expected); + } + + #[rstest] + #[case::enabled(legacy(), true)] + #[case::opt_in(bedrock_opt_in(), true)] + #[case::replaces_unforceable_tool(NativeOutputSupport { replaces_unforceable_tool: true, ..NativeOutputSupport::default() }, true)] + #[case::unsupported(NativeOutputSupport::default(), false)] + fn native_output_retry_needs_native_support( + #[case] native: NativeOutputSupport, + #[case] expected: bool, + ) { + let params = schema(json!({"output_config": {"format": {"type": "json_schema"}}})); + assert_eq!( + bedrock_native_output_retry_requested(¶ms, native), + expected + ); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/messages/AGENTS.md b/litellm/rust/crates/llms/src/bedrock/messages/AGENTS.md index 48c222a1cdc..e82ac89380c 100644 --- a/litellm/rust/crates/llms/src/bedrock/messages/AGENTS.md +++ b/litellm/rust/crates/llms/src/bedrock/messages/AGENTS.md @@ -1,6 +1,7 @@ # rules -- Claude on Bedrock InvokeModel: reuses `anthropic/messages` shaping, moves the model into the URL, and decodes the AWS event stream +- Claude on Bedrock InvokeModel: reuses `anthropic/messages` shaping, moves the model into the URL, and decodes the AWS event stream through `chat/invoke_handler.rs` +- `invoke_transformations/` holds only what Python's `AmazonAnthropicClaudeMessagesConfig` holds, under its method names. Helpers Python keeps in `bedrock/common_utils.py` and `bedrock/request_metadata.py` live in the matching Rust modules - Invocation metrics in the response map onto Messages usage here # references diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs index fa037974342..b0cba0c9d3a 100644 --- a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs @@ -1,57 +1,63 @@ use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; -use litellm_core_utils::decline::Decline; use litellm_host::interceptors::RetriedRequest; -use std::convert::Infallible; use crate::{ anthropic::messages::handler::shape_anthropic_messages_request, - base_llm::messages::context::{MessagesShapeOptions, MessagesTransformContext}, + base_llm::messages::context::{ + MessagesShapeOptions, MessagesTransformContext, NativeOutputSupport, + }, }; -use futures_util::StreamExt; -use litellm_auth::AwsParams; use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; use litellm_auth_aws::{ - AwsCredentialSource, bedrock_model_id_and_region, + AwsCredentialSource, CallerSigningHeaders, bedrock_runtime_endpoint, constants::{ AWS_BEARER_TOKEN_BEDROCK, AWS_BEDROCK_RUNTIME_ENDPOINT, AWS_DEFAULT_REGION, AWS_REGION, - AWS_REGION_NAME, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE, + AWS_REGION_NAME, BEDROCK_SERVICE, }, resolve_bedrock_region, }; -use litellm_llms_types::formats::messages::{ - MessagesOptionalParams, MessagesRequest, - streaming::{MessagesStreamEvent, MessagesStreamUsage}, -}; -use litellm_llms_types::providers::bedrock::{ - INVOCATION_METRICS_KEY, INVOKE_PATH, INVOKE_STREAM_PATH, -}; +use litellm_llms_types::formats::messages::{MessagesOptionalParams, MessagesRequest}; +use litellm_llms_types::providers::bedrock::{INVOKE_PATH, INVOKE_STREAM_PATH}; use litellm_router_types::LitellmParams; -use serde_json::{Map, Value}; +use serde_json::Value; use crate::{ Error, base_llm::{ auth::AuthScheme, - base_model_iterator::{StreamError, StreamTransformer, transform_stream}, messages::{ - streaming::{ByteStream, EventStream, StreamDecoder}, + streaming::StreamDecoder, transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment}, }, }, - bedrock::chat::invoke_handler::{decode_invoke_anthropic_chunk, invoke_chunk_stream}, }; -const METRICS_USAGE_KEYS: [(&str, &str); 4] = [ - ("input_tokens", "inputTokenCount"), - ("output_tokens", "outputTokenCount"), - ("cache_read_input_tokens", "cacheReadInputTokenCount"), - ("cache_creation_input_tokens", "cacheWriteInputTokenCount"), -]; +use crate::bedrock::{ + chat::invoke_transformations::base_invoke_transformation::{ + GUARDRAIL_CONFIG_PARAM, bedrock_invoke_guardrail_headers, + }, + common_utils::{ + strip_bedrock_routing_prefix, + structured_output::{ + NATIVE_OUTPUT_UNSUPPORTED, bedrock_native_output_format, + bedrock_native_output_retry_requested, is_bedrock_native_output_rejection, + requested_output_format, sent_output_config_format, + }, + validated_region, + }, + request_metadata::{bedrock_request_metadata_headers, merge_bedrock_invoke_headers}, +}; + +use super::{request, stream}; const INVOKE_MODEL_PREFIX: &str = "invoke/"; +const MODEL_ID_PARAM: &str = "model_id"; + +const MODEL_SPEC_PREFIXES: [&str; 2] = ["nova-2/", "nova/"]; + const SECRET_NAMES: &[&str] = &[ AWS_BEARER_TOKEN_BEDROCK, AWS_BEDROCK_RUNTIME_ENDPOINT, @@ -76,16 +82,89 @@ fn bearer_token( .filter(|token| !token.is_empty()) } +/// Port of Python's `BaseAWSLLM.encode_model_id`. +pub fn encode_model_id(text: &str) -> String { + text.bytes() + .map(|byte| match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' => { + char::from(byte).to_string() + } + other => format!("%{other:02X}"), + }) + .collect() +} + +/// Port of Python's `BaseAWSLLM.get_bedrock_model_id` for Claude. +fn get_bedrock_model_id(model: &str, model_id: Option<&str>) -> String { + if let Some(model_id) = model_id { + return encode_model_id(model_id).replacen(INVOKE_MODEL_PREFIX, "", 1); + } + let stripped = strip_bedrock_routing_prefix(model); + if stripped.starts_with("arn:") { + return encode_model_id(stripped); + } + let model_id = stripped.replacen(INVOKE_MODEL_PREFIX, "", 1); + match MODEL_SPEC_PREFIXES + .iter() + .find(|spec| model_id.contains(**spec)) + { + Some(spec) => encode_model_id(&model_id.replace(spec, "")), + None => model_id, + } +} + +fn is_arn_partition_char(character: char) -> bool { + character.is_ascii_lowercase() || character.is_ascii_digit() || character == '-' +} + +/// Port of Python's `contains_bedrock_arn`. +fn contains_bedrock_arn(model: &str) -> bool { + model.match_indices("arn:aws").any(|(index, prefix)| { + let rest = &model[index + prefix.len()..]; + let after_partition = match rest.strip_prefix('-') { + Some(partition) => { + let trimmed = partition.trim_start_matches(is_arn_partition_char); + if trimmed.len() == partition.len() { + return false; + } + trimmed + } + None => rest, + }; + after_partition.starts_with(":bedrock") + }) +} + +/// Port of Python's `BaseAWSLLM._get_aws_region_from_model_arn`. +fn get_aws_region_from_model_arn(model: &str) -> Option { + contains_bedrock_arn(model) + .then(|| model.split(':').nth(3)) + .flatten() + .filter(|region| !region.is_empty() && region.chars().all(is_arn_partition_char)) + .map(str::to_string) +} + +/// Port of Python's `AmazonInvokeConfig.get_complete_url`. fn invoke_url( api_base: Option<&str>, model: &str, - aws: &AwsParams, + litellm_params: &LitellmParams, stream: bool, env_lookup: &dyn Fn(&str) -> Option, -) -> String { - let (model_id, model_region) = - bedrock_model_id_and_region(model.strip_prefix(INVOKE_MODEL_PREFIX).unwrap_or(model)); - let region = resolve_bedrock_region(model_region.as_deref(), aws, env_lookup); +) -> Result { + let aws = &litellm_params.aws; + let model_id = get_bedrock_model_id( + model, + litellm_params + .extra + .get(MODEL_ID_PARAM) + .and_then(Value::as_str), + ); + let region = resolve_bedrock_region( + get_aws_region_from_model_arn(model).as_deref(), + aws, + env_lookup, + ); let path = if stream { INVOKE_STREAM_PATH } else { @@ -97,11 +176,41 @@ fn invoke_url( .filter(|value| !value.is_empty()) .map(str::to_string) }; - let endpoint = configured(api_base) + let endpoint = match configured(api_base) .or_else(|| configured(aws.aws_bedrock_runtime_endpoint.as_deref())) .or_else(|| env_lookup(AWS_BEDROCK_RUNTIME_ENDPOINT)) - .unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion)); - format!("{}/model/{model_id}/{path}", endpoint.trim_end_matches('/')) + { + Some(endpoint) => endpoint, + None => bedrock_runtime_endpoint(&validated_region(region)?), + }; + Ok(format!( + "{}/model/{model_id}/{path}", + endpoint.trim_end_matches('/') + )) +} + +/// The header half of Python's `validate_anthropic_messages_environment`. +fn invoke_headers(headers: Headers, litellm_params: &LitellmParams) -> Result { + let guardrail = match litellm_params.extra.get(GUARDRAIL_CONFIG_PARAM) { + None | Some(Value::Null) => Vec::new(), + Some(raw) => bedrock_invoke_guardrail_headers(raw)?, + }; + Ok(merge_bedrock_invoke_headers( + headers, + guardrail, + bedrock_request_metadata_headers(&litellm_params.extra), + )) +} + +fn without_native_output(context: &MessagesTransformContext) -> MessagesTransformContext { + let mut fallback = *context; + fallback.thinking.capabilities.bedrock.native_output = NativeOutputSupport { + enabled: false, + opt_in: false, + replaces_unforceable_tool: false, + ..fallback.thinking.capabilities.bedrock.native_output + }; + fallback } impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { @@ -131,23 +240,72 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { stream: bool, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { - Ok(invoke_url( - api_base, - model, - &litellm_params.aws, - stream, - env_lookup, - )) + invoke_url(api_base, model, litellm_params, stream, env_lookup) } - fn transform_anthropic_messages_request( + /// Python's `transform_anthropic_messages_request`, which picks the betas from the caller's + /// header and the request and sends them in the body. + fn finalize_request( &self, - _request: MessagesRequest, - _context: &MessagesTransformContext, - ) -> Result { - Err(Error::Declined(Decline::new( - "Bedrock invoke messages request shaping", - ))) + request: MessagesRequest, + headers: Headers, + context: &MessagesTransformContext, + ) -> Result<(MessagesRequest, Headers), Error> { + request::transform_anthropic_messages_request(request, context, headers) + } + + /// Port of Python's `native_output_fallback_litellm_params`. + fn native_output_fallback( + &self, + request: &MessagesRequest, + context: &MessagesTransformContext, + ) -> Option { + let native = context.thinking.capabilities.bedrock.native_output; + let Ok(Value::Object(params)) = serde_json::to_value(&request.params) else { + return None; + }; + if !bedrock_native_output_retry_requested(¶ms, native) { + return None; + } + let output_format = requested_output_format(¶ms)?; + bedrock_native_output_format(output_format, native) + .ok() + .flatten() + .map(|_| without_native_output(context)) + } + + /// Port of Python's `should_retry_anthropic_messages_without_native_output`. + fn retries_without_native_output( + &self, + status: u16, + error_text: &str, + request_body: &Value, + model: &str, + ) -> bool { + let retries = sent_output_config_format(request_body) + && is_bedrock_native_output_rejection(status, error_text); + if retries { + let opt_out_advice = if error_text.contains(NATIVE_OUTPUT_UNSUPPORTED) { + " Set `supports_bedrock_native_structured_output: false` in this deployment's model_info to skip the rejected call." + } else { + "" + }; + litellm_tracing::warn!( + "Bedrock rejected native structured output for model={model}, so litellm sent the request again without it.{opt_out_advice}" + ); + } + retries + } + + fn wire_body(&self, body: Value) -> Value { + match body { + Value::Object(mut fields) => { + fields.shift_remove("model"); + fields.shift_remove("stream"); + Value::Object(fields) + } + other => other, + } } fn secret_names(&self) -> &'static [&'static str] { @@ -165,6 +323,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { + let headers = invoke_headers(headers, litellm_params)?; if let Some(token) = bearer_token(api_key, env_lookup) { return Ok(ValidatedEnvironment { headers, @@ -174,15 +333,18 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { }, }); } - let (_, model_region) = - bedrock_model_id_and_region(model.strip_prefix(INVOKE_MODEL_PREFIX).unwrap_or(model)); let params = &litellm_params.aws; Ok(ValidatedEnvironment { headers, auth: AuthScheme::AwsSigV4 { - region: resolve_bedrock_region(model_region.as_deref(), params, env_lookup), + region: resolve_bedrock_region( + get_aws_region_from_model_arn(model).as_deref(), + params, + env_lookup, + ), service: BEDROCK_SERVICE, credentials: Box::new(AwsCredentialSource::from_params(params, env_lookup)), + caller_headers: CallerSigningHeaders::Botocore, }, }) } @@ -191,343 +353,20 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { &[("content-type", "application/json")] } + /// Port of Python's `get_async_streaming_response_iterator`. fn stream_decoder(&self) -> Option { - Some(bedrock_anthropic_messages_event_stream) - } -} - -fn with_invocation_usage(chunk: Value) -> Value { - match chunk { - Value::Object(fields) => Value::Object(with_metrics_usage(fields)), - other => other, - } -} - -fn with_metrics_usage(mut fields: Map) -> Map { - let Some(Value::Object(metrics)) = fields.remove(INVOCATION_METRICS_KEY) else { - return fields; - }; - if metrics.is_empty() { - return fields; - } - let preserved = match fields.remove("usage") { - Some(Value::Object(usage)) => usage, - _ => Map::new(), - }; - let usage: Map = METRICS_USAGE_KEYS - .iter() - .filter_map(|(anthropic, metric)| { - Some((anthropic.to_string(), metrics.get(*metric)?.clone())) - }) - .chain(preserved) - .collect(); - fields.insert("usage".to_string(), Value::Object(usage)); - fields -} - -pub fn bedrock_anthropic_messages_event_stream(bytes: ByteStream) -> EventStream { - let events = invoke_chunk_stream(bytes) - .map(|chunk| decode_invoke_anthropic_chunk(with_invocation_usage(chunk?))); - Box::pin( - transform_stream(events, MessageStopUsagePromoter::default()) - .map(|item| item.map_err(StreamError::into_decode)), - ) -} - -#[derive(Default)] -pub struct MessageStopUsagePromoter { - pending_delta: Option, - start_usage: Option, -} - -fn promoted_usage( - delta: Option, - stop: Option<&MessagesStreamUsage>, - start: Option<&MessagesStreamUsage>, -) -> Option { - let delta = delta.unwrap_or_default(); - let merged = MessagesStreamUsage { - input_tokens: stop - .and_then(|stop| stop.input_tokens) - .or(delta.input_tokens), - cache_creation_input_tokens: stop - .and_then(|stop| stop.cache_creation_input_tokens) - .or(delta.cache_creation_input_tokens) - .or_else(|| start.and_then(|start| start.cache_creation_input_tokens)), - cache_read_input_tokens: stop - .and_then(|stop| stop.cache_read_input_tokens) - .or(delta.cache_read_input_tokens) - .or_else(|| start.and_then(|start| start.cache_read_input_tokens)), - extra: delta - .extra - .into_iter() - .chain( - start - .and_then(|start| start.extra.get_key_value("cache_creation")) - .map(|(key, value)| (key.clone(), value.clone())), - ) - .fold(Map::new(), |mut extra, (key, value)| { - extra.entry(key).or_insert(value); - extra - }), - ..delta - }; - (merged != MessagesStreamUsage::default()).then_some(merged) -} - -fn promoted( - event: MessagesStreamEvent, - stop: Option<&MessagesStreamUsage>, - start: Option<&MessagesStreamUsage>, -) -> MessagesStreamEvent { - match event { - MessagesStreamEvent::MessageDelta { - delta, - usage, - context_management, - } => MessagesStreamEvent::MessageDelta { - delta, - usage: promoted_usage(usage, stop, start), - context_management, - }, - other => other, - } -} - -impl StreamTransformer for MessageStopUsagePromoter { - type Input = MessagesStreamEvent; - type Output = MessagesStreamEvent; - type Error = Infallible; - - fn transform( - &mut self, - input: MessagesStreamEvent, - ) -> Result, Infallible> { - let pending = self.pending_delta.take(); - match input { - MessagesStreamEvent::MessageDelta { .. } => { - self.pending_delta = Some(input); - Ok(pending.into_iter().collect()) - } - MessagesStreamEvent::MessageStop { usage } => Ok(pending - .map(|delta| promoted(delta, usage.as_ref(), self.start_usage.as_ref())) - .into_iter() - .chain([MessagesStreamEvent::MessageStop { usage }]) - .collect()), - MessagesStreamEvent::MessageStart { message } => { - self.start_usage = Some(message.usage.clone()); - Ok(pending - .into_iter() - .chain([MessagesStreamEvent::MessageStart { message }]) - .collect()) - } - other => Ok(pending.into_iter().chain([other]).collect()), - } - } - - fn finish(&mut self) -> Result, Infallible> { - Ok(self - .pending_delta - .take() - .map(|delta| promoted(delta, None, self.start_usage.as_ref())) - .into_iter() - .collect()) + Some(stream::get_async_streaming_response_iterator) } } #[cfg(test)] mod tests { - use aws_smithy_eventstream::frame::write_message_to; - use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; - use base64::{Engine, engine::general_purpose::STANDARD}; - use bytes::Bytes; - use futures_util::TryStreamExt; use rstest::rstest; - use serde_json::json; + use litellm_auth::AwsParams; use litellm_auth_aws::constants::DEFAULT_BEDROCK_REGION; use super::*; - use crate::base_llm::messages::streaming; - - fn event(value: Value) -> MessagesStreamEvent { - serde_json::from_value(value).unwrap() - } - - fn message_start(usage: Value) -> MessagesStreamEvent { - event(json!({ - "type": "message_start", - "message": { - "id": "msg_1", "type": "message", "role": "assistant", "model": "m", - "content": [], "stop_reason": null, "stop_sequence": null, "usage": usage - } - })) - } - - fn message_delta(usage: Value) -> MessagesStreamEvent { - event(json!({ - "type": "message_delta", - "delta": {"stop_reason": "end_turn"}, - "usage": usage - })) - } - - fn message_stop(usage: Option) -> MessagesStreamEvent { - match usage { - Some(usage) => event(json!({"type": "message_stop", "usage": usage})), - None => event(json!({"type": "message_stop"})), - } - } - - fn promote(events: Vec) -> Vec { - let mut promoter = MessageStopUsagePromoter::default(); - let mut output: Vec<_> = events - .into_iter() - .flat_map(|event| promoter.transform(event).unwrap()) - .collect(); - output.extend(promoter.finish().unwrap()); - output - } - - #[rstest] - #[case::cache_fields_on_message_stop( - json!({"input_tokens": 10, "output_tokens": 0}), - json!({"output_tokens": 5}), - Some(json!({"input_tokens": 3, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20})), - json!({"input_tokens": 3, "output_tokens": 5, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20}), - )] - #[case::cache_only_on_message_start( - json!({"input_tokens": 10, "output_tokens": 0, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}), - json!({"output_tokens": 5}), - Some(json!({"input_tokens": 10})), - json!({"input_tokens": 10, "output_tokens": 5, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}), - )] - #[case::message_stop_wins_over_message_start( - json!({"input_tokens": 10, "cache_read_input_tokens": 80}), - json!({"output_tokens": 5}), - Some(json!({"cache_read_input_tokens": 100})), - json!({"output_tokens": 5, "cache_read_input_tokens": 100}), - )] - #[case::delta_cache_fields_are_kept( - json!({"input_tokens": 10, "cache_read_input_tokens": 80}), - json!({"output_tokens": 5, "cache_read_input_tokens": 7}), - None, - json!({"output_tokens": 5, "cache_read_input_tokens": 7}), - )] - fn message_delta_usage_is_completed_from_stop_then_start( - #[case] start: Value, - #[case] delta: Value, - #[case] stop: Option, - #[case] expected: Value, - ) { - let output = promote(vec![ - message_start(start), - message_delta(delta), - message_stop(stop.clone()), - ]); - - assert_eq!(output.len(), 3); - assert_eq!(output[1], message_delta(expected)); - assert_eq!(output[2], message_stop(stop)); - } - - #[test] - fn a_delta_is_flushed_with_start_usage_when_the_stream_ends_without_a_stop() { - let output = promote(vec![ - message_start(json!({"input_tokens": 10, "cache_read_input_tokens": 80})), - message_delta(json!({"output_tokens": 5})), - ]); - - assert_eq!( - output[1], - message_delta(json!({"output_tokens": 5, "cache_read_input_tokens": 80})) - ); - } - - #[test] - fn events_around_the_delta_keep_their_order() { - let ping = event(json!({"type": "ping"})); - let output = promote(vec![ - message_delta(json!({"output_tokens": 5})), - ping.clone(), - message_stop(None), - ]); - - assert_eq!( - output, - vec![ - message_delta(json!({"output_tokens": 5})), - ping, - message_stop(None) - ] - ); - } - - #[rstest] - #[case::metrics_fill_missing_usage( - json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "outputTokenCount": 9}}), - json!({"type": "message_stop", "usage": {"input_tokens": 3, "output_tokens": 9}}), - )] - #[case::the_chunks_own_usage_wins( - json!({"type": "message_stop", "usage": {"input_tokens": 1}, "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}}), - json!({"type": "message_stop", "usage": {"cache_read_input_tokens": 40, "input_tokens": 1}}), - )] - #[case::no_metrics_leaves_the_chunk( - json!({"type": "message_stop"}), - json!({"type": "message_stop"}), - )] - #[case::empty_metrics_are_dropped( - json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {}}), - json!({"type": "message_stop"}), - )] - fn invocation_metrics_become_anthropic_usage(#[case] chunk: Value, #[case] expected: Value) { - assert_eq!(with_invocation_usage(chunk), expected); - } - - fn aws_frame(chunk: &Value) -> Vec { - let payload = json!({"bytes": STANDARD.encode(chunk.to_string())}); - let message = Message::new(Bytes::from(serde_json::to_vec(&payload).unwrap())).add_header( - Header::new(":event-type", HeaderValue::String("chunk".into())), - ); - let mut wire = Vec::new(); - write_message_to(&message, &mut wire).unwrap(); - wire - } - - #[tokio::test] - async fn bedrock_stream_yields_the_sse_an_anthropic_client_reads() { - let chunks = [ - json!({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}}), - json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}}), - ]; - let wire: Vec = chunks.iter().flat_map(aws_frame).collect(); - let bytes: ByteStream = futures_util::stream::iter( - wire.chunks(7) - .map(|chunk| Ok(Bytes::copy_from_slice(chunk))) - .collect::>(), - ) - .boxed(); - - let sse = bedrock_anthropic_messages_event_stream(bytes) - .map_ok(|event| streaming::encode(&event).unwrap()) - .try_collect::>() - .await - .unwrap() - .concat(); - - let expected: Vec = [ - message_delta( - json!({"output_tokens": 5, "cache_read_input_tokens": 40, "input_tokens": 3}), - ), - message_stop(Some( - json!({"input_tokens": 3, "cache_read_input_tokens": 40}), - )), - ] - .iter() - .flat_map(|event| streaming::encode(event).unwrap()) - .collect(); - assert_eq!(sse, expected); - } fn in_region(region: &str) -> LitellmParams { LitellmParams { @@ -619,6 +458,37 @@ mod tests { ); } + #[rstest] + #[case::host_injection("evil.example.com/x?")] + #[case::uppercase("US-WEST-2")] + #[case::port("us-west-2:8443")] + fn a_region_outside_the_aws_alphabet_never_builds_a_url(#[case] region: &str) { + let url = AmazonAnthropicClaudeMessagesConfig.get_complete_url( + None, + "anthropic.claude-3", + &in_region(region), + false, + &|_| None, + ); + assert!(matches!(url, Err(Error::InvalidRequest(_))), "{url:?}"); + } + + #[test] + fn a_configured_endpoint_does_not_read_the_region() { + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .get_complete_url( + Some("https://base.test"), + "anthropic.claude-3", + &in_region("evil.example.com/x?"), + false, + &|_| None, + ) + .unwrap(), + "https://base.test/model/anthropic.claude-3/invoke" + ); + } + #[test] fn sigv4_scope_and_credentials_come_from_the_litellm_params() { let litellm_params = LitellmParams { @@ -702,4 +572,276 @@ mod tests { (other, _) => panic!("unexpected auth {other:?}"), } } + + const ARN: &str = "arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0"; + const ENCODED_ARN: &str = "arn%3Aaws%3Abedrock%3Aus-east-1%3A086734376398%3Ainference-profile%2Fglobal.anthropic.claude-sonnet-4-5-20250929-v1%3A0"; + + #[rstest] + #[case::bedrock_prefixed_arn( + "bedrock/arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0", + None, + ENCODED_ARN + )] + #[case::compound_prefixed_arn( + "bedrock/invoke/arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0", + None, + ENCODED_ARN + )] + #[case::bare_arn(ARN, None, ENCODED_ARN)] + #[case::regular_id( + "anthropic.claude-3-sonnet-20240229-v1:0", + None, + "anthropic.claude-3-sonnet-20240229-v1:0" + )] + #[case::invoke_prefix( + "invoke/anthropic.claude-3-sonnet-20240229-v1:0", + None, + "anthropic.claude-3-sonnet-20240229-v1:0" + )] + #[case::model_id_override( + "anthropic.claude-3", + Some("arn:aws:bedrock:us-east-1:1:application-inference-profile/abc"), + "arn%3Aaws%3Abedrock%3Aus-east-1%3A1%3Aapplication-inference-profile%2Fabc" + )] + fn invoke_model_ids_match_python( + #[case] model: &str, + #[case] model_id: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!(get_bedrock_model_id(model, model_id), expected); + } + + #[rstest] + #[case::region_segment_stays_in_the_path( + "bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", + "us-west-2" + )] + #[case::region_segment_does_not_pick_the_region( + "invoke/eu-central-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/eu-central-1/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", + "us-west-2" + )] + #[case::params_region_hosts_a_region_segment( + "invoke/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + Some("us-east-1"), + "https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke", + "us-east-1" + )] + #[case::arn_after_a_region_segment_picks_the_region( + "invoke/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + None, + "https://bedrock-runtime.eu-west-1.amazonaws.com/model/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke", + "eu-west-1" + )] + #[case::arn_after_any_segment_picks_the_region( + "invoke/foo/arn:aws:bedrock:eu-west-1:1:x", + None, + "https://bedrock-runtime.eu-west-1.amazonaws.com/model/foo/arn:aws:bedrock:eu-west-1:1:x/invoke", + "eu-west-1" + )] + #[case::partitioned_arn( + "invoke/arn:aws-us-gov:bedrock:us-gov-west-1:1:inference-profile/x", + None, + "https://bedrock-runtime.us-gov-west-1.amazonaws.com/model/arn%3Aaws-us-gov%3Abedrock%3Aus-gov-west-1%3A1%3Ainference-profile%2Fx/invoke", + "us-gov-west-1" + )] + #[case::empty_partition_is_not_an_arn( + "invoke/arn:aws-:bedrock:eu-west-1:1:x", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws-%3Abedrock%3Aeu-west-1%3A1%3Ax/invoke", + "us-west-2" + )] + #[case::uppercase_arn_region_is_ignored( + "invoke/arn:aws:bedrock:EU:1:x", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3AEU%3A1%3Ax/invoke", + "us-west-2" + )] + #[case::nova_spec_is_removed_and_the_id_encoded( + "invoke/us-west-2/nova/anthropic.claude-3", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2%2Fanthropic.claude-3/invoke", + "us-west-2" + )] + #[case::one_inner_invoke_segment_is_removed( + "invoke/us-west-2/invoke/anthropic.claude-3", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2/anthropic.claude-3/invoke", + "us-west-2" + )] + fn url_and_signing_region_match_python_for_every_model_shape( + #[case] model: &str, + #[case] region_param: Option<&str>, + #[case] expected_url: &str, + #[case] expected_region: &str, + ) { + let litellm_params = region_param.map_or_else(LitellmParams::default, in_region); + let url = AmazonAnthropicClaudeMessagesConfig + .get_complete_url(None, model, &litellm_params, false, &|_| None) + .unwrap(); + let validated = AmazonAnthropicClaudeMessagesConfig + .validate_environment( + Vec::new(), + None, + model, + &MessagesOptionalParams::default(), + &litellm_params, + &|_| None, + ) + .unwrap(); + let AuthScheme::AwsSigV4 { + region, + caller_headers, + .. + } = validated.auth + else { + panic!("expected SigV4, got {:?}", validated.auth); + }; + assert_eq!( + (url.as_str(), region.as_str(), caller_headers), + ( + expected_url, + expected_region, + CallerSigningHeaders::Botocore + ) + ); + } + + #[test] + fn an_arn_url_is_percent_encoded_in_the_arn_region() { + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .get_complete_url( + None, + &format!("bedrock/{ARN}"), + &LitellmParams::default(), + true, + &|_| None + ) + .unwrap(), + format!( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/{ENCODED_ARN}/invoke-with-response-stream" + ) + ); + } + + #[test] + fn the_model_id_param_overrides_the_url_model() { + let litellm_params: LitellmParams = serde_json::from_value(serde_json::json!({ + "model": "bedrock/anthropic.claude-3", + "aws_region_name": "us-west-2", + "model_id": "my/profile", + })) + .unwrap(); + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .get_complete_url(None, "anthropic.claude-3", &litellm_params, false, &|_| { + None + }) + .unwrap(), + "https://bedrock-runtime.us-west-2.amazonaws.com/model/my%2Fprofile/invoke" + ); + } + + #[test] + fn guardrail_and_metadata_headers_are_added_for_both_auth_schemes() { + let litellm_params: LitellmParams = serde_json::from_value(serde_json::json!({ + "model": "bedrock/anthropic.claude-3", + "guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "2", "trace": "enabled"}, + })) + .unwrap(); + let validated = AmazonAnthropicClaudeMessagesConfig + .validate_environment( + Vec::new(), + Some("token"), + "anthropic.claude-3", + &MessagesOptionalParams::default(), + &litellm_params, + &|_| None, + ) + .unwrap(); + assert_eq!( + validated.headers, + vec![ + ( + "X-Amzn-Bedrock-GuardrailIdentifier".to_string(), + "gid".to_string() + ), + ( + "X-Amzn-Bedrock-GuardrailVersion".to_string(), + "2".to_string() + ), + ("X-Amzn-Bedrock-Trace".to_string(), "ENABLED".to_string()), + ] + ); + } + + fn native_context(native_output: NativeOutputSupport) -> MessagesTransformContext { + let mut context = MessagesTransformContext::default(); + context.thinking.capabilities.bedrock.native_output = native_output; + context + } + + fn structured_request() -> MessagesRequest { + serde_json::from_value(serde_json::json!({ + "model": "anthropic.claude-sonnet-5", + "max_tokens": 10, + "messages": [{"role": "user", "content": "hi"}], + "output_config": {"format": {"type": "json_schema", "schema": {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}}}, + })) + .unwrap() + } + + #[rstest] + #[case::native_enabled(NativeOutputSupport { enabled: true, mapped: true, ..NativeOutputSupport::default() }, true)] + #[case::native_opt_in(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, true)] + #[case::inline_only(NativeOutputSupport::default(), false)] + fn a_native_output_request_carries_an_inline_fallback( + #[case] native_output: NativeOutputSupport, + #[case] expected: bool, + ) { + let fallback = AmazonAnthropicClaudeMessagesConfig + .native_output_fallback(&structured_request(), &native_context(native_output)); + assert_eq!(fallback.is_some(), expected); + if let Some(fallback) = fallback { + let (rebuilt, _) = AmazonAnthropicClaudeMessagesConfig + .finalize_request(structured_request(), Vec::new(), &fallback) + .unwrap(); + let body = serde_json::to_value(rebuilt).unwrap(); + assert_eq!(body.get("output_config"), None); + assert_eq!( + body["messages"][0]["content"][1]["type"], + serde_json::json!("text") + ); + } + } + + #[rstest] + #[case::rejected_native_output(400, "output_config.format: Extra inputs are not permitted", serde_json::json!({"output_config": {"format": {}}}), true)] + #[case::inline_body_is_not_retried(400, "output_config.format: Extra inputs are not permitted", serde_json::json!({"messages": []}), false)] + #[case::other_error(429, "Too many requests", serde_json::json!({"output_config": {"format": {}}}), false)] + fn native_output_rejections_retry_without_it( + #[case] status: u16, + #[case] text: &str, + #[case] body: Value, + #[case] expected: bool, + ) { + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .retries_without_native_output(status, text, &body, "m"), + expected + ); + } + + #[test] + fn the_wire_body_drops_model_and_stream() { + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .wire_body(serde_json::json!({"model": "m", "stream": true, "max_tokens": 1})), + serde_json::json!({"max_tokens": 1}) + ); + } } diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/betas.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/betas.rs new file mode 100644 index 00000000000..8014c99d673 --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/betas.rs @@ -0,0 +1,702 @@ +use std::collections::BTreeSet; + +use litellm_llms_types::{ + providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet}, + recognized::python_truthy, +}; +use serde_json::Value; + +use crate::{ + Error, ErrorDetail, + anthropic::beta_headers::BetaPolicy, + base_llm::{auth::Headers, messages::context::MessagesModelCapabilities}, +}; + +use super::request::Body; + +const PER_TURN_CONTROL: &str = "per-turn-control-2026-07-01"; +const DANGEROUS_TOOL_USE: &str = "dangerous-tool-use-2026-09-03"; +const TOOL_SEARCH_TOOL: &str = "tool-search-tool-2025-10-19"; +const ADVANCED_TOOL_USE: &str = "advanced-tool-use-2025-11-20"; +const TOOL_EXAMPLES: &str = "tool-examples-2025-10-29"; +const COMPACT_2026_09_04: &str = "compact-2026-09-04"; +const THINKING_BINDING_CONTROLS: &str = "thinking-binding-controls-2026-08-01"; +const THINKING_DISPLAY_UPDATES: &str = "thinking-display-updates-2026-08-18"; +const MID_CONVERSATION_OUTPUT_CONFIG: &str = "mid-conversation-output-config-2026-07-01"; +const MID_CONVERSATION_TOOL_CHANGES: &str = "mid-conversation-tool-changes-2026-07-01"; +const PROGRAMMATIC_CALLER: &str = "code_execution_20250825"; +const TOOL_SEARCH_TYPES: [&str; 2] = [ + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", +]; +const CONTEXT_MANAGEMENT_EDIT_BETAS: [(&str, &str); 2] = [ + ("compact_20260112", "compact-2026-01-12"), + ("clear_tool_uses_20250919", "context-management-2025-06-27"), +]; +const COMPUTER_TOOL_BETAS: [(&str, &str); 3] = [ + ("computer_20251124", "computer-use-2025-11-24"), + ("computer_20250124", "computer-use-2025-01-24"), + ("computer_20241022", "computer-use-2024-10-22"), +]; +const OPUS_4_5_EFFORT_PATTERNS: [&str; 2] = ["opus-4-5", "opus_4_5"]; + +fn on_bedrock(beta: &str) -> Option { + beta.parse::() + .unwrap_or_else(|never| match never {}) + .on(BetaProvider::Bedrock) + .map(|beta| beta.as_str().to_string()) +} + +/// Port of Python's `model_specific_anthropic_betas`. +fn model_specific_anthropic_betas( + capabilities: &MessagesModelCapabilities, +) -> BTreeSet<&'static str> { + if capabilities.bedrock.supports_per_turn_control { + BTreeSet::from([PER_TURN_CONTROL]) + } else { + BTreeSet::new() + } +} + +fn header_value<'a>(headers: &'a Headers, name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) +} + +fn user_betas(headers: &Headers) -> BTreeSet { + let Some(header) = header_value(headers, BETA_HEADER).filter(|value| !value.is_empty()) else { + return BTreeSet::new(); + }; + if header.starts_with('[') + && header.ends_with(']') + && let Ok(Value::Array(values)) = serde_json::from_str::(header) + { + return values + .iter() + .map(|value| match value { + Value::String(text) => text.trim().to_string(), + other => other.to_string(), + }) + .filter(|value| !value.is_empty()) + .collect(); + } + header + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .collect() +} + +fn tools(value: Option<&Value>) -> &[Value] { + match value { + Some(Value::Array(tools)) => tools, + _ => &[], + } +} + +fn tool_type(tool: &Value) -> Option<&str> { + tool.get("type").and_then(Value::as_str) +} + +fn lists_programmatic_caller(value: Option<&Value>) -> bool { + matches!(value, Some(Value::Array(callers)) if callers.iter().any(|caller| caller.as_str() == Some(PROGRAMMATIC_CALLER))) +} + +fn programmatic_tool_calling_used(tools: &[Value]) -> bool { + tools.iter().any(|tool| { + lists_programmatic_caller(tool.get("allowed_callers")) + || lists_programmatic_caller( + tool.get("function") + .and_then(|function| function.get("allowed_callers")), + ) + }) +} + +fn non_empty_list(value: Option<&Value>) -> bool { + matches!(value, Some(Value::Array(items)) if !items.is_empty()) +} + +fn input_examples_used(tools: &[Value]) -> bool { + tools.iter().any(|tool| { + non_empty_list(tool.get("input_examples")) + || non_empty_list( + tool.get("function") + .and_then(|function| function.get("input_examples")), + ) + }) +} + +fn eager_input_streaming_flag(tool: &Value) -> Result, Error> { + let Value::Object(tool) = tool else { + return Ok(None); + }; + let strict_bool = |value: Option<&Value>| match value { + None | Some(Value::Null) => Ok(None), + Some(Value::Bool(flag)) => Ok(Some(*flag)), + Some(_) => Err(Error::InvalidRequest(ErrorDetail::Expected { + subject: "eager_input_streaming", + expected: "a boolean", + })), + }; + let top_level = strict_bool(tool.get("eager_input_streaming"))?; + let function = match tool.get("function") { + None | Some(Value::Null) => None, + Some(Value::Object(function)) => strict_bool(function.get("eager_input_streaming"))?, + Some(_) => { + return Err(Error::InvalidRequest(ErrorDetail::Expected { + subject: "eager_input_streaming", + expected: "a boolean", + })); + } + }; + Ok(top_level.or(function)) +} + +fn eager_input_streaming_used(tools: &[Value]) -> Result { + tools.iter().try_fold(false, |used, tool| { + Ok(eager_input_streaming_flag(tool)? == Some(true) || used) + }) +} + +fn effort_used(params: &Body, model: &str, capabilities: &MessagesModelCapabilities) -> bool { + if params.is_empty() || capabilities.supports_adaptive_thinking { + return false; + } + let lowered = model.to_lowercase(); + let opus_4_5_effort = OPUS_4_5_EFFORT_PATTERNS + .iter() + .any(|pattern| lowered.contains(pattern)) + && params + .get("reasoning_effort") + .and_then(Value::as_str) + .is_some_and(|effort| !effort.is_empty()); + let output_config_effort = params + .get("output_config") + .and_then(|output_config| output_config.get("effort")) + .and_then(Value::as_str) + .is_some_and(|effort| !effort.is_empty()); + opus_4_5_effort || output_config_effort +} + +fn system_messages(messages: &[Value]) -> impl Iterator { + messages + .iter() + .filter(|message| message.get("role").and_then(Value::as_str) == Some("system")) +} + +fn mid_conversation_tool_change_used(messages: &[Value]) -> bool { + system_messages(messages).any(|message| { + tools(message.get("content")).iter().any(|block| { + matches!(tool_type(block), Some("tool_addition" | "tool_removal")) + && block.get("tool").and_then(tool_type) == Some("tool_reference") + }) + }) +} + +fn thinking_betas(thinking: Option<&Value>) -> impl Iterator { + let display = super::request::is_thinking_display_updates_used(thinking) + .then_some(THINKING_DISPLAY_UPDATES); + let binding = + matches!(thinking, Some(Value::Object(thinking)) if thinking.contains_key("block_binding")) + .then_some(THINKING_BINDING_CONTROLS); + display.into_iter().chain(binding) +} + +fn carries_signed_compaction_block(messages: &[Value]) -> bool { + messages + .iter() + .flat_map(|message| tools(message.get("content"))) + .any(|block| { + tool_type(block) == Some("compaction") + && block + .get("signature") + .and_then(Value::as_str) + .is_some_and(|signature| !signature.is_empty()) + }) +} + +/// Port of Python's `_filter_context_management_for_bedrock_invoke`. +fn filter_context_management_for_bedrock_invoke(body: &mut Body, betas: &mut BTreeSet) { + let Some(Value::Object(context_management)) = body.get("context_management") else { + return; + }; + let Some(Value::Array(edits)) = context_management.get("edits") else { + body.shift_remove("context_management"); + return; + }; + let retained: Vec<(Value, &'static str)> = edits + .iter() + .filter_map(|edit| { + let edit_type = edit.as_object()?.get("type")?.as_str()?; + CONTEXT_MANAGEMENT_EDIT_BETAS + .iter() + .find(|(supported, _)| *supported == edit_type) + .map(|(_, beta)| (edit.clone(), *beta)) + }) + .collect(); + if retained.is_empty() { + body.shift_remove("context_management"); + return; + } + let mut filtered = context_management.clone(); + betas.extend(retained.iter().map(|(_, beta)| (*beta).to_string())); + filtered.insert( + "edits".into(), + Value::Array(retained.into_iter().map(|(edit, _)| edit).collect()), + ); + body.insert("context_management".into(), Value::Object(filtered)); +} + +/// Port of the handler's `update_headers_with_filtered_beta`, keeping +/// `model_specific_anthropic_betas`. +fn update_headers_with_filtered_beta(headers: Headers, keep: &BTreeSet<&'static str>) -> Headers { + BetaPolicy::Filter(BetaProvider::Bedrock).apply_keeping( + headers, + &keep + .iter() + .map(|beta| { + beta.parse::() + .unwrap_or_else(|never| match never {}) + }) + .collect::(), + ) +} + +/// Port of Python's `AnthropicModelInfo.get_anthropic_beta_list` for `custom_llm_provider="bedrock"`. +fn get_anthropic_beta_list( + body: &Body, + params: &Body, + model: &str, + capabilities: &MessagesModelCapabilities, + outgoing_tools: &[Value], + messages: &[Value], +) -> BTreeSet { + let mut betas = BTreeSet::new(); + if effort_used(params, model, capabilities) { + betas.insert("effort-2025-11-24".into()); + } + if let Some(beta) = outgoing_tools + .iter() + .find_map(|tool| tool_type(tool).filter(|kind| kind.starts_with("computer_"))) + .and_then(|kind| COMPUTER_TOOL_BETAS.iter().find(|(tool, _)| *tool == kind)) + .map(|(_, beta)| *beta) + { + betas.insert(beta.into()); + } + if params.get("mcp_servers").is_some_and(python_truthy) { + betas.insert("mcp-client-2025-04-04".into()); + } + if messages.iter().any(|message| { + message + .as_object() + .is_some_and(|message| message.contains_key("output_config")) + }) { + betas.insert(MID_CONVERSATION_OUTPUT_CONFIG.into()); + } + betas.extend(thinking_betas(body.get("thinking")).map(str::to_string)); + if mid_conversation_tool_change_used(messages) { + betas.insert(MID_CONVERSATION_TOOL_CHANGES.into()); + } + betas +} + +/// Port of Python's `_add_compaction_beta_for_bedrock_invoke`. +fn add_compaction_beta_for_bedrock_invoke( + body: &Body, + messages: &[Value], + betas: &mut BTreeSet, +) { + if body.get("compaction").is_some_and(Value::is_object) + || carries_signed_compaction_block(messages) + { + betas.insert(COMPACT_2026_09_04.into()); + } +} + +/// Port of Python's `_get_tool_search_beta_header_for_bedrock`. +fn get_tool_search_beta_header_for_bedrock( + capabilities: &MessagesModelCapabilities, + tool_search_used: bool, + programmatic_tool_calling_used: bool, + input_examples_used: bool, + betas: &mut BTreeSet, +) { + if tool_search_used && !(programmatic_tool_calling_used || input_examples_used) { + betas.remove(ADVANCED_TOOL_USE); + if capabilities.bedrock.supports_tool_search { + betas.insert(TOOL_SEARCH_TOOL.into()); + } + } +} + +/// Port of Python's `_get_bedrock_invoke_anthropic_beta_headers`. +pub(super) fn get_bedrock_invoke_anthropic_beta_headers( + body: &mut Body, + params: &Body, + headers: Headers, + model: &str, + capabilities: &MessagesModelCapabilities, + injected_thinking: bool, +) -> Result<(Vec, Headers), Error> { + let keep = model_specific_anthropic_betas(capabilities); + let headers = update_headers_with_filtered_beta(headers, &keep); + let user = user_betas(&headers); + let mut betas = user.clone(); + let outgoing_tools = tools(body.get("tools")).to_vec(); + let messages = tools(body.get("messages")).to_vec(); + betas.extend(get_anthropic_beta_list( + body, + params, + model, + capabilities, + &outgoing_tools, + &messages, + )); + if injected_thinking { + betas.insert("interleaved-thinking-2025-05-14".into()); + } + if eager_input_streaming_used(&outgoing_tools)? { + betas.insert("fine-grained-tool-streaming-2025-05-14".into()); + } + if params + .get("safeguards") + .is_some_and(|safeguards| !safeguards.is_null()) + { + betas.insert(DANGEROUS_TOOL_USE.into()); + } else { + betas.remove(DANGEROUS_TOOL_USE); + } + filter_context_management_for_bedrock_invoke(body, &mut betas); + add_compaction_beta_for_bedrock_invoke(body, &messages, &mut betas); + let tool_search_used = outgoing_tools + .iter() + .any(|tool| tool_type(tool).is_some_and(|kind| TOOL_SEARCH_TYPES.contains(&kind))); + get_tool_search_beta_header_for_bedrock( + capabilities, + tool_search_used, + programmatic_tool_calling_used(&outgoing_tools), + input_examples_used(&outgoing_tools), + &mut betas, + ); + if betas.contains(TOOL_SEARCH_TOOL) { + betas.insert(TOOL_EXAMPLES.into()); + } + let model_supported = user.iter().filter(|beta| keep.contains(beta.as_str())); + let filtered: BTreeSet = betas + .iter() + .filter_map(|beta| on_bedrock(beta)) + .chain(model_supported.cloned()) + .collect(); + let dropped: Vec<&String> = user + .iter() + .filter(|beta| !keep.contains(beta.as_str()) && on_bedrock(beta).is_none()) + .collect(); + if !dropped.is_empty() { + litellm_tracing::warn!( + "Bedrock Invoke: dropping unsupported anthropic-beta values from client headers: {dropped:?}. Bedrock has no mapping entry for these; forwarding them would cause a 400." + ); + } + Ok((filtered.into_iter().collect(), headers)) +} + +#[cfg(test)] +mod tests { + use litellm_llms_types::formats::messages::MessagesRequest; + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::{ + base_llm::messages::{ + context::{BedrockCapabilities, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, + bedrock::messages::invoke_transformations::anthropic_claude_transformation::AmazonAnthropicClaudeMessagesConfig, + }; + + fn sent( + params: Value, + capabilities: MessagesModelCapabilities, + header: Option<&str>, + drop_params: bool, + ) -> (Value, Headers) { + let mut request = json!({ + "model": "anthropic.claude-3-haiku-20240307-v1:0", + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "max_tokens": 128, + }); + request + .as_object_mut() + .unwrap() + .extend(params.as_object().unwrap().clone()); + let request: MessagesRequest = serde_json::from_value(request).unwrap(); + let context = MessagesTransformContext { + drops_unsupported_params: drop_params, + ..MessagesTransformContext::new(capabilities, drop_params) + }; + let headers = header + .map(|value| vec![("anthropic-beta".to_string(), value.to_string())]) + .unwrap_or_default(); + let (request, headers) = AmazonAnthropicClaudeMessagesConfig + .finalize_request(request, headers, &context) + .unwrap(); + (serde_json::to_value(request).unwrap(), headers) + } + + fn betas( + params: Value, + capabilities: MessagesModelCapabilities, + header: Option<&str>, + ) -> Value { + sent(params, capabilities, header, false) + .0 + .get("anthropic_beta") + .cloned() + .unwrap_or(Value::Null) + } + + fn bedrock(bedrock: BedrockCapabilities) -> MessagesModelCapabilities { + MessagesModelCapabilities { + bedrock, + ..MessagesModelCapabilities::default() + } + } + + #[rstest] + #[case::unlisted_beta_dropped("advisor-tool-2026-03-01,context-1m-2025-08-07", json!(["context-1m-2025-08-07"]))] + #[case::aliased_beta_renamed("advanced-tool-use-2025-11-20", json!(["tool-search-tool-2025-10-19"]))] + #[case::unknown_beta_dropped("not-a-real-beta-2099-01-01", Value::Null)] + #[case::dangerous_tool_use_dropped_without_safeguards( + "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14", + Value::Null + )] + fn user_betas_go_through_the_bedrock_column(#[case] header: &str, #[case] expected: Value) { + assert_eq!( + betas( + json!({}), + MessagesModelCapabilities::default(), + Some(header) + ), + expected + ); + } + + #[rstest] + #[case::client_sends_beta("dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")] + #[case::client_omits_beta("interleaved-thinking-2025-05-14")] + fn safeguards_carry_the_dangerous_tool_use_beta(#[case] header: &str) { + let (body, _) = sent( + json!({"safeguards": {"mode": "block"}}), + MessagesModelCapabilities::default(), + Some(header), + false, + ); + assert_eq!( + body["anthropic_beta"], + json!(["dangerous-tool-use-2026-09-03"]) + ); + assert_eq!(body["safeguards"], json!({"mode": "block"})); + } + + #[rstest] + #[case::flagged(true, json!(["per-turn-control-2026-07-01"]), Some("per-turn-control-2026-07-01"))] + #[case::not_flagged(false, Value::Null, None)] + fn per_turn_control_is_kept_only_where_the_model_flags_it( + #[case] supports_per_turn_control: bool, + #[case] expected: Value, + #[case] header: Option<&str>, + ) { + let (body, headers) = sent( + json!({}), + bedrock(BedrockCapabilities { + supports_per_turn_control, + ..BedrockCapabilities::default() + }), + Some("per-turn-control-2026-07-01"), + false, + ); + assert_eq!( + body.get("anthropic_beta").cloned().unwrap_or(Value::Null), + expected + ); + assert_eq!( + headers + .iter() + .find(|(name, _)| name == "anthropic-beta") + .map(|(_, value)| value.as_str()), + header + ); + } + + #[rstest] + #[case::supported(true, json!(["tool-search-tool-2025-10-19"]))] + #[case::unsupported(false, Value::Null)] + fn tool_search_beta_follows_the_model_map( + #[case] supports_tool_search: bool, + #[case] expected: Value, + ) { + assert_eq!( + betas( + json!({"tools": [{"type": "tool_search_tool_regex_20251119", "name": "tool_search"}]}), + bedrock(BedrockCapabilities { + supports_tool_search, + ..BedrockCapabilities::default() + }), + None, + ), + expected + ); + } + + #[test] + fn tool_search_with_programmatic_calling_keeps_the_users_beta() { + assert_eq!( + betas( + json!({"tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search"}, + {"name": "t", "input_schema": {"type": "object"}, "allowed_callers": ["code_execution_20250825"]}, + ]}), + MessagesModelCapabilities::default(), + Some("advanced-tool-use-2025-11-20"), + ), + json!(["tool-search-tool-2025-10-19"]) + ); + } + + #[rstest] + #[case::computer_20250124("computer_20250124", json!(["computer-use-2025-01-24"]))] + #[case::computer_20251124("computer_20251124", json!(["computer-use-2025-11-24"]))] + #[case::computer_20241022_is_rejected("computer_20241022", Value::Null)] + fn computer_tools_add_their_beta(#[case] tool_type: &str, #[case] expected: Value) { + assert_eq!( + betas( + json!({"tools": [{"type": tool_type, "name": "computer", "display_width_px": 1, "display_height_px": 1}]}), + MessagesModelCapabilities::default(), + None, + ), + expected + ); + } + + #[test] + fn output_config_effort_adds_the_effort_beta_on_non_adaptive_models() { + assert_eq!( + betas( + json!({"output_config": {"effort": "high"}}), + MessagesModelCapabilities { + supports_output_config: true, + ..MessagesModelCapabilities::default() + }, + None, + ), + json!(["effort-2025-11-24"]) + ); + } + + #[rstest] + #[case::display_updates(json!({"type": "adaptive", "display": "updates"}), json!(["thinking-display-updates-2026-08-18"]))] + #[case::block_binding(json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}), json!(["thinking-binding-controls-2026-08-01"]))] + fn thinking_controls_add_their_betas(#[case] thinking: Value, #[case] expected: Value) { + let capabilities = MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + ..MessagesModelCapabilities::default() + }; + assert_eq!( + betas(json!({"thinking": thinking}), capabilities, None), + expected + ); + } + + #[test] + fn mid_conversation_output_config_keeps_its_auto_added_beta() { + let (body, _) = sent( + json!({"messages": [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]}, + {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}}, + {"role": "user", "content": [{"type": "text", "text": "Go on"}]}, + ]}), + MessagesModelCapabilities { + supports_mid_conversation_system: true, + ..MessagesModelCapabilities::default() + }, + None, + true, + ); + assert_eq!( + body["anthropic_beta"], + json!(["mid-conversation-output-config-2026-07-01"]) + ); + assert_eq!( + body["messages"][2]["output_config"], + json!({"effort": "high"}) + ); + } + + #[test] + fn mid_conversation_tool_changes_add_their_beta() { + let (body, _) = sent( + json!({"messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "tool_name": "x"}}]}, + {"role": "user", "content": "Go on"}, + ]}), + MessagesModelCapabilities { + supports_mid_conversation_system: true, + ..MessagesModelCapabilities::default() + }, + None, + true, + ); + assert_eq!( + body["anthropic_beta"], + json!(["mid-conversation-tool-changes-2026-07-01"]) + ); + } + + #[rstest] + #[case::clear_at_stripped( + json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}), + true, + json!([ + {"role": "user", "content": "a"}, + {"role": "system", "content": "note"}, + {"role": "user", "content": "b"} + ]) + )] + #[case::emptied_turn_dropped( + json!({"role": "system", "content": "", "clear_at": {"type": "turns"}}), + true, + json!([{"role": "user", "content": "a"}, {"role": "user", "content": "b"}]) + )] + #[case::kept_without_drop_params( + json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}), + false, + json!([ + {"role": "user", "content": "a"}, + {"role": "system", "content": "note", "clear_at": {"type": "turns"}}, + {"role": "user", "content": "b"} + ]) + )] + fn a_system_field_the_body_betas_do_not_unlock_is_stripped_under_drop_params( + #[case] system: Value, + #[case] drop_params: bool, + #[case] expected: Value, + ) { + let (body, _) = sent( + json!({"messages": [{"role": "user", "content": "a"}, system, {"role": "user", "content": "b"}]}), + MessagesModelCapabilities { + supports_mid_conversation_system: true, + ..MessagesModelCapabilities::default() + }, + None, + drop_params, + ); + assert_eq!(body["messages"], expected); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs index 68ceaac3444..333753b6a5c 100644 --- a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs @@ -1 +1,4 @@ pub mod anthropic_claude_transformation; +mod betas; +pub mod request; +mod stream; diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/request.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/request.rs new file mode 100644 index 00000000000..d357bfd2a5d --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/request.rs @@ -0,0 +1,1088 @@ +use litellm_llms_types::{ + formats::messages::{Message, MessagesOptionalParams, MessagesRequest, ThinkingConfig}, + providers::anthropic::{AnthropicBeta, BetaProvider}, + recognized::Recognized, +}; +use serde_json::{Map, Value, json}; + +use crate::{ + Error, ErrorDetail, + anthropic::{ + beta_gated_fields::strip_beta_gated_request, common_utils::supports_effort_param, + messages::transformation::transform_messages_request, + }, + base_llm::{ + auth::Headers, + messages::{ + context::{MessagesModelCapabilities, MessagesTransformContext, ThinkingBudgets}, + normalization::normalize_system_role_messages, + }, + }, +}; + +use crate::bedrock::common_utils::{ + clamp_effort, ensure_bedrock_anthropic_messages_tool_names, is_int, + normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, + normalize_tool_input_schema_types_for_bedrock_invoke, + strip_unsupported_bedrock_invoke_output_config_keys, + structured_output::apply_bedrock_invoke_structured_output, tools_without_eager_input_streaming, +}; + +use super::betas::get_bedrock_invoke_anthropic_beta_headers; + +const BEDROCK_INVOKE_LOG_PROVIDER: &str = "bedrock_invoke"; + +pub(super) type Body = Map; + +pub const DEFAULT_BEDROCK_ANTHROPIC_API_VERSION: &str = "bedrock-2023-05-31"; + +/// Port of Python's `BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS`. +const BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS: [&str; 17] = [ + "anthropic_version", + "max_tokens", + "messages", + "anthropic_beta", + "system", + "stop_sequences", + "temperature", + "top_p", + "top_k", + "tools", + "tool_choice", + "thinking", + "metadata", + "output_config", + "safeguards", + "context_management", + "compaction", +]; + +const ADAPTIVE_THINKING_KEYS: [&str; 3] = ["type", "display", "block_binding"]; + +const EXTENDED_THINKING_PATTERNS: [&str; 12] = [ + "opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5", "opus-4.1", "opus_4.1", "opus-4-1", "opus_4_1", + "opus-4", "opus_4", "sonnet-4", "sonnet_4", +]; + +const CLEAR_THINKING_EDIT: &str = "clear_thinking_20251015"; + +fn to_body(request: &MessagesRequest) -> Result { + match serde_json::to_value(request) { + Ok(Value::Object(body)) => Ok(body), + Ok(_) => Err(Error::InvalidRequest(ErrorDetail::Expected { + subject: "Bedrock invoke request", + expected: "an object", + })), + Err(error) => Err(Error::InvalidRequest(ErrorDetail::failed( + "Bedrock invoke request serialization", + error, + ))), + } +} + +fn from_body(body: Body) -> Result { + serde_json::from_value(Value::Object(body)).map_err(|error| { + Error::InvalidRequest(ErrorDetail::invalid("Bedrock invoke request", error)) + }) +} + +/// Port of Python's `_get_bedrock_output_config_effort_ceiling`. +fn get_bedrock_output_config_effort_ceiling( + capabilities: &MessagesModelCapabilities, +) -> Option { + capabilities + .bedrock + .output_config_effort_ceiling + .map(|level| <&str>::from(level).to_string()) +} + +/// Port of Python's `_clamp_adaptive_reasoning_effort_for_bedrock`. +fn clamp_adaptive_reasoning_effort_for_bedrock( + request: MessagesRequest, + capabilities: &MessagesModelCapabilities, +) -> Result { + if !capabilities.supports_adaptive_thinking { + return Ok(request); + } + let mut body = to_body(&request)?; + let Some(clamped) = body + .get("reasoning_effort") + .and_then(Value::as_str) + .and_then(|effort| { + clamp_effort( + effort, + get_bedrock_output_config_effort_ceiling(capabilities).as_deref(), + ) + }) + else { + return Ok(request); + }; + body.insert("reasoning_effort".into(), Value::String(clamped)); + from_body(body) +} + +/// Port of Python's `AnthropicModelInfo.is_thinking_display_updates_used`. +pub(super) fn is_thinking_display_updates_used(thinking: Option<&Value>) -> bool { + let Some(Value::Object(thinking)) = thinking else { + return false; + }; + matches!( + thinking.get("type").and_then(Value::as_str), + Some("adaptive" | "enabled") + ) && thinking.get("display").and_then(Value::as_str) == Some("updates") +} + +/// Port of Python's `carried_thinking_keys`. +fn carried_thinking_keys(thinking: &Body, rebuilt: &[&str], drops_unknown: bool) -> Body { + thinking + .iter() + .filter(|(key, _)| !rebuilt.contains(&key.as_str())) + .filter(|(key, _)| ADAPTIVE_THINKING_KEYS.contains(&key.as_str()) || !drops_unknown) + .map(|(key, value)| (key.clone(), value.clone())) + .collect() +} + +fn readd_display_updates(body: &mut Body, drops_unknown: bool) { + let Some(Value::Object(thinking)) = body.get("thinking") else { + return; + }; + if thinking.get("type").and_then(Value::as_str) != Some("adaptive") { + return; + } + let mut rebuilt = carried_thinking_keys(thinking, &["display"], drops_unknown); + rebuilt.insert("display".into(), Value::String("updates".into())); + body.insert("thinking".into(), Value::Object(rebuilt)); +} + +/// Python's `strip_beta_gated_native_request` over the final body, whose `anthropic_beta` list +/// is the only one this leg sends. +fn strip_beta_gated_body(body: &mut Body, model: &str, betas: &[String]) -> Result<(), Error> { + let invalid = + |error| Error::InvalidRequest(ErrorDetail::invalid("Bedrock invoke request", error)); + let thinking: Option> = body + .get("thinking") + .cloned() + .map(serde_json::from_value) + .transpose() + .map_err(invalid)?; + let messages: Vec> = body + .get("messages") + .cloned() + .map(serde_json::from_value) + .transpose() + .map_err(invalid)? + .unwrap_or_default(); + let stripped = strip_beta_gated_request( + MessagesRequest { + model: model.to_string(), + messages, + params: MessagesOptionalParams { + thinking, + ..MessagesOptionalParams::default() + }, + }, + &betas + .iter() + .map(|beta| { + beta.parse::() + .unwrap_or_else(|never| match never {}) + }) + .collect(), + true, + BEDROCK_INVOKE_LOG_PROVIDER, + ); + let failed = |error| { + Error::InvalidRequest(ErrorDetail::failed( + "Bedrock invoke request serialization", + error, + )) + }; + if let Some(thinking) = stripped.params.thinking { + body.insert( + "thinking".into(), + serde_json::to_value(thinking).map_err(failed)?, + ); + } + if body.contains_key("messages") { + body.insert( + "messages".into(), + serde_json::to_value(stripped.messages).map_err(failed)?, + ); + } + Ok(()) +} + +/// Port of Python's `_reject_unsupported_web_search_tools`. +fn reject_unsupported_web_search_tools(body: &Body, model: &str) -> Result<(), Error> { + let Some(Value::Array(tools)) = body.get("tools") else { + return Ok(()); + }; + let web_search = tools.iter().filter_map(Value::as_object).find_map(|tool| { + tool.get("type") + .and_then(Value::as_str) + .filter(|tool_type| tool_type.starts_with("web_search_") && *tool_type != "function") + }); + match web_search { + Some(tool_type) => Err(Error::InvalidRequest( + ErrorDetail::UnsupportedBedrockWebSearchTool { + tool_type: tool_type.to_string(), + model: model.to_string(), + }, + )), + None => Ok(()), + } +} + +/// Port of Python's `_supports_extended_thinking_on_bedrock`. +pub(super) fn supports_extended_thinking_on_bedrock( + model: &str, + capabilities: &MessagesModelCapabilities, +) -> bool { + let lowered = model.to_lowercase(); + capabilities.supports_adaptive_thinking + || EXTENDED_THINKING_PATTERNS + .iter() + .any(|pattern| lowered.contains(pattern)) +} + +/// Port of Python's `_effort_from_thinking_budget`. +fn effort_from_thinking_budget(budget: i128, budgets: &ThinkingBudgets) -> &'static str { + if budget >= i128::from(budgets.xhigh) { + "xhigh" + } else if budget >= i128::from(budgets.high) { + "high" + } else if budget >= i128::from(budgets.medium) { + "medium" + } else { + "low" + } +} + +/// Port of Python's `_resolve_clear_thinking_budget_tokens`. +fn resolve_clear_thinking_budget_tokens( + value: Option<&Value>, + minimum: u64, +) -> Result { + match value { + None | Some(Value::Null) => Ok(i128::from(minimum)), + Some(Value::Bool(flag)) => Ok(i128::from(*flag)), + Some(Value::Number(number)) => number + .as_i64() + .map(i128::from) + .or_else(|| number.as_u64().map(i128::from)) + .or_else(|| { + number + .as_f64() + .filter(|value| value.is_finite()) + .map(|value| value.trunc() as i128) + }) + .ok_or_else(|| { + Error::InvalidRequest(ErrorDetail::Expected { + subject: "thinking.budget_tokens", + expected: "a finite number", + }) + }), + Some(_) => Err(Error::InvalidRequest(ErrorDetail::Expected { + subject: "thinking.budget_tokens", + expected: "an integer", + })), + } +} + +/// Port of Python's `_inject_adaptive_thinking_for_clear_thinking`. +fn inject_adaptive_thinking_for_clear_thinking( + body: &mut Body, + budget: i128, + budgets: &ThinkingBudgets, + drops_unknown: bool, +) { + let mut output_config = match body.get("output_config") { + Some(Value::Object(output_config)) => output_config.clone(), + _ => Map::new(), + }; + output_config + .entry("effort") + .or_insert_with(|| Value::String(effort_from_thinking_budget(budget, budgets).into())); + let thinking = match body.get("thinking") { + Some(Value::Object(thinking)) => thinking.clone(), + _ => Map::new(), + }; + let mut rebuilt = carried_thinking_keys(&thinking, &["type", "budget_tokens"], drops_unknown); + rebuilt.insert("type".into(), Value::String("adaptive".into())); + body.insert("output_config".into(), Value::Object(output_config)); + body.insert("thinking".into(), Value::Object(rebuilt)); +} + +/// Port of Python's `_ensure_thinking_for_clear_thinking_context_management`. +fn ensure_thinking_for_clear_thinking_context_management( + body: &mut Body, + model: &str, + capabilities: &MessagesModelCapabilities, + budgets: &ThinkingBudgets, + drops_unknown: bool, +) -> Result { + let Some(Value::Object(context_management)) = body.get("context_management") else { + return Ok(false); + }; + let Some(Value::Array(edits)) = context_management.get("edits") else { + return Ok(false); + }; + let needs_thinking = edits + .iter() + .any(|edit| edit.get("type").and_then(Value::as_str) == Some(CLEAR_THINKING_EDIT)); + if !needs_thinking || !supports_extended_thinking_on_bedrock(model, capabilities) { + return Ok(false); + } + let adaptive = capabilities.supports_adaptive_thinking; + if let Some(Value::Object(thinking)) = body.get("thinking") { + match thinking.get("type").and_then(Value::as_str) { + Some("adaptive") => return Ok(false), + Some("enabled") if !adaptive => return Ok(false), + Some("enabled") => { + let budget = resolve_clear_thinking_budget_tokens( + thinking.get("budget_tokens"), + budgets.bedrock_min, + )?; + inject_adaptive_thinking_for_clear_thinking(body, budget, budgets, drops_unknown); + return Ok(true); + } + _ => {} + } + } + let minimum = budgets.bedrock_min; + let too_small_max_tokens = body + .get("max_tokens") + .filter(|value| is_int(value)) + .and_then(|value| { + value + .as_i64() + .map(i128::from) + .or_else(|| value.as_u64().map(i128::from)) + }) + .filter(|max_tokens| *max_tokens <= i128::from(minimum)); + if let Some(max_tokens) = too_small_max_tokens { + litellm_tracing::warn!( + "Bedrock clear_thinking_20251015: max_tokens={max_tokens} is not greater than minimum thinking budget ({minimum}); cannot inject thinking safely" + ); + return Ok(false); + } + if adaptive { + inject_adaptive_thinking_for_clear_thinking( + body, + i128::from(minimum), + budgets, + drops_unknown, + ); + } else { + body.insert( + "thinking".into(), + json!({"type": "enabled", "budget_tokens": minimum}), + ); + } + Ok(true) +} + +/// Port of Python's `_remove_ttl_from_cache_control._sanitize_cache_control`. +fn sanitize_cache_control(cache_control: &mut Value, supports_ttl: bool) { + let Value::Object(cache_control) = cache_control else { + return; + }; + cache_control.shift_remove("scope"); + let keeps_ttl = supports_ttl + && matches!( + cache_control.get("ttl").and_then(Value::as_str), + Some("5m" | "1h") + ); + if !keeps_ttl { + cache_control.shift_remove("ttl"); + } +} + +/// Port of Python's `_remove_ttl_from_cache_control._process_content_list`. +fn process_content_list(blocks: &mut [Value], supports_ttl: bool) { + blocks + .iter_mut() + .filter_map(|block| block.get_mut("cache_control")) + .for_each(|cache_control| sanitize_cache_control(cache_control, supports_ttl)); +} + +/// Port of Python's `_remove_ttl_from_cache_control`. +fn remove_ttl_from_cache_control(body: &mut Body, supports_ttl: bool) { + if let Some(Value::Array(tools)) = body.get_mut("tools") { + process_content_list(tools, supports_ttl); + } + if let Some(Value::Array(system)) = body.get_mut("system") { + process_content_list(system, supports_ttl); + } + if let Some(Value::Array(messages)) = body.get_mut("messages") { + messages + .iter_mut() + .filter_map(|message| message.get_mut("content")) + .filter_map(Value::as_array_mut) + .for_each(|content| process_content_list(content, supports_ttl)); + } +} + +fn drop_compaction_without_beta(body: &mut Body, betas: &[String]) { + if body.contains_key("compaction") + && !(body.get("compaction").is_some_and(Value::is_object) + && betas.iter().any(|beta| beta == "compact-2026-09-04")) + { + litellm_tracing::debug!( + "Bedrock Invoke: dropping `compaction`; compact-2026-09-04 is not served on this route" + ); + body.shift_remove("compaction"); + } +} + +fn pop_output_config_under_global_drop( + body: &mut Body, + global_drop_params: bool, + supports_effort: bool, +) { + let has_non_format_keys = matches!( + body.get("output_config"), + Some(Value::Object(output_config)) if output_config.keys().any(|key| key != "format") + ); + if global_drop_params && has_non_format_keys && !supports_effort { + litellm_tracing::warn!( + "Bedrock Invoke: dropping `output_config` the model does not accept" + ); + body.shift_remove("output_config"); + } +} + +/// Port of Python's `_strip_unsupported_bedrock_invoke_fields`. +fn strip_unsupported_bedrock_invoke_fields(body: Body) -> Body { + let stripped: Vec<&String> = body + .keys() + .filter(|key| !BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS.contains(&key.as_str())) + .collect(); + if !stripped.is_empty() { + litellm_tracing::debug!( + "Bedrock Invoke: stripping unsupported top-level request fields: {stripped:?}" + ); + } + body.into_iter() + .filter(|(key, _)| BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS.contains(&key.as_str())) + .collect() +} + +/// Port of Python's `AmazonAnthropicClaudeMessagesConfig.transform_anthropic_messages_request`. +pub(super) fn transform_anthropic_messages_request( + request: MessagesRequest, + context: &MessagesTransformContext, + headers: Headers, +) -> Result<(MessagesRequest, Headers), Error> { + let model = request.model.clone(); + let stream = request.params.stream.clone(); + let capabilities = &context.thinking.capabilities; + let drops_unknown_thinking = context.drop_params || context.global_drop_params; + let requested_display_updates = is_thinking_display_updates_used( + serde_json::to_value(&request.params.thinking).ok().as_ref(), + ); + let request = clamp_adaptive_reasoning_effort_for_bedrock(request, capabilities)?; + let request = normalize_system_role_messages( + transform_messages_request(request, context)?, + capabilities.supports_mid_conversation_system, + context.drops_unsupported_params, + BetaProvider::Bedrock, + ); + let mut body = to_body(&request)?; + let shared_params = body.clone(); + if requested_display_updates { + readd_display_updates(&mut body, drops_unknown_thinking); + } + reject_unsupported_web_search_tools(&body, &model)?; + body.entry("anthropic_version") + .or_insert_with(|| Value::String(DEFAULT_BEDROCK_ANTHROPIC_API_VERSION.into())); + body.shift_remove("stream"); + body.shift_remove("model"); + let injected_thinking = ensure_thinking_for_clear_thinking_context_management( + &mut body, + &model, + capabilities, + &context.thinking.budgets, + drops_unknown_thinking, + )?; + remove_ttl_from_cache_control(&mut body, capabilities.bedrock.supports_prompt_cache_ttl); + apply_bedrock_invoke_structured_output(&mut body, &model, capabilities.bedrock.native_output)?; + let ceiling = get_bedrock_output_config_effort_ceiling(capabilities); + normalize_bedrock_opus_output_config_effort(body.get_mut("output_config"), ceiling.as_deref()); + let supports_effort = supports_effort_param(capabilities); + strip_unsupported_bedrock_invoke_output_config_keys( + &model, + &mut body, + capabilities.bedrock.supports_output_config || supports_effort, + ); + normalize_custom_field_on_tools(&mut body); + normalize_tool_input_schema_types_for_bedrock_invoke(&mut body); + ensure_bedrock_anthropic_messages_tool_names(&mut body); + let (betas, headers) = get_bedrock_invoke_anthropic_beta_headers( + &mut body, + &shared_params, + headers, + &model, + capabilities, + injected_thinking, + )?; + if !betas.is_empty() { + body.insert( + "anthropic_beta".into(), + Value::Array(betas.iter().cloned().map(Value::String).collect()), + ); + } + drop_compaction_without_beta(&mut body, &betas); + if context.drops_unsupported_params { + strip_beta_gated_body(&mut body, &model, &betas)?; + } + tools_without_eager_input_streaming(&mut body); + pop_output_config_under_global_drop(&mut body, context.global_drop_params, supports_effort); + let mut body = strip_unsupported_bedrock_invoke_fields(body); + body.insert("model".into(), Value::String(model)); + if let Some(stream) = stream { + body.insert( + "stream".into(), + serde_json::to_value(stream).unwrap_or(Value::Null), + ); + } + Ok((from_body(body)?, headers)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + use crate::{ + base_llm::messages::{ + context::{BedrockCapabilities, NativeOutputSupport, SupportedEffortTiers}, + transformation::BaseMessagesConfig, + }, + bedrock::messages::invoke_transformations::anthropic_claude_transformation::AmazonAnthropicClaudeMessagesConfig, + }; + use litellm_llms_types::formats::messages::EffortLevel; + + fn body(value: Value) -> Body { + value.as_object().unwrap().clone() + } + + fn adaptive() -> MessagesModelCapabilities { + MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + low: true, + medium: true, + high: true, + max: true, + xhigh: true, + ..SupportedEffortTiers::default() + }, + ..MessagesModelCapabilities::default() + } + } + + fn with_bedrock( + capabilities: MessagesModelCapabilities, + bedrock: BedrockCapabilities, + ) -> MessagesModelCapabilities { + MessagesModelCapabilities { + bedrock, + ..capabilities + } + } + + fn context( + capabilities: MessagesModelCapabilities, + drop_params: bool, + ) -> MessagesTransformContext { + MessagesTransformContext { + drops_unsupported_params: drop_params, + ..MessagesTransformContext::new(capabilities, drop_params) + } + } + + fn transformed_with( + request: Value, + context: &MessagesTransformContext, + headers: Headers, + ) -> Result<(Value, Headers), Error> { + let request: MessagesRequest = serde_json::from_value(request).unwrap(); + let (request, headers) = + AmazonAnthropicClaudeMessagesConfig.finalize_request(request, headers, context)?; + Ok(( + AmazonAnthropicClaudeMessagesConfig.wire_body(serde_json::to_value(&request).unwrap()), + headers, + )) + } + + fn rejection(error: Error) -> String { + match error { + Error::InvalidRequest(detail) => detail.to_string(), + other => panic!("expected a request rejection, got {other:?}"), + } + } + + fn transformed(request: Value, capabilities: MessagesModelCapabilities) -> Value { + transformed_with(request, &context(capabilities, false), Vec::new()) + .unwrap() + .0 + } + + fn hello(model: &str, params: Value) -> Value { + let mut request = json!({ + "model": model, + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "max_tokens": 4096, + }); + request + .as_object_mut() + .unwrap() + .extend(params.as_object().unwrap().clone()); + request + } + + #[test] + fn allowlist_filters_anthropic_only_fields() { + let wire = transformed( + hello( + "anthropic.claude-opus-4-7", + json!({ + "temperature": 0.5, + "mcp_servers": [{"type": "url", "url": "https://example.com"}], + "container": {"skills": []}, + "inference_geo": "us", + "output_config": {"effort": "low"}, + "context_management": {"edits": []}, + "stream": true, + }), + ), + MessagesModelCapabilities { + supports_sampling_params: true, + ..adaptive() + }, + ); + let fields = wire.as_object().unwrap(); + assert!( + fields + .keys() + .all(|key| BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS.contains(&key.as_str())), + "{wire}" + ); + assert_eq!(wire["output_config"], json!({"effort": "low"})); + assert_eq!(wire["max_tokens"], json!(4096)); + assert_eq!(wire["temperature"], json!(0.5)); + assert_eq!( + wire["anthropic_version"], + json!(DEFAULT_BEDROCK_ANTHROPIC_API_VERSION) + ); + } + + #[test] + fn a_callers_anthropic_version_wins() { + let wire = transformed( + hello("anthropic.claude-3", json!({"anthropic_version": "custom"})), + MessagesModelCapabilities::default(), + ); + assert_eq!(wire["anthropic_version"], json!("custom")); + } + + #[rstest] + #[case::ttl_dropped_without_support(false, json!({"type": "ephemeral", "ttl": "1h", "scope": "global"}), json!({"type": "ephemeral"}))] + #[case::ttl_kept_with_support(true, json!({"type": "ephemeral", "ttl": "1h", "scope": "global"}), json!({"type": "ephemeral", "ttl": "1h"}))] + #[case::unknown_ttl_dropped_with_support(true, json!({"type": "ephemeral", "ttl": "2h"}), json!({"type": "ephemeral"}))] + fn cache_control_keeps_only_supported_ttl_on_tools_system_and_messages( + #[case] supports_ttl: bool, + #[case] cache_control: Value, + #[case] expected: Value, + ) { + let mut request = body(json!({ + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}, "cache_control": cache_control.clone()}], + "system": [{"type": "text", "text": "You are helpful.", "cache_control": cache_control.clone()}], + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": cache_control}]}], + })); + remove_ttl_from_cache_control(&mut request, supports_ttl); + assert_eq!( + ( + &request["tools"][0]["cache_control"], + &request["system"][0]["cache_control"], + &request["messages"][0]["content"][0]["cache_control"], + ), + (&expected, &expected, &expected) + ); + } + + #[test] + fn eager_input_streaming_adds_its_beta_and_is_stripped_from_tools() { + let wire = transformed( + hello( + "anthropic.claude-sonnet-4-5", + json!({"tools": [{"name": "t", "input_schema": {"type": "object"}, "eager_input_streaming": true}]}), + ), + MessagesModelCapabilities::default(), + ); + assert_eq!( + wire["anthropic_beta"], + json!(["fine-grained-tool-streaming-2025-05-14"]) + ); + assert_eq!( + wire["tools"], + json!([{"name": "t", "input_schema": {"type": "object"}}]) + ); + } + + #[test] + fn a_non_bool_eager_input_streaming_is_refused() { + let error = transformed_with( + hello( + "anthropic.claude-sonnet-4-5", + json!({"tools": [{"name": "t", "input_schema": {"type": "object"}, "eager_input_streaming": "yes"}]}), + ), + &context(MessagesModelCapabilities::default(), false), + Vec::new(), + ) + .unwrap_err(); + assert_eq!(rejection(error), "eager_input_streaming must be a boolean"); + } + + #[test] + fn output_config_format_is_inlined_without_native_output() { + let schema = json!({"type": "object", "properties": {"answer": {"type": "string"}}}); + let wire = transformed( + hello( + "anthropic.claude-opus-4-7", + json!({"output_config": {"effort": "xhigh", "format": {"type": "json_schema", "schema": schema}}}), + ), + adaptive(), + ); + assert_eq!(wire["output_config"], json!({"effort": "xhigh"})); + let content = wire["messages"][0]["content"].as_array().unwrap(); + assert_eq!( + content.last().unwrap()["text"], + json!(r#"{"type": "object", "properties": {"answer": {"type": "string"}}}"#) + ); + } + + #[rstest] + #[case::opus_4_5_ceiling_high(Some(EffortLevel::High), "high")] + #[case::opus_4_6_ceiling_max(Some(EffortLevel::Max), "max")] + #[case::opus_4_7_no_ceiling(None, "xhigh")] + fn output_config_effort_is_clamped_to_the_bedrock_ceiling( + #[case] ceiling: Option, + #[case] expected: &str, + ) { + let capabilities = with_bedrock( + MessagesModelCapabilities { + supports_output_config: true, + ..MessagesModelCapabilities::default() + }, + BedrockCapabilities { + output_config_effort_ceiling: ceiling, + ..BedrockCapabilities::default() + }, + ); + let wire = transformed( + hello( + "anthropic.claude-opus-4-5", + json!({"output_config": {"effort": "xhigh"}}), + ), + capabilities, + ); + assert_eq!(wire["output_config"], json!({"effort": expected})); + } + + #[test] + fn global_drop_params_pops_output_config_the_model_does_not_take() { + let context = MessagesTransformContext { + global_drop_params: true, + ..context(MessagesModelCapabilities::default(), false) + }; + let mut request = + body(json!({"output_config": {"effort": "high", "format": {"type": "json_schema"}}})); + pop_output_config_under_global_drop(&mut request, context.global_drop_params, false); + assert_eq!(request.get("output_config"), None); + } + + #[rstest] + #[case::adaptive_without_thinking(adaptive(), json!({"max_tokens": 32000}), true, Some(json!({"type": "adaptive"})), Some("low"))] + #[case::adaptive_already_adaptive(adaptive(), json!({"max_tokens": 32000, "thinking": {"type": "adaptive"}}), false, Some(json!({"type": "adaptive"})), None)] + #[case::adaptive_legacy_budget(adaptive(), json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 4096}}), true, Some(json!({"type": "adaptive"})), Some("high"))] + #[case::adaptive_explicit_zero_budget(adaptive(), json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 0}}), true, Some(json!({"type": "adaptive"})), Some("low"))] + #[case::non_adaptive_keeps_enabled(MessagesModelCapabilities::default(), json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 2048}}), false, Some(json!({"type": "enabled", "budget_tokens": 2048})), None)] + #[case::non_adaptive_injects_enabled(MessagesModelCapabilities::default(), json!({"max_tokens": 32000}), true, Some(json!({"type": "enabled", "budget_tokens": 1024})), None)] + #[case::max_tokens_too_small(MessagesModelCapabilities::default(), json!({"max_tokens": 1024}), false, None, None)] + #[case::display_updates_preserved(adaptive(), json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 8192, "display": "updates"}}), true, Some(json!({"display": "updates", "type": "adaptive"})), Some("xhigh"))] + fn clear_thinking_injects_the_thinking_bedrock_requires( + #[case] capabilities: MessagesModelCapabilities, + #[case] request: Value, + #[case] injected: bool, + #[case] thinking: Option, + #[case] effort: Option<&str>, + ) { + let mut request = body(request); + request.insert( + "context_management".into(), + json!({"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}), + ); + let changed = ensure_thinking_for_clear_thinking_context_management( + &mut request, + "us.anthropic.claude-opus-4-5", + &capabilities, + &ThinkingBudgets::default(), + false, + ) + .unwrap(); + assert_eq!( + ( + changed, + request.get("thinking").cloned(), + request + .get("output_config") + .and_then(|config| config.get("effort")) + .and_then(Value::as_str) + ), + (injected, thinking, effort) + ); + } + + #[rstest] + #[case::xhigh(8192, "xhigh")] + #[case::high(4096, "high")] + #[case::medium(2048, "medium")] + #[case::low(2047, "low")] + #[case::zero(0, "low")] + fn effort_follows_thinking_budget_tiers(#[case] budget: i128, #[case] expected: &str) { + assert_eq!( + effort_from_thinking_budget(budget, &ThinkingBudgets::default()), + expected + ); + } + + #[rstest] + #[case::sonnet_4("anthropic.claude-sonnet-4-20250514-v1:0", false, true)] + #[case::opus_4_1("anthropic.claude-opus-4-1", false, true)] + #[case::haiku_3("anthropic.claude-3-haiku", false, false)] + #[case::adaptive_flag("anthropic.claude-fable-5", true, true)] + fn extended_thinking_follows_adaptive_flag_then_names( + #[case] model: &str, + #[case] supports_adaptive_thinking: bool, + #[case] expected: bool, + ) { + let capabilities = MessagesModelCapabilities { + supports_adaptive_thinking, + ..MessagesModelCapabilities::default() + }; + assert_eq!( + supports_extended_thinking_on_bedrock(model, &capabilities), + expected + ); + } + + #[test] + fn clear_thinking_edit_is_never_forwarded() { + let wire = transformed( + hello( + "anthropic.claude-sonnet-4-5", + json!({"max_tokens": 32000, "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}, {"type": "clear_tool_uses_20250919"}]}}), + ), + MessagesModelCapabilities::default(), + ); + assert_eq!( + wire["context_management"], + json!({"edits": [{"type": "clear_tool_uses_20250919"}]}) + ); + assert_eq!( + wire["thinking"], + json!({"type": "enabled", "budget_tokens": 1024}) + ); + assert_eq!( + wire["anthropic_beta"], + json!(["context-management-2025-06-27"]) + ); + } + + #[rstest] + #[case::non_list_edits_dropped(json!({"edits": "x"}), None, json!(null))] + #[case::unsupported_edits_dropped(json!({"edits": [{"type": "clear_thinking_20251015"}]}), None, json!(null))] + #[case::compact_kept(json!({"edits": [{"type": "compact_20260112"}]}), Some(json!({"edits": [{"type": "compact_20260112"}]})), json!(["compact-2026-01-12"]))] + fn context_management_keeps_bedrock_edits_with_their_betas( + #[case] context_management: Value, + #[case] expected: Option, + #[case] betas: Value, + ) { + let wire = transformed( + hello( + "anthropic.claude-3", + json!({"context_management": context_management}), + ), + MessagesModelCapabilities::default(), + ); + assert_eq!(wire.get("context_management").cloned(), expected); + assert_eq!( + wire.get("anthropic_beta").cloned().unwrap_or(Value::Null), + betas + ); + } + + #[rstest] + #[case::object_compaction_kept(json!({"type": "auto"}), Some(json!({"type": "auto"})))] + #[case::string_compaction_dropped(json!("auto"), None)] + #[case::list_compaction_dropped(json!([]), None)] + fn compaction_needs_an_object_and_its_beta( + #[case] compaction: Value, + #[case] expected: Option, + ) { + let wire = transformed( + hello("anthropic.claude-3", json!({"compaction": compaction})), + MessagesModelCapabilities::default(), + ); + assert_eq!(wire.get("compaction").cloned(), expected); + } + + #[test] + fn leading_system_turns_are_hoisted_and_string_system_becomes_blocks() { + let wire = transformed( + json!({ + "model": "anthropic.claude-3", + "max_tokens": 10, + "system": "base", + "messages": [ + {"role": "system", "content": "first"}, + {"role": "user", "content": "hi"}, + ], + }), + MessagesModelCapabilities::default(), + ); + assert_eq!( + wire["system"], + json!([{"type": "text", "text": "base"}, {"type": "text", "text": "first"}]) + ); + assert_eq!(wire["messages"], json!([{"role": "user", "content": "hi"}])); + } + + #[rstest] + #[case::kept_in_place(true, "system")] + #[case::converted_to_user(false, "user")] + fn mid_conversation_system_turns_follow_model_support( + #[case] supports_mid_conversation_system: bool, + #[case] role: &str, + ) { + let wire = transformed( + json!({ + "model": "anthropic.claude-3", + "max_tokens": 10, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "system", "content": "reminder"}, + ], + }), + MessagesModelCapabilities { + supports_mid_conversation_system, + ..MessagesModelCapabilities::default() + }, + ); + assert_eq!(wire["messages"][2]["role"], json!(role)); + } + + #[rstest] + #[case::web_search_20250305("web_search_20250305")] + #[case::web_search_20260209("web_search_20260209")] + fn server_web_search_tools_are_rejected(#[case] tool_type: &str) { + let error = transformed_with( + hello( + "anthropic.claude-sonnet-4-5", + json!({"tools": [{"type": tool_type, "name": "web_search"}]}), + ), + &context(MessagesModelCapabilities::default(), false), + Vec::new(), + ) + .unwrap_err(); + assert_eq!( + rejection(error), + format!( + "Bedrock does not support Anthropic's server-side web search tool (tool type '{tool_type}', model 'anthropic.claude-sonnet-4-5'). To use web search with this model, enable LiteLLM's web search interception so the proxy executes the search instead: https://docs.litellm.ai/docs/integrations/websearch_interception. Alternatively, remove the web_search tool from the request." + ) + ); + } + + #[test] + fn a_converted_websearch_function_tool_is_allowed() { + let wire = transformed( + hello( + "anthropic.claude-sonnet-4-5", + json!({"tools": [{"name": "litellm_web_search", "input_schema": {"type": "object"}}]}), + ), + MessagesModelCapabilities::default(), + ); + assert_eq!(wire["tools"][0]["name"], json!("litellm_web_search")); + } + + #[test] + fn reasoning_effort_is_clamped_before_the_effort_gate_on_adaptive_models() { + let capabilities = with_bedrock( + MessagesModelCapabilities { + effort_tiers: SupportedEffortTiers { + low: true, + medium: true, + high: true, + max: true, + ..SupportedEffortTiers::default() + }, + ..adaptive() + }, + BedrockCapabilities { + output_config_effort_ceiling: Some(EffortLevel::Max), + ..BedrockCapabilities::default() + }, + ); + let wire = transformed( + hello( + "anthropic.claude-opus-4-6", + json!({"reasoning_effort": "xhigh"}), + ), + capabilities, + ); + assert_eq!(wire["output_config"]["effort"], json!("max")); + assert_eq!(wire.get("reasoning_effort"), None); + } + + #[test] + fn native_output_forwards_a_cleaned_schema() { + let capabilities = with_bedrock( + adaptive(), + BedrockCapabilities { + native_output: NativeOutputSupport { + enabled: true, + mapped: true, + ..NativeOutputSupport::default() + }, + ..BedrockCapabilities::default() + }, + ); + let wire = transformed( + hello( + "anthropic.claude-opus-4-7", + json!({"output_config": {"format": {"type": "json_schema", "schema": {"type": "object", "properties": {"n": {"type": "integer", "minimum": 1}}, "required": ["n"]}}}}), + ), + capabilities, + ); + assert_eq!( + wire["output_config"], + json!({"format": {"type": "json_schema", "schema": {"type": "object", "properties": {"n": {"description": "Note: minimum value: 1.", "type": "integer"}}, "required": ["n"], "additionalProperties": false}}}) + ); + assert_eq!(wire["messages"][0]["content"].as_array().unwrap().len(), 1); + } + + #[test] + fn model_and_stream_leave_the_wire_body() { + let wire = transformed( + hello("anthropic.claude-3", json!({"stream": true})), + MessagesModelCapabilities::default(), + ); + assert_eq!((wire.get("model"), wire.get("stream")), (None, None)); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/stream.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/stream.rs new file mode 100644 index 00000000000..b5531bcfc08 --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/stream.rs @@ -0,0 +1,443 @@ +use std::collections::VecDeque; + +use bytes::Bytes; +use futures_util::{Stream, StreamExt, stream}; +use litellm_llms_types::providers::bedrock::INVOCATION_METRICS_KEY; +use litellm_python_compat::{Value, json, repr, truthy::truthy}; + +use crate::{ + Error, ErrorDetail, + base_llm::messages::streaming::{ByteStream, FrameStream}, +}; + +use crate::bedrock::chat::invoke_handler::{BedrockStreamRaise, aiter_bytes, raised}; + +const METRICS_USAGE_KEYS: [(&str, &str); 4] = [ + ("input_tokens", "inputTokenCount"), + ("output_tokens", "outputTokenCount"), + ("cache_read_input_tokens", "cacheReadInputTokenCount"), + ("cache_creation_input_tokens", "cacheWriteInputTokenCount"), +]; + +const CACHE_FIELDS: [&str; 2] = ["cache_creation_input_tokens", "cache_read_input_tokens"]; + +type Fields = Vec<(Value, Value)>; + +fn key(name: &str) -> Value { + Value::Str(name.to_string()) +} + +fn position(fields: &Fields, name: &str) -> Option { + fields + .iter() + .position(|(key, _)| matches!(key, Value::Str(key) if key == name)) +} + +fn get<'a>(fields: &'a Fields, name: &str) -> Option<&'a Value> { + position(fields, name).map(|at| &fields[at].1) +} + +fn set(fields: &mut Fields, name: &str, value: Value) { + match position(fields, name) { + Some(at) => fields[at].1 = value, + None => fields.push((key(name), value)), + } +} + +fn set_pair(fields: &mut Fields, name: Value, value: Value) { + match fields.iter().position(|(key, _)| *key == name) { + Some(at) => fields[at].1 = value, + None => fields.push((name, value)), + } +} + +fn pop(fields: &mut Fields, name: &str) -> Option { + position(fields, name).map(|at| fields.remove(at).1) +} + +fn is_type(fields: &Fields, event_type: &str) -> bool { + matches!(get(fields, "type"), Some(Value::Str(value)) if value == event_type) +} + +fn dict_or_empty(value: Option<&Value>) -> Fields { + match value { + Some(Value::Dict(fields)) => fields.clone(), + _ => Fields::new(), + } +} + +/// `AmazonAnthropicClaudeMessagesStreamDecoder._chunk_parser`: Bedrock's invocation metrics +/// become Anthropic usage, under the chunk's own usage. +fn chunk_parser(chunk: Value) -> Result { + let mut fields = match chunk { + Value::Dict(fields) => fields, + other => return Err(raised(BedrockStreamRaise::NotAnObject(dumped(&other)?))), + }; + let Some(metrics) = pop(&mut fields, INVOCATION_METRICS_KEY).filter(truthy) else { + return Ok(fields); + }; + let metrics = dict_or_empty(Some(&metrics)); + let preserved = dict_or_empty(get(&fields, "usage")); + let usage = preserved.into_iter().fold( + METRICS_USAGE_KEYS + .iter() + .filter_map(|(anthropic, metric)| { + Some((key(anthropic), get(&metrics, metric)?.clone())) + }) + .collect::(), + |mut usage, (name, value)| { + set_pair(&mut usage, name, value); + usage + }, + ); + set(&mut fields, "usage", Value::Dict(usage)); + Ok(fields) +} + +/// `AmazonAnthropicClaudeMessagesConfig._promote_message_stop_usage`, on the decoded chunks. +#[derive(Default)] +struct UsagePromoter { + pending_delta: Option, + start_usage: Option, +} + +impl UsagePromoter { + /// Port of Python's `_merge_message_start_cache_into_delta_usage`. + fn merge_message_start_cache_into_delta_usage(&self, delta_usage: &mut Fields) { + let Some(start) = self.start_usage.as_ref().filter(|start| !start.is_empty()) else { + return; + }; + for field in CACHE_FIELDS.iter().chain(["cache_creation"].iter()) { + if get(delta_usage, field).is_none() + && let Some(value) = get(start, field).filter(|value| **value != Value::None) + { + set(delta_usage, field, value.clone()); + } + } + } + + fn completed(&self, mut delta: Fields, stop: Option<&Fields>) -> Fields { + let mut delta_usage = dict_or_empty(get(&delta, "usage").filter(|usage| truthy(usage))); + if let Some(stop) = stop { + for field in CACHE_FIELDS { + if let Some(value) = get(stop, field) { + set(&mut delta_usage, field, value.clone()); + } + } + if let Some(raw) = get(stop, "input_tokens").filter(|raw| **raw != Value::None) { + let tokens = match raw { + Value::Int(_) | Value::Bool(_) => raw.clone(), + _ => Value::from(0), + }; + set(&mut delta_usage, "input_tokens", tokens); + } + } + self.merge_message_start_cache_into_delta_usage(&mut delta_usage); + if !delta_usage.is_empty() { + set(&mut delta, "usage", Value::Dict(delta_usage)); + } + delta + } + + fn feed(&mut self, chunk: Fields) -> Vec { + if is_type(&chunk, "message_start") { + let message = dict_or_empty(get(&chunk, "message").filter(|message| truthy(message))); + if let Some(Value::Dict(usage)) = get(&message, "usage") { + self.start_usage = Some(usage.clone()); + } + return self + .pending_delta + .take() + .into_iter() + .chain([chunk]) + .collect(); + } + if is_type(&chunk, "message_delta") { + self.pending_delta = Some(chunk); + return Vec::new(); + } + if is_type(&chunk, "message_stop") + && let Some(delta) = self.pending_delta.take() + { + let stop_usage = dict_or_empty(get(&chunk, "usage").filter(|usage| truthy(usage))); + return vec![self.completed(delta, Some(&stop_usage)), chunk]; + } + self.pending_delta + .take() + .into_iter() + .chain([chunk]) + .collect() + } + + fn finish(&mut self) -> Option { + let delta = self.pending_delta.take()?; + Some(self.completed(delta, None)) + } +} + +fn dumped(value: &Value) -> Result { + json::dumps(value) + .map_err(|error| Error::InvalidResponse(ErrorDetail::failed("SSE event", error))) +} + +/// `BaseAnthropicMessagesStreamingIterator._convert_chunk_to_sse_format` for a decoded chunk. +fn sse_frame(chunk: Fields) -> Result { + let event_type = get(&chunk, "type").map_or_else(|| "message".to_string(), repr::to_str); + let data = dumped(&Value::Dict(chunk))?; + Ok(Bytes::from(format!( + "event: {event_type}\ndata: {data}\n\n" + ))) +} + +struct Relay { + chunks: S, + promoter: UsagePromoter, + queued: VecDeque, + finished: bool, +} + +impl> + Unpin> Relay { + fn queue(&mut self, chunks: impl IntoIterator) -> Result<(), Error> { + for chunk in chunks { + self.queued.push_back(sse_frame(chunk)?); + } + Ok(()) + } + + async fn advance(&mut self) -> Result<(), Error> { + match self.chunks.next().await { + Some(chunk) => { + let chunk = chunk_parser(chunk?)?; + let promoted = self.promoter.feed(chunk); + self.queue(promoted) + } + None => { + self.finished = true; + let flushed = self.promoter.finish(); + self.queue(flushed) + } + } + } +} + +/// Bedrock InvokeModel's event stream as the SSE frames Python's Bedrock Messages stream yields: +/// each decoded chunk re-serialized with `json.dumps`, with usage promoted onto `message_delta`. +pub fn get_async_streaming_response_iterator( + bytes: ByteStream, + headers: Vec<(String, String)>, +) -> FrameStream { + let relay = Relay { + chunks: Box::pin(aiter_bytes(bytes, headers)), + promoter: UsagePromoter::default(), + queued: VecDeque::new(), + finished: false, + }; + stream::unfold(Some(relay), |relay| async move { + let mut relay = relay?; + loop { + if let Some(frame) = relay.queued.pop_front() { + return Some((Ok(frame), Some(relay))); + } + if relay.finished { + return None; + } + if let Err(error) = relay.advance().await { + return Some((Err(error), None)); + } + } + }) + .boxed() +} + +#[cfg(test)] +mod tests { + use aws_smithy_eventstream::frame::write_message_to; + use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; + use base64::{Engine, engine::general_purpose::STANDARD}; + use futures_util::TryStreamExt; + use rstest::rstest; + + use super::*; + + fn aws_frame(chunk: &str) -> Vec { + let payload = format!(r#"{{"bytes": "{}"}}"#, STANDARD.encode(chunk)); + let message = Message::new(Bytes::from(payload)) + .add_header(Header::new( + ":event-type", + HeaderValue::String("chunk".into()), + )) + .add_header(Header::new( + ":message-type", + HeaderValue::String("event".into()), + )); + let mut wire = Vec::new(); + write_message_to(&message, &mut wire).unwrap(); + wire + } + + fn in_pieces(chunks: &[&str]) -> ByteStream { + let wire: Vec = chunks.iter().flat_map(|chunk| aws_frame(chunk)).collect(); + stream::iter( + wire.chunks(7) + .map(|piece| Ok(Bytes::copy_from_slice(piece))) + .collect::>(), + ) + .boxed() + } + + async fn relayed(chunks: &[&str]) -> Result { + let frames: Vec = + get_async_streaming_response_iterator(in_pieces(chunks), Vec::new()) + .try_collect() + .await?; + Ok(frames + .iter() + .map(|frame| String::from_utf8(frame.to_vec()).unwrap()) + .collect()) + } + + const MESSAGE_STOP: &str = "event: message_stop\ndata: {\"type\": \"message_stop\"}\n\n"; + + #[rstest] + #[case::stop_sequence_null_is_kept( + r#"{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":4}}"#, + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\", \"stop_sequence\": null}, \"usage\": {\"output_tokens\": 4}}\n\n" + )] + #[case::an_absent_stop_sequence_stays_absent( + r#"{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":4}}"#, + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 4}}\n\n" + )] + #[case::unknown_keys_and_non_ascii_are_dumped_as_python_does( + r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hé"},"extra":[1.0,2]}"#, + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"h\\u00e9\"}, \"extra\": [1.0, 2]}\n\n" + )] + #[tokio::test] + async fn each_chunk_is_relayed_as_json_dumps_writes_it( + #[case] chunk: &str, + #[case] expected: &str, + ) { + assert_eq!( + relayed(&[chunk, r#"{"type":"message_stop"}"#]) + .await + .unwrap(), + format!("{expected}{MESSAGE_STOP}") + ); + } + + #[tokio::test] + async fn a_chunk_without_a_type_is_named_message() { + assert_eq!( + relayed(&[r#"{"message":"x"}"#, r#"{"type":"message_stop"}"#]) + .await + .unwrap(), + format!("event: message\ndata: {{\"message\": \"x\"}}\n\n{MESSAGE_STOP}") + ); + } + + #[rstest] + #[case::cache_fields_on_message_stop( + r#"{"input_tokens": 10, "output_tokens": 0}"#, + r#"{"output_tokens": 5}"#, + r#"{"input_tokens": 3, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20}"#, + r#"{"output_tokens": 5, "cache_creation_input_tokens": 20, "cache_read_input_tokens": 100, "input_tokens": 3}"# + )] + #[case::cache_only_on_message_start( + r#"{"input_tokens": 10, "output_tokens": 0, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}"#, + r#"{"output_tokens": 5}"#, + r#"{"input_tokens": 10}"#, + r#"{"output_tokens": 5, "input_tokens": 10, "cache_creation_input_tokens": 4, "cache_read_input_tokens": 80, "cache_creation": {"ephemeral_5m_input_tokens": 4}}"# + )] + #[case::a_float_input_count_is_zeroed( + r#"{"input_tokens": 10}"#, + r#"{"output_tokens": 5}"#, + r#"{"input_tokens": 3.5}"#, + r#"{"output_tokens": 5, "input_tokens": 0}"# + )] + #[tokio::test] + async fn message_delta_usage_is_completed_from_stop_then_start( + #[case] start: &str, + #[case] delta: &str, + #[case] stop: &str, + #[case] expected: &str, + ) { + let start = format!(r#"{{"type": "message_start", "message": {{"usage": {start}}}}}"#); + let delta = format!(r#"{{"type": "message_delta", "usage": {delta}}}"#); + let stop = format!(r#"{{"type": "message_stop", "usage": {stop}}}"#); + + let relayed = relayed(&[&start, &delta, &stop]).await.unwrap(); + + assert!( + relayed.contains(&format!( + "event: message_delta\ndata: {{\"type\": \"message_delta\", \"usage\": {expected}}}\n\n" + )), + "{relayed}" + ); + } + + #[rstest] + #[case::metrics_fill_missing_usage( + r#"{"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "outputTokenCount": 9}}"#, + r#"{"type": "message_stop", "usage": {"input_tokens": 3, "output_tokens": 9}}"# + )] + #[case::the_chunks_own_usage_wins_in_place( + r#"{"type": "message_stop", "usage": {"input_tokens": 1}, "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}, "amazon-bedrock-trace": {}}"#, + r#"{"type": "message_stop", "usage": {"input_tokens": 1, "cache_read_input_tokens": 40}, "amazon-bedrock-trace": {}}"# + )] + #[case::empty_metrics_are_dropped( + r#"{"type": "message_stop", "amazon-bedrock-invocationMetrics": {}}"#, + r#"{"type": "message_stop"}"# + )] + #[tokio::test] + async fn invocation_metrics_become_anthropic_usage( + #[case] chunk: &str, + #[case] expected: &str, + ) { + assert_eq!( + relayed(&[chunk]).await.unwrap(), + format!("event: message_stop\ndata: {expected}\n\n") + ); + } + + #[rstest] + #[case::no_message_stop( + &[r#"{"type":"message_start","message":{}}"#], + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {}}\n\n" + )] + #[case::a_held_delta_is_flushed( + &[r#"{"type":"message_delta","usage":{"output_tokens":1}}"#], + "event: message_delta\ndata: {\"type\": \"message_delta\", \"usage\": {\"output_tokens\": 1}}\n\n" + )] + #[tokio::test] + async fn a_stream_without_message_stop_ends_with_its_own_frames( + #[case] chunks: &[&str], + #[case] expected: &str, + ) { + assert_eq!(relayed(chunks).await.unwrap(), expected); + } + + #[tokio::test] + async fn a_provider_error_event_is_terminal() { + let error = r#"{"type":"error","error":{"type":"overloaded_error","message":"busy"}}"#; + + assert_eq!( + relayed(&[error]).await.unwrap(), + "event: error\ndata: {\"type\": \"error\", \"error\": {\"type\": \"overloaded_error\", \"message\": \"busy\"}}\n\n" + ); + } + + #[rstest] + #[case::a_list(r#"[1]"#, BedrockStreamRaise::NotAnObject("[1]".into()))] + #[case::not_json(r#"{"type": "#, BedrockStreamRaise::InvalidJson(r#"{"type": "#.into()))] + #[tokio::test] + async fn a_payload_python_cannot_parse_raises_what_python_raises( + #[case] chunk: &str, + #[case] expected: BedrockStreamRaise, + ) { + let error = relayed(&[chunk]).await.unwrap_err(); + + assert_eq!( + litellm_error::find::(&error), + Some(&expected) + ); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/mod.rs b/litellm/rust/crates/llms/src/bedrock/mod.rs index b2dc8937a8a..7c825c85812 100644 --- a/litellm/rust/crates/llms/src/bedrock/mod.rs +++ b/litellm/rust/crates/llms/src/bedrock/mod.rs @@ -3,3 +3,4 @@ pub mod chat; pub mod claude_platform; pub mod common_utils; pub mod messages; +pub mod request_metadata; diff --git a/litellm/rust/crates/llms/src/bedrock/request_metadata.rs b/litellm/rust/crates/llms/src/bedrock/request_metadata.rs new file mode 100644 index 00000000000..ae066ca66f6 --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/request_metadata.rs @@ -0,0 +1,336 @@ +use litellm_python_compat::{json as python_json, repr}; +use serde_json::{Map, Value}; + +use crate::base_llm::auth::Headers; + +pub const REQUEST_METADATA_FIELDS_PARAM: &str = "bedrock_request_metadata_fields"; +pub const METADATA_PARAMS: [&str; 2] = ["metadata", "litellm_metadata"]; + +pub const REQUEST_METADATA_HEADER: &str = "X-Amzn-Bedrock-Request-Metadata"; +const REQUEST_METADATA_MAX_PAIRS: usize = 16; +const REQUEST_METADATA_IDENTITY_PREFIX: &str = "user_api_key_"; +const REQUEST_METADATA_CLIENT_FIELD: &str = "spend_logs_metadata"; +const REQUEST_METADATA_HEADER_MAX_CHARS: usize = 8500; +const REQUEST_METADATA_MAX_CHARS: usize = 256; + +fn is_allowed_character(character: char) -> bool { + character.is_ascii_alphanumeric() + || matches!( + character, + ' ' | '\t' + | '\n' + | '\r' + | '\x0b' + | '\x0c' + | ':' + | '_' + | '@' + | '$' + | '#' + | '=' + | '/' + | '+' + | ',' + | '.' + | '-' + ) +} + +fn matches_constraint(text: &str, minimum: usize) -> bool { + let count = text.chars().count(); + (minimum..=REQUEST_METADATA_MAX_CHARS).contains(&count) + && text.chars().all(is_allowed_character) +} + +fn is_forwardable(key: &str, value: &str) -> bool { + matches_constraint(key, 1) && matches_constraint(value, 0) +} + +fn is_reserved(key: &str) -> bool { + key.trim() + .to_lowercase() + .starts_with(REQUEST_METADATA_IDENTITY_PREFIX) +} + +fn text_pairs(source: Option<&Value>) -> Vec<(&str, &str)> { + match source { + Some(Value::Object(fields)) => fields + .iter() + .filter_map(|(key, value)| Some((key.as_str(), value.as_str()?))) + .collect(), + _ => Vec::new(), + } +} + +fn allowed_fields(configured: Option<&Value>) -> Vec { + let Some(Value::Array(fields)) = configured else { + return Vec::new(); + }; + let fields: Vec = fields + .iter() + .map(|field| match field { + Value::String(text) => text.clone(), + other => repr::to_str(&python_json::from_json(other.clone())), + }) + .collect(); + fields + .iter() + .enumerate() + .filter(|(index, field)| !fields[..*index].contains(field)) + .map(|(_, field)| field.clone()) + .collect() +} + +/// Port of Python's `resolve_bedrock_request_metadata`. +fn resolve_bedrock_request_metadata( + sources: &[&Map], + allowed: &[String], +) -> Vec<(String, String)> { + let identity: Vec<(String, String)> = allowed + .iter() + .filter(|field| field.starts_with(REQUEST_METADATA_IDENTITY_PREFIX)) + .filter_map(|field| { + let value = sources + .iter() + .find_map(|source| source.get(field.as_str()).and_then(Value::as_str))?; + is_forwardable(field, value).then(|| (field.clone(), value.to_string())) + }) + .take(REQUEST_METADATA_MAX_PAIRS) + .collect(); + let spend_logs_pairs: Vec<(&str, &str)> = if allowed + .iter() + .any(|field| field == REQUEST_METADATA_CLIENT_FIELD) + { + sources + .iter() + .flat_map(|source| text_pairs(source.get(REQUEST_METADATA_CLIENT_FIELD))) + .collect() + } else { + Vec::new() + }; + let mut seen: Vec<&str> = Vec::new(); + let client: Vec<(String, String)> = spend_logs_pairs + .into_iter() + .filter(|(key, value)| !is_reserved(key) && is_forwardable(key, value)) + .filter(|(key, _)| { + let first = !seen.contains(key); + seen.push(key); + first + }) + .take(REQUEST_METADATA_MAX_PAIRS - identity.len()) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + identity + .into_iter() + .chain(client) + .fold(Vec::new(), |mut resolved, (key, value)| { + match resolved.iter_mut().find(|(existing, _)| *existing == key) { + Some(slot) => slot.1 = value, + None => resolved.push((key, value)), + } + resolved + }) +} + +fn header_value(pairs: &[(String, String)]) -> String { + let object: Map = pairs + .iter() + .map(|(key, value)| (key.clone(), Value::String(value.clone()))) + .collect(); + python_json::dumps(&python_json::from_json(Value::Object(object))) + .map(|text| compact_python_json(&text)) + .unwrap_or_default() +} + +fn compact_python_json(text: &str) -> String { + let mut compact = String::with_capacity(text.len()); + let mut in_string = false; + let mut escaped = false; + let mut characters = text.chars().peekable(); + while let Some(character) = characters.next() { + if in_string { + compact.push(character); + match (escaped, character) { + (true, _) => escaped = false, + (false, '\\') => escaped = true, + (false, '"') => in_string = false, + _ => {} + } + continue; + } + match character { + '"' => { + in_string = true; + compact.push(character); + } + ',' | ':' if characters.peek() == Some(&' ') => { + compact.push(character); + characters.next(); + } + _ => compact.push(character), + } + } + compact +} + +/// Port of Python's `_longest_fitting_prefix`. +fn longest_fitting_prefix(pairs: &[(String, String)]) -> Option { + (1..=pairs.len()) + .rev() + .map(|count| header_value(&pairs[..count])) + .find(|value| value.chars().count() <= REQUEST_METADATA_HEADER_MAX_CHARS) +} + +/// Port of Python's `bedrock_request_metadata_headers`. +pub(crate) fn bedrock_request_metadata_headers( + extra: &std::collections::BTreeMap, +) -> Option> { + let allowed = allowed_fields(extra.get(REQUEST_METADATA_FIELDS_PARAM)); + if allowed.is_empty() { + return None; + } + let sources: Vec<&Map> = METADATA_PARAMS + .iter() + .filter_map(|name| extra.get(*name).and_then(Value::as_object)) + .collect(); + let resolved = resolve_bedrock_request_metadata(&sources, &allowed); + Some( + longest_fitting_prefix(&resolved).map(|value| (REQUEST_METADATA_HEADER.to_string(), value)), + ) +} + +/// Port of Python's `merge_bedrock_invoke_headers`. +pub(crate) fn merge_bedrock_invoke_headers( + headers: Headers, + caller_owned: Vec<(String, String)>, + proxy_owned: Option>, +) -> Headers { + let owned_names: Vec = proxy_owned + .as_ref() + .map(|_| vec![REQUEST_METADATA_HEADER.to_lowercase()]) + .unwrap_or_default(); + if caller_owned.is_empty() && owned_names.is_empty() { + return headers; + } + let existing: Vec = headers + .iter() + .map(|(name, _)| name.to_lowercase()) + .collect(); + let kept = headers + .into_iter() + .filter(|(name, _)| !owned_names.contains(&name.to_lowercase())); + let added = caller_owned + .into_iter() + .filter(|(name, _)| !existing.contains(&name.to_lowercase())); + kept.chain(added).chain(proxy_owned.flatten()).fold( + Vec::new(), + |mut merged: Headers, (name, value)| { + match merged.iter_mut().find(|(existing, _)| *existing == name) { + Some(slot) => slot.1 = value, + None => merged.push((name, value)), + } + merged + }, + ) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::bedrock::chat::invoke_transformations::base_invoke_transformation::{ + GUARDRAIL_IDENTIFIER_HEADER, GUARDRAIL_VERSION_HEADER, + }; + + fn extra(value: Value) -> std::collections::BTreeMap { + value + .as_object() + .unwrap() + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect() + } + + #[test] + fn a_callers_guardrail_header_wins() { + let merged = merge_bedrock_invoke_headers( + vec![("x-amzn-bedrock-guardrailidentifier".into(), "caller".into())], + vec![ + (GUARDRAIL_IDENTIFIER_HEADER.into(), "deployment".into()), + (GUARDRAIL_VERSION_HEADER.into(), "1".into()), + ], + None, + ); + assert_eq!( + merged, + vec![ + ( + "x-amzn-bedrock-guardrailidentifier".to_string(), + "caller".to_string() + ), + (GUARDRAIL_VERSION_HEADER.to_string(), "1".to_string()), + ] + ); + } + + #[rstest] + #[case::disabled_by_default(json!({}), None)] + #[case::identity_then_spend_logs( + json!({ + "bedrock_request_metadata_fields": ["user_api_key_team_id", "spend_logs_metadata", "user_api_key_team_id"], + "metadata": {"user_api_key_team_id": "team-1", "spend_logs_metadata": {"project": "alpha", "user_api_key_alias": "forged", "bad key!": "x"}}, + }), + Some(Some(r#"{"user_api_key_team_id":"team-1","project":"alpha"}"#)), + )] + #[case::litellm_metadata_is_read_too( + json!({"bedrock_request_metadata_fields": ["user_api_key_user_id"], "litellm_metadata": {"user_api_key_user_id": "u 1"}}), + Some(Some(r#"{"user_api_key_user_id":"u 1"}"#)), + )] + #[case::owned_but_empty( + json!({"bedrock_request_metadata_fields": ["user_api_key_team_id"], "metadata": {}}), + Some(None), + )] + fn request_metadata_header_follows_the_allow_list( + #[case] params: Value, + #[case] expected: Option>, + ) { + assert_eq!( + bedrock_request_metadata_headers(&extra(params)), + expected.map(|header| header + .map(|value| (REQUEST_METADATA_HEADER.to_string(), value.to_string()))) + ); + } + + #[test] + fn the_proxy_owns_the_request_metadata_header() { + let merged = merge_bedrock_invoke_headers( + vec![ + ("x-amzn-bedrock-request-metadata".into(), "forged".into()), + ("x-keep".into(), "1".into()), + ], + Vec::new(), + Some(None), + ); + assert_eq!(merged, vec![("x-keep".to_string(), "1".to_string())]); + } + + #[test] + fn the_header_keeps_the_longest_prefix_that_fits() { + let long = "v".repeat(256); + let spend_logs: Map = (0..40) + .map(|index| (format!("key{index:02}"), Value::String(long.clone()))) + .collect(); + let params = extra(json!({ + "bedrock_request_metadata_fields": ["spend_logs_metadata"], + "metadata": {"spend_logs_metadata": spend_logs}, + })); + let Some(Some((_, value))) = bedrock_request_metadata_headers(¶ms) else { + panic!("expected a header"); + }; + let pairs: Map = serde_json::from_str(&value).unwrap(); + assert_eq!(pairs.len(), 16); + assert!(value.len() <= REQUEST_METADATA_HEADER_MAX_CHARS); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock_mantle/messages/transformation.rs b/litellm/rust/crates/llms/src/bedrock_mantle/messages/transformation.rs index 0e67ea1a139..51238c6539c 100644 --- a/litellm/rust/crates/llms/src/bedrock_mantle/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock_mantle/messages/transformation.rs @@ -5,8 +5,8 @@ use std::sync::LazyLock; use litellm_auth::{AwsParams, CredentialPlacement}; use litellm_secret_value::SecretValue; -use litellm_auth_aws::AwsCredentialSource; use litellm_auth_aws::bedrock_model_id_and_region; +use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders}; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{with_header, without_headers}; use litellm_llms_types::{ @@ -102,6 +102,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { region: mantle_region(params.api_base.as_deref(), model, params, env)?, service: "bedrock", credentials: Box::new(AwsCredentialSource::from_params(¶ms.aws, env)), + caller_headers: CallerSigningHeaders::Botocore, }, }; Ok(ValidatedEnvironment { headers, auth }) diff --git a/litellm/rust/crates/llms/src/error.rs b/litellm/rust/crates/llms/src/error.rs index af7ceb7da17..3aa1cc6d9df 100644 --- a/litellm/rust/crates/llms/src/error.rs +++ b/litellm/rust/crates/llms/src/error.rs @@ -220,6 +220,10 @@ pub enum ErrorDetail { #[source] cause: Source, }, + #[error( + "Bedrock does not support Anthropic's server-side web search tool (tool type '{tool_type}', model '{model}'). To use web search with this model, enable LiteLLM's web search interception so the proxy executes the search instead: https://docs.litellm.ai/docs/integrations/websearch_interception. Alternatively, remove the web_search tool from the request." + )] + UnsupportedBedrockWebSearchTool { tool_type: String, model: String }, #[error("{operation} failed: {detail}")] RemoteFailure { operation: &'static str, diff --git a/litellm/rust/crates/python-bridge/Cargo.toml b/litellm/rust/crates/python-bridge/Cargo.toml index d3c851ad162..8d7d714cd60 100644 --- a/litellm/rust/crates/python-bridge/Cargo.toml +++ b/litellm/rust/crates/python-bridge/Cargo.toml @@ -94,6 +94,7 @@ tempfile.workspace = true tokio-tungstenite.workspace = true wiremock.workspace = true aws-sdk-secretsmanager = "1.117.0" +aws-credential-types = "1.3.0" [[bench]] name = "serialization" diff --git a/litellm/rust/crates/python-bridge/src/errors.rs b/litellm/rust/crates/python-bridge/src/errors.rs index e19cd8880e1..6ca10c38a54 100644 --- a/litellm/rust/crates/python-bridge/src/errors.rs +++ b/litellm/rust/crates/python-bridge/src/errors.rs @@ -1,6 +1,6 @@ use std::error::Error as StdError; -use litellm_error::Report; +use litellm_error::{Report, Source}; use litellm_inference::Error; use pyo3::{ exceptions::{PyRuntimeError, PyTypeError, PyValueError}, @@ -42,28 +42,81 @@ const MESSAGES_PYTHON_RAISED: &str = "messages_python_raised"; /// The text of a refusal Python's provider config raises from its request transform, which its /// HTTP handler maps as a provider error. const MESSAGES_PROVIDER_REJECTION: &str = "messages_provider_rejection"; +/// Python's provider config raises the refusal as litellm's `BadRequestError`. +const MESSAGES_PROVIDER_BAD_REQUEST: &str = "messages_provider_bad_request"; +/// Which failure Python's AWS signing raises for a credential Rust could not acquire, or `replay` +/// when only running Python's signing again gives its exception. +const MESSAGES_CREDENTIAL_FAILURE: &str = "messages_credential_failure"; -fn provider_rejection(rejection: &(dyn StdError + 'static)) -> Option { +struct ProviderRejection { + text: String, + bad_request: bool, +} + +fn credential_failure(error: &Error) -> Option<&'static str> { + let (Error::Auth(auth) | Error::Llm(litellm_llms::Error::Auth(auth))) = error else { + return None; + }; + match auth { + litellm_auth::Error::Provider(_) => Some(aws_credential_failure(auth)), + litellm_auth::Error::CredentialAcquisition(_) + | litellm_auth::Error::CredentialChain { .. } => Some("replay"), + litellm_auth::Error::InvalidConfiguration(_) + | litellm_auth::Error::EmptyCallerCredential { .. } + | litellm_auth::Error::MissingApiKey { .. } + | litellm_auth::Error::MissingParam { .. } + | litellm_auth::Error::MissingApiBase { .. } + | litellm_auth::Error::InvalidHeader => None, + } +} + +fn aws_credential_failure(error: &(dyn StdError + 'static)) -> &'static str { + let innermost = std::iter::successors(Some(error), |&cause| cause.source()) + .filter_map(|cause| { + cause + .downcast_ref::() + .or_else(|| cause.downcast_ref::()?.downcast_ref()) + }) + .last(); + match innermost { + Some(litellm_auth_aws::Error::WebIdentityTokenReference) => "web_identity_reference", + Some(litellm_auth_aws::Error::MissingWebIdentityToken) => "missing_web_identity_token", + Some(failure) if failure.is_empty_default_chain() => "no_credentials", + _ => "replay", + } +} + +fn provider_rejection(rejection: &(dyn StdError + 'static)) -> Option { match rejection.downcast_ref::() { Some(litellm_llms::Error::MissingField("max_tokens")) => { - return Some("max_tokens is required for Anthropic /v1/messages API".to_owned()); + return Some(ProviderRejection { + text: "max_tokens is required for Anthropic /v1/messages API".to_owned(), + bad_request: false, + }); } Some(_) => return None, None => {} } match rejection.downcast_ref::() { Some( - detail @ litellm_llms::ErrorDetail::InvalidChoice { + detail @ (litellm_llms::ErrorDetail::InvalidChoice { field: "reasoning effort", .. - }, - ) => Some(format!("litellm.BadRequestError: {detail}")), + } + | litellm_llms::ErrorDetail::UnsupportedBedrockWebSearchTool { .. }), + ) => Some(ProviderRejection { + text: detail.to_string(), + bad_request: true, + }), Some( detail @ (litellm_llms::ErrorDetail::InvalidReasoningEffort { .. } | litellm_llms::ErrorDetail::UnsupportedValue { field: "effort", .. }), - ) => Some(detail.to_string()), + ) => Some(ProviderRejection { + text: detail.to_string(), + bad_request: false, + }), _ => None, } } @@ -182,8 +235,11 @@ pub(crate) fn route_failure( ) { value.setattr(MESSAGES_LATE_REJECTION, true)?; } - if let Some(text) = provider_rejection(rejection) { - value.setattr(MESSAGES_PROVIDER_REJECTION, text)?; + if let Some(provider) = provider_rejection(rejection) { + value.setattr(MESSAGES_PROVIDER_REJECTION, provider.text)?; + if provider.bad_request { + value.setattr(MESSAGES_PROVIDER_BAD_REQUEST, true)?; + } } native } @@ -204,6 +260,9 @@ pub(crate) fn route_failure( if error.is_missing_credential() { value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?; } + if let Some(failure) = credential_failure(&error) { + value.setattr(MESSAGES_CREDENTIAL_FAILURE, failure)?; + } native } (RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error), @@ -389,7 +448,7 @@ mod tests { #[rstest::rstest] #[case::missing_max_tokens( litellm_llms::Error::MissingField("max_tokens").into(), - Some("max_tokens is required for Anthropic /v1/messages API") + Some(("max_tokens is required for Anthropic /v1/messages API", false)) )] #[case::unmapped_effort( Error::invalid_request(litellm_llms::ErrorDetail::InvalidChoice { @@ -397,7 +456,17 @@ mod tests { actual: "'ultra'".into(), choices: vec!["low", "none"], }), - Some("litellm.BadRequestError: Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'.") + Some(("Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'.", true)) + )] + #[case::bedrock_web_search( + Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedBedrockWebSearchTool { + tool_type: "web_search_20250305".into(), + model: "claude".into(), + }), + Some(( + "Bedrock does not support Anthropic's server-side web search tool (tool type 'web_search_20250305', model 'claude'). To use web search with this model, enable LiteLLM's web search interception so the proxy executes the search instead: https://docs.litellm.ai/docs/integrations/websearch_interception. Alternatively, remove the web_search tool from the request.", + true + )) )] #[case::other_missing_field(litellm_llms::Error::MissingField("model").into(), None)] #[case::unsupported_param( @@ -411,17 +480,169 @@ mod tests { )] fn provider_config_rejections_carry_python_text( #[case] error: Error, - #[case] expected: Option<&str>, + #[case] expected: Option<(&str, bool)>, ) { Python::initialize(); Python::attach(|py| { let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); - let text = native - .value(py) + let value = native.value(py); + let text = value .getattr_opt(MESSAGES_PROVIDER_REJECTION) .unwrap() .map(|value| value.extract::().unwrap()); - assert_eq!(text.as_deref(), expected); + let bad_request = value + .getattr_opt(MESSAGES_PROVIDER_BAD_REQUEST) + .unwrap() + .is_some_and(|value| value.extract::().unwrap()); + assert_eq!(text.as_deref().map(|text| (text, bad_request)), expected); + assert!(text.is_some() || !bad_request); + }); + } + + fn web_identity_failure(failure: litellm_auth_aws::Error) -> litellm_auth_aws::Error { + litellm_auth_aws::Error::Credentials { + flow: litellm_auth_aws::CredentialFlow::WebIdentity, + source: Source::new( + aws_credential_types::provider::error::CredentialsError::provider_error(failure), + ), + } + } + + #[rstest::rstest] + #[case::environment_reference( + litellm_auth_aws::Error::WebIdentityTokenReference, + "web_identity_reference" + )] + #[case::unresolved_token( + web_identity_failure(litellm_auth_aws::Error::MissingWebIdentityToken), + "missing_web_identity_token" + )] + #[case::empty_default_chain( + litellm_auth_aws::Error::Credentials { + flow: litellm_auth_aws::CredentialFlow::DefaultChain, + source: Source::new(aws_credential_types::provider::error::CredentialsError::not_loaded( + "no providers in chain provided credentials", + )), + }, + "no_credentials" + )] + #[case::unreadable_token( + web_identity_failure(litellm_auth_aws::Error::WebIdentityToken(Source::new( + std::io::Error::other("secret source unavailable"), + ))), + "replay" + )] + #[case::sts_rejection( + web_identity_failure(litellm_auth_aws::Error::MissingWebIdentityCredentials), + "replay" + )] + #[case::assume_role( + litellm_auth_aws::Error::Credentials { + flow: litellm_auth_aws::CredentialFlow::AssumeRole, + source: Source::new(aws_credential_types::provider::error::CredentialsError::provider_error( + std::io::Error::other("AccessDenied"), + )), + }, + "replay" + )] + #[case::empty_profile( + litellm_auth_aws::Error::Credentials { + flow: litellm_auth_aws::CredentialFlow::Profile, + source: Source::new(aws_credential_types::provider::error::CredentialsError::not_loaded( + "profile not found", + )), + }, + "replay" + )] + #[case::expired(litellm_auth_aws::Error::ExpiredCredentials, "replay")] + fn an_aws_credential_failure_names_what_python_signing_raises( + #[case] failure: litellm_auth_aws::Error, + #[case] expected: &str, + #[values(false, true)] through_llm: bool, + ) { + let auth = litellm_auth::Error::from(failure); + let error = match through_llm { + true => Error::Llm(litellm_llms::Error::Auth(auth)), + false => Error::Auth(auth), + }; + Python::initialize(); + Python::attach(|py| { + let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); + assert!(native.is_instance_of::(py)); + let marked = native + .value(py) + .getattr(MESSAGES_CREDENTIAL_FAILURE) + .unwrap() + .extract::() + .unwrap(); + assert_eq!(marked, expected); + }); + } + + #[rstest::rstest] + #[case::acquisition( + litellm_auth::Error::CredentialAcquisition(litellm_auth::ErrorDetail::Empty { + subject: "token" + }), + Some("replay") + )] + #[case::chain( + litellm_auth::Error::CredentialChain { + last: Box::new(litellm_auth::Error::CredentialAcquisition( + litellm_auth::ErrorDetail::Empty { subject: "token" }, + )), + earlier: Vec::new(), + }, + Some("replay") + )] + #[case::missing_key( + litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: "ANTHROPIC_API_KEY", + }, + None + )] + #[case::invalid_header(litellm_auth::Error::InvalidHeader, None)] + fn only_credential_acquisition_failures_carry_the_credential_marker( + #[case] auth: litellm_auth::Error, + #[case] expected: Option<&str>, + ) { + Python::initialize(); + Python::attach(|py| { + let unmarked = + route_failure(py, Error::Auth(auth.clone()), RequestMarker::Unmarked, Ok).unwrap(); + assert!( + unmarked + .value(py) + .getattr_opt(MESSAGES_CREDENTIAL_FAILURE) + .unwrap() + .is_none() + ); + let native = route_failure(py, Error::Auth(auth), RequestMarker::Messages, Ok).unwrap(); + let marked = native + .value(py) + .getattr_opt(MESSAGES_CREDENTIAL_FAILURE) + .unwrap() + .map(|value| value.extract::().unwrap()); + assert_eq!(marked.as_deref(), expected); + }); + } + + #[test] + fn a_wrapped_credential_failure_keeps_the_native_mapping() { + let error = Error::post_call(Error::Auth(litellm_auth::Error::from( + litellm_auth_aws::Error::WebIdentityTokenReference, + ))); + Python::initialize(); + Python::attach(|py| { + let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); + assert!( + native + .value(py) + .getattr_opt(MESSAGES_CREDENTIAL_FAILURE) + .unwrap() + .is_none() + ); }); } diff --git a/litellm/rust/crates/python-bridge/src/http.rs b/litellm/rust/crates/python-bridge/src/http.rs index e7cc8e35172..806d883a33e 100644 --- a/litellm/rust/crates/python-bridge/src/http.rs +++ b/litellm/rust/crates/python-bridge/src/http.rs @@ -80,11 +80,16 @@ fn decode_ssl_verify(field: &Field<'_>) -> Result, ProjectionE Err(field.invalid("a Boolean, Boolean string, CA path, or None")) } -static RESOURCES: LazyLock = LazyLock::new(|| { - litellm_inference::resources::CoreResources::new(Arc::new(HttpClientPool::new(Arc::new( - PublicDnsResolver, - )))) -}); +static RESOURCES: LazyLock = + LazyLock::new(|| litellm_inference::resources::CoreResources { + pool: Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))), + auth: Arc::new(litellm_auth::AuthServices { + aws: litellm_auth_aws::AwsAuthService::with_web_identity_tokens(Arc::new( + crate::secrets::HostWebIdentityTokens, + )), + ..litellm_auth::AuthServices::default() + }), + }); pub(crate) fn resources() -> &'static litellm_inference::resources::CoreResources { &RESOURCES diff --git a/litellm/rust/crates/python-bridge/src/logger/machine.rs b/litellm/rust/crates/python-bridge/src/logger/machine.rs index 54f8f3d4b3f..12d9d184786 100644 --- a/litellm/rust/crates/python-bridge/src/logger/machine.rs +++ b/litellm/rust/crates/python-bridge/src/logger/machine.rs @@ -26,7 +26,9 @@ impl Machine for LoggedMachine { type Complete = M::Complete; fn resume(&mut self) -> Step<'_, Self> { - let logger = self.logger.get_or_init(|| Python::attach(super::capture)); + let logger = self + .logger + .get_or_init(|| Python::attach(super::capture_call)); Box::pin(logger.instrument(logger.scope(|| self.machine.resume()))) } @@ -34,7 +36,9 @@ impl Machine for LoggedMachine { &mut self, failure: HostFailure<::Error>, ) -> Interrupted<'_, Self> { - let logger = self.logger.get_or_init(|| Python::attach(super::capture)); + let logger = self + .logger + .get_or_init(|| Python::attach(super::capture_call)); Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure)))) } } diff --git a/litellm/rust/crates/python-bridge/src/logger/mod.rs b/litellm/rust/crates/python-bridge/src/logger/mod.rs index 70d323bbd84..1dae55262a5 100644 --- a/litellm/rust/crates/python-bridge/src/logger/mod.rs +++ b/litellm/rust/crates/python-bridge/src/logger/mod.rs @@ -2,6 +2,8 @@ mod machine; pub(crate) use machine::LoggedMachine; +use std::sync::atomic::{AtomicU8, Ordering}; + use litellm_host_python::{Pythonized, release_gil}; use litellm_tracing::{ DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink, TokenRedactor, @@ -118,6 +120,40 @@ fn processing_error(error: fancy_regex::Error) -> PyErr { struct PythonSink { correlation: (String, String), + levels: Option, +} + +#[derive(Default)] +struct Levels([AtomicU8; 4]); + +const UNKNOWN: u8 = 0; +const DISABLED: u8 = 1; +const ENABLED: u8 = 2; + +impl Levels { + fn enabled(&self, level: u8, query: impl FnOnce() -> bool) -> bool { + let slot = &self.0[usize::from(level / 10 - 1)]; + match slot.load(Ordering::Relaxed) { + UNKNOWN => { + let enabled = query(); + slot.store(if enabled { ENABLED } else { DISABLED }, Ordering::Relaxed); + enabled + } + state => state == ENABLED, + } + } +} + +fn python_enabled(level: u8) -> bool { + Python::try_attach(|py| { + report( + py, + py.import(MODULE) + .and_then(|module| module.call_method1("enabled", (level,))) + .and_then(|enabled| enabled.extract()), + ) + }) + .unwrap_or(false) } fn level(level: &Level) -> u8 { @@ -145,15 +181,11 @@ impl Sink for PythonSink { { return false; } - Python::try_attach(|py| { - report( - py, - py.import(MODULE) - .and_then(|module| module.call_method1("enabled", (level(metadata.level()),))) - .and_then(|enabled| enabled.extract()), - ) - }) - .unwrap_or(false) + let level = level(metadata.level()); + match &self.levels { + Some(levels) => levels.enabled(level, || python_enabled(level)), + None => python_enabled(level), + } } fn emit(&self, record: &Record) { @@ -182,12 +214,25 @@ impl Sink for PythonSink { } pub(crate) fn capture(py: Python<'_>) -> Logger { + logger(py, None) +} + +pub(crate) fn capture_call(py: Python<'_>) -> Logger { + logger(py, Some(Levels::default())) +} + +fn logger(py: Python<'_>, levels: Option) -> Logger { report( py, py.import(MODULE) .and_then(|module| module.call_method0("context")) .and_then(|value| value.extract()) - .map(|correlation| Logger::new(PythonSink { correlation })), + .map(|correlation| { + Logger::new(PythonSink { + correlation, + levels, + }) + }), ) } diff --git a/litellm/rust/crates/python-bridge/src/logger/tests.rs b/litellm/rust/crates/python-bridge/src/logger/tests.rs index 49e0cbf07f2..81af9ca883b 100644 --- a/litellm/rust/crates/python-bridge/src/logger/tests.rs +++ b/litellm/rust/crates/python-bridge/src/logger/tests.rs @@ -1,4 +1,4 @@ -use std::{process::Command, task::Poll}; +use std::{cell::Cell, process::Command, task::Poll}; use litellm_host::{ machine::{HostFailure, Interrupted, Machine, MachineStep, Step}, @@ -39,6 +39,51 @@ impl Machine for DiagnosticMachine { } } +struct ChunkMachine { + raise_level: bool, +} + +impl Machine for ChunkMachine { + type Protocol = DiagnosticMachine; + type Complete = (); + + fn resume(&mut self) -> Step<'_, Self> { + let raise_level = self.raise_level; + Box::pin(async move { + for index in 0..3 { + litellm_tracing::debug!(index, "stream chunk"); + if raise_level { + Python::attach(|py| { + py.import("litellm._logging")? + .getattr("verbose_logger")? + .call_method1("setLevel", (10,)) + .map(|_| ()) + }) + .map_err(|error| error.to_string())?; + } + tokio::task::yield_now().await; + } + Ok(MachineStep::Complete(())) + }) + } + + fn interrupt(&mut self, _: HostFailure) -> Interrupted<'_, Self> { + Box::pin(async { Ok(()) }) + } +} + +#[pyfunction] +fn machine_chunks(py: Python<'_>, raise_level: bool) -> PyResult> { + let mut machine = super::LoggedMachine::new(ChunkMachine { raise_level }); + litellm_host_python::run_async_value(py, async move { + machine + .resume() + .await + .map(|_| ()) + .map_err(pyo3::exceptions::PyValueError::new_err) + }) +} + #[pyfunction] fn machine_warning(py: Python<'_>) -> PyResult> { let mut machine = super::LoggedMachine::new(DiagnosticMachine); @@ -122,6 +167,24 @@ fn http_warning(py: Python<'_>) -> PyResult<()> { crate::http::call_config(py, &PyDict::new(py), false).map(|_| ()) } +#[rstest::rstest] +#[case::debug(10)] +#[case::info(20)] +#[case::warning(30)] +#[case::error(40)] +fn call_levels_ask_python_once_per_level(#[case] level: u8, #[values(true, false)] enabled: bool) { + let levels = super::Levels::default(); + let queries = Cell::new(0); + let query = || { + queries.set(queries.get() + 1); + enabled + }; + assert_eq!(levels.enabled(level, query), enabled); + assert_eq!(levels.enabled(level, query), enabled); + assert_eq!(levels.enabled(level, query), enabled); + assert_eq!(queries.get(), 1); +} + #[rstest::rstest] fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() { if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() { @@ -157,6 +220,12 @@ fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication wrap_pyfunction!(machine_warning, py).unwrap(), ) .unwrap(); + locals + .set_item( + "machine_chunks", + wrap_pyfunction!(machine_chunks, py).unwrap(), + ) + .unwrap(); locals .set_item( "synchronous_failure", @@ -281,6 +350,42 @@ try: ('warn', logging.WARNING), ('error', logging.ERROR), ] + import litellm.rust_bridge.host.logger as host_logger + original_enabled = host_logger.enabled + queries = [] + def counting_enabled(level): + queries.append(level) + return original_enabled(level) + host_logger.enabled = counting_enabled + async def chunks(raise_level): + await machine_chunks(raise_level) + try: + verbose_logger.setLevel(logging.WARNING) + before_chunks = len(capture.records) + asyncio.run(chunks(False)) + assert capture.records[before_chunks:] == [] + assert queries == [logging.DEBUG] + + queries.clear() + verbose_logger.setLevel(logging.DEBUG) + asyncio.run(chunks(False)) + assert [(r.getMessage(), r.levelno, r.rust_fields) for r in capture.records[before_chunks:]] == [ + ('stream chunk', logging.DEBUG, {'index': 0}), + ('stream chunk', logging.DEBUG, {'index': 1}), + ('stream chunk', logging.DEBUG, {'index': 2}), + ] + assert queries == [logging.DEBUG] * 4 + + queries.clear() + verbose_logger.setLevel(logging.WARNING) + before_raise = len(capture.records) + asyncio.run(chunks(True)) + assert capture.records[before_raise:] == [] + assert queries == [logging.DEBUG] + finally: + host_logger.enabled = original_enabled + verbose_logger.setLevel(logging.DEBUG) + before_spans = len(capture.records) async def traced_request(): session = session_id_var.set('span-session') diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/host.rs b/litellm/rust/crates/python-bridge/src/routes/messages/host.rs index 915e0581793..a31e0ea8061 100644 --- a/litellm/rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm/rust/crates/python-bridge/src/routes/messages/host.rs @@ -21,6 +21,7 @@ use litellm_llms::{ CALLABLE_TOKEN_PROVIDER, CALLER_TOKEN_PROVIDER_PARAM, ENTRA_ID_PARAMS, }, base_llm::messages::context::MessagesModelCapabilities, + bedrock::chat::invoke_handler::BedrockStreamRaise, }; use litellm_llms_types::headers::ProviderSpecificHeaders; use litellm_router_types::LitellmParams; @@ -51,14 +52,14 @@ type MergedHeaders = (Option>, Option>); const ANTHROPIC_BETA_HEADER: &str = "anthropic-beta"; -/// Python filters `anthropic-beta` before its client encodes it, so the wire check decides it. -/// Only for `azure_ai`: by the owner's ruling `anthropic` keeps main's hold-back of the header. +/// A config whose `should_filter_anthropic_beta_headers` holds filters `anthropic-beta` before +/// Python's client encodes it, so the wire check decides it. fn is_filtered_beta_header( name: &Bound<'_, PyAny>, value: &Bound<'_, PyAny>, - provider_filters_beta: bool, + filters_beta: bool, ) -> bool { - provider_filters_beta + filters_beta && name .extract::() .is_ok_and(|name| name.eq_ignore_ascii_case(ANTHROPIC_BETA_HEADER)) @@ -68,7 +69,7 @@ fn is_filtered_beta_header( fn ascii_headers<'py>( headers: &Bound<'py, PyAny>, unencodable: &mut Option>, - provider_filters_beta: bool, + filters_beta: bool, ) -> PyResult> { let Ok(headers) = headers.cast::() else { return Ok(headers.clone()); @@ -77,7 +78,7 @@ fn ascii_headers<'py>( let kept = PyDict::new(py); for (name, value) in headers.iter() { if value.is_instance_of::() - && !is_filtered_beta_header(&name, &value, provider_filters_beta) + && !is_filtered_beta_header(&name, &value, filters_beta) && let Err(error) = value.call_method1("encode", ("ascii",)) { if !error.is_instance_of::(py) { @@ -99,7 +100,7 @@ const PROVIDER_SPECIFIC_HEADERS_MODULE: &str = fn ascii_scoped_headers<'py>( value: &Bound<'py, PyAny>, - provider_filters_beta: bool, + filters_beta: bool, ) -> PyResult> { let ascii_entry = |entry: Bound<'py, PyAny>| -> PyResult> { let Ok(entry) = entry.cast::() else { @@ -111,7 +112,7 @@ fn ascii_scoped_headers<'py>( let kept = entry.copy()?; kept.set_item( "extra_headers", - ascii_headers(&headers, &mut None, provider_filters_beta)?, + ascii_headers(&headers, &mut None, filters_beta)?, )?; Ok(kept.into_any()) }; @@ -146,7 +147,8 @@ fn json_messages(messages: &Bound<'_, PyAny>) -> PyResult { } pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; -const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities"; +const MESSAGES_STREAM_FAILURE: &str = "messages_stream_failure"; +pub(super) const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities"; fn merge_headers( forwarded: Option>, @@ -162,6 +164,63 @@ const FEDERATING_PROVIDER: &str = "anthropic"; const AZURE_AI_PROVIDER: &str = "azure_ai"; const ENABLE_TOKEN_REFRESH: &str = "enable_azure_ad_token_refresh"; +const BEDROCK_PROVIDER: &str = "bedrock"; +const BEDROCK_PASSTHROUGH_PARAMS: [&str; 2] = ["model_id", "guardrailConfig"]; +const BEDROCK_METADATA_PARAMS: [&str; 2] = ["metadata", "litellm_metadata"]; +const BEDROCK_SPEND_LOGS_METADATA: &str = "spend_logs_metadata"; +const BEDROCK_REQUEST_METADATA_FIELDS: &str = "bedrock_request_metadata_fields"; + +fn text_pairs(value: &Bound<'_, PyAny>) -> PyResult> { + let Ok(mapping) = value.cast::() else { + return Ok(Map::new()); + }; + mapping + .items()? + .iter() + .filter_map(|item| { + let (key, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = match item.extract() { + Ok(pair) => pair, + Err(error) => return Some(Err(error)), + }; + let key = key.cast::().ok()?.to_str().ok()?.to_string(); + if key == BEDROCK_SPEND_LOGS_METADATA { + return Some(text_pairs(&value).map(|pairs| (key, Value::Object(pairs)))); + } + let value = value.cast::().ok()?.to_str().ok()?.to_string(); + Some(Ok((key, Value::String(value)))) + }) + .collect() +} + +fn bedrock_litellm_params<'py>( + py: Python<'py>, + argument: impl Fn(&str) -> PyResult>>, +) -> PyResult> { + let mut extra = project_optional_fields(BEDROCK_PASSTHROUGH_PARAMS, &argument)?; + for name in BEDROCK_METADATA_PARAMS { + if let Some(value) = argument(name)? { + extra.insert(name.to_string(), Value::Object(text_pairs(&value)?)); + } + } + if let Some(configured) = module_global(py, BEDROCK_REQUEST_METADATA_FIELDS)? + && (configured.is_instance_of::() || configured.is_instance_of::()) + { + let fields = configured + .try_iter()? + .map(|field| Ok(Value::String(field?.str()?.to_str()?.to_string()))) + .collect::>>()?; + extra.insert( + BEDROCK_REQUEST_METADATA_FIELDS.to_string(), + Value::Array(fields), + ); + } + Ok(extra) +} + +fn global_drop_params(py: Python<'_>) -> PyResult { + Ok(module_global(py, "drop_params")?.is_some_and(|value| value.is(PyBool::new(py, true)))) +} + fn workload_identity_fields(py: Python<'_>) -> PyResult<&'static [String]> { static FIELDS: PyOnceLock> = PyOnceLock::new(); FIELDS @@ -266,6 +325,16 @@ impl litellm_host::protocol::Protocol for MessagesHosted { type HostCall = MessagesHostCall; type Chunk = Bytes; type StreamHead = MessagesStreamHead; + + fn detached_drain(head: &MessagesStreamHead) -> Option { + ::detached_drain(head) + } + + fn unobserved_tail( + head: &MessagesStreamHead, + ) -> Option { + ::unobserved_tail(head) + } } pub(super) enum MessagesHostCall { @@ -325,6 +394,7 @@ pub(super) struct MessagesPythonHost { pending_short_circuit: Option<(bool, Reply)>, answer: Option>, answered: Option, + stream_opened: bool, } impl MessagesPythonHost { @@ -354,6 +424,7 @@ impl MessagesPythonHost { pending_short_circuit: None, answer: None, answered: None, + stream_opened: false, } } @@ -550,32 +621,41 @@ impl MessagesPythonHost { .call1((&model, api_base.as_deref()))? .extract()?, }; + let filters_beta: bool = py + .import(ROUTE_HOST_MODULE)? + .getattr("filters_beta_headers")? + .call1((&model, custom_llm_provider.as_deref(), api_base.as_deref()))? + .extract()?; let (extra_headers, mut unencodable_request_header) = match settings.extra_headers { Some(headers) => (Some(headers), None), - None => self.merged_headers( - py, - arguments, - custom_llm_provider.as_deref() == Some(AZURE_AI_PROVIDER), - )?, + None => self.merged_headers(py, arguments, filters_beta)?, }; let provider_specific_header = self.provider_specific_header( py, arguments, custom_llm_provider.as_deref(), + filters_beta, &mut unencodable_request_header, )?; let azure = match custom_llm_provider.as_deref() { Some(AZURE_AI_PROVIDER) => Some(azure_module_settings(|name| module_global(py, name))?), _ => None, }; + let bedrock = custom_llm_provider.as_deref() == Some(BEDROCK_PROVIDER); + let mut capabilities = self.capabilities(py, &model, custom_llm_provider.as_deref())?; + if bedrock { + capabilities.bedrock = + super::model_catalog::bedrock_capabilities(py, &model, argument("model_info")?)?; + } let shaping = MessagesShaping { - capabilities: self.capabilities(py, &model, custom_llm_provider.as_deref())?, + capabilities, settings: MessagesSettings { whole_sse_frames: settings.stream_cost_model.is_some(), presanitized, intercepted: self.may_intercept(py)?, unencodable_request_header, module_api_key: azure.as_ref().and_then(|azure| azure.api_key.clone()), + global_drop_params: global_drop_params(py)?, ..settings.settings }, }; @@ -587,47 +667,42 @@ impl MessagesPythonHost { Some(AZURE_AI_PROVIDER) => ENTRA_ID_PARAMS.to_vec(), _ => Vec::new(), }; - let settled = match azure { - Some(azure) => azure_settled_params(azure, argument(CALLER_TOKEN_PROVIDER_PARAM)?)?, - None => Map::new(), + let settled = match (azure, bedrock) { + (Some(azure), _) => { + azure_settled_params(azure, argument(CALLER_TOKEN_PROVIDER_PARAM)?)? + } + (None, true) => bedrock_litellm_params(py, argument)?, + (None, false) => Map::new(), }; let litellm_params = project_litellm_params(provider_fields, settled, argument, |name| { module_global(py, name) })?; - Ok( - messages_body(body, custom_llm_provider.as_deref()).and_then(|body| { - Ok(MessagesCall { - body, - api_key, - api_base, - extra_headers, - provider_specific_header, - custom_llm_provider, - litellm_params: litellm_params?, - timeout: optional_timeout(timeout), - shaping, - }) - }), - ) + Ok(messages_body(body).and_then(|body| { + Ok(MessagesCall { + body, + api_key, + api_base, + extra_headers, + provider_specific_header, + custom_llm_provider, + litellm_params: litellm_params?, + timeout: optional_timeout(timeout), + shaping, + }) + })) } fn merged_headers( &self, py: Python<'_>, arguments: &Bound<'_, PyDict>, - provider_filters_beta: bool, + filters_beta: bool, ) -> PyResult { let mut unencodable = None; let mut mapping = |name: &str| -> PyResult>> { self.view .argument(py, arguments, name)? - .map(|value| { - from_py(&ascii_headers( - &value, - &mut unencodable, - provider_filters_beta, - )?) - }) + .map(|value| from_py(&ascii_headers(&value, &mut unencodable, filters_beta)?)) .transpose() }; let headers = mapping("headers")?; @@ -640,6 +715,7 @@ impl MessagesPythonHost { py: Python<'_>, arguments: &Bound<'_, PyDict>, provider: Option<&str>, + filters_beta: bool, unencodable: &mut Option>, ) -> PyResult> { let Some(value) = self @@ -655,13 +731,9 @@ impl MessagesPythonHost { .getattr("ProviderSpecificHeaderUtils")? .call_method1("get_provider_specific_headers", (&value, provider, true)) { - ascii_headers(&scoped, unencodable, provider == AZURE_AI_PROVIDER)?; + ascii_headers(&scoped, unencodable, filters_beta)?; } - from_py(&ascii_scoped_headers( - &value, - provider == Some(AZURE_AI_PROVIDER), - )?) - .map(Some) + from_py(&ascii_scoped_headers(&value, filters_beta)?).map(Some) } fn capabilities( @@ -723,6 +795,89 @@ impl MessagesPythonHost { Err(_) => error, } } + + fn bedrock_stream_raise(&self, py: Python<'_>, raise: &BedrockStreamRaise) -> PyResult { + let (kind, arguments) = match raise { + BedrockStreamRaise::ChecksumMismatch { + expected, + calculated, + } => ( + "ChecksumMismatch", + serde_json::json!([expected, calculated]), + ), + BedrockStreamRaise::InvalidHeadersLength(length) => { + ("InvalidHeadersLength", serde_json::json!([length])) + } + BedrockStreamRaise::InvalidPayloadLength(length) => { + ("InvalidPayloadLength", serde_json::json!([length])) + } + BedrockStreamRaise::DuplicateHeader(name) => { + ("DuplicateHeader", serde_json::json!([name])) + } + BedrockStreamRaise::InvalidJson(text) => ("InvalidJson", serde_json::json!([text])), + BedrockStreamRaise::NotAnObject(chunk) => ("NotAnObject", serde_json::json!([chunk])), + }; + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("bedrock_stream_raise")? + .call1((kind, to_py(py, &arguments)?))?; + Ok(PyErr::from_value(raised)) + } + + fn mapped_error(&self, py: Python<'_>, error: Error) -> PyResult { + if let Some(raise) = litellm_error::find::(&error) { + return self.bedrock_stream_raise(py, raise); + } + if let Some((raised, raw)) = python_failure(py, &error) { + return if raw { + Ok(raised) + } else { + self.map_transport_failure(py, raised) + }; + } + if let Some(raised) = super::workload_identity::python_error(py, &error) + .or_else(|| super::workload_identity::entra_token_error(py, &error)) + { + let mapped = py + .import(ROUTE_HOST_MODULE)? + .getattr("map_token_failure")? + .call1(( + raised.value(py), + self.view.request(py)?, + self.provider(py), + self.asynchronous, + ))?; + return Ok(PyErr::from_value(mapped)); + } + if let Some(unencodable) = litellm_error::find::(&error) { + let (value, received) = match unencodable { + UnencodableHeader::Sent(value) => (value, false), + UnencodableHeader::Received(value) => (value, true), + }; + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("unencodable_header")? + .call1((PyBytes::new(py, value), received))?; + return self.map_transport_failure(py, PyErr::from_value(raised)); + } + if let Some(undecodable) = litellm_error::find::(&error) { + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("decoding_failure")? + .call1(( + undecodable.to_string(), + self.view.request(py)?, + self.provider(py), + ))?; + return Ok(PyErr::from_value(raised)); + } + route_failure(py, error, RequestMarker::Messages, |native| { + if self.stream_opened { + native.value(py).setattr(MESSAGES_STREAM_FAILURE, true)?; + } + Ok(self.map_failure(py, native)) + }) + } } impl MessagesPythonHost { @@ -839,6 +994,9 @@ impl PythonBinding for MessagesPythonHost { return Ok(answer); } let public = public_json_response(py, ROUTE_HOST_MODULE, &response)?; + py.import(ROUTE_HOST_MODULE)? + .getattr("record_guardrail")? + .call1((&public, self.view.request(py)?))?; self.safeguards.published(py, public) } @@ -859,15 +1017,12 @@ impl PythonBinding for MessagesPythonHost { py: Python<'_>, head: MessagesStreamHead, ) -> PyResult> { + self.stream_opened = true; let module = py.import(ROUTE_HOST_MODULE)?; let head = module .getattr("stream_hidden_params")? - .call1((to_py(py, &head.headers)?,))?; - if let Some(arguments) = self - .arguments - .as_ref() - .filter(|_| self.asynchronous && self.provider(py) == AZURE_AI_PROVIDER) - { + .call1((to_py(py, &head.headers)?, head.unobserved_tail.is_some()))?; + if let Some(arguments) = self.arguments.as_ref().filter(|_| self.asynchronous) { module .getattr("python_stream_types")? .call1((&head, arguments))?; @@ -896,52 +1051,15 @@ impl PythonBinding for MessagesPythonHost { } fn map_error(&self, py: Python<'_>, error: Error) -> PyResult { - if let Some((raised, raw)) = python_failure(py, &error) { - return if raw { - Ok(raised) - } else { - self.map_transport_failure(py, raised) - }; + let mapped = self.mapped_error(py, error)?; + if !self.stream_opened { + return Ok(mapped); } - if let Some(raised) = super::workload_identity::python_error(py, &error) - .or_else(|| super::workload_identity::entra_token_error(py, &error)) - { - let mapped = py - .import(ROUTE_HOST_MODULE)? - .getattr("map_token_failure")? - .call1(( - raised.value(py), - self.view.request(py)?, - self.provider(py), - self.asynchronous, - ))?; - return Ok(PyErr::from_value(mapped)); - } - if let Some(unencodable) = litellm_error::find::(&error) { - let (value, received) = match unencodable { - UnencodableHeader::Sent(value) => (value, false), - UnencodableHeader::Received(value) => (value, true), - }; - let raised = py - .import(ROUTE_HOST_MODULE)? - .getattr("unencodable_header")? - .call1((PyBytes::new(py, value), received))?; - return self.map_transport_failure(py, PyErr::from_value(raised)); - } - if let Some(undecodable) = litellm_error::find::(&error) { - let raised = py - .import(ROUTE_HOST_MODULE)? - .getattr("decoding_failure")? - .call1(( - undecodable.to_string(), - self.view.request(py)?, - self.provider(py), - ))?; - return Ok(PyErr::from_value(raised)); - } - route_failure(py, error, RequestMarker::Messages, |native| { - Ok(self.map_failure(py, native)) - }) + let described = py + .import(ROUTE_HOST_MODULE)? + .getattr("stream_failure")? + .call1((mapped.value(py), self.view.request(py)?, self.provider(py)))?; + Ok(PyErr::from_value(described)) } fn host_error(error: &PyErr) -> Error { @@ -1115,7 +1233,7 @@ mod tests { )] fn a_header_python_cannot_encode_is_held_back_unless_the_provider_filters_it( #[case] headers: &std::ffi::CStr, - #[case] provider_filters_beta: bool, + #[case] filters_beta: bool, #[case] kept: Value, #[case] unencodable: Option>, ) { @@ -1124,8 +1242,7 @@ mod tests { let headers = py.eval(headers, None, None).unwrap(); let mut held_back = None; let sent: Value = - from_py(&ascii_headers(&headers, &mut held_back, provider_filters_beta).unwrap()) - .unwrap(); + from_py(&ascii_headers(&headers, &mut held_back, filters_beta).unwrap()).unwrap(); assert_eq!((sent, held_back), (kept, unencodable)); }); } @@ -1143,6 +1260,14 @@ mod tests { json!({"model": "m", "vertex_project": "p", "vertex_ai_location": "us-east5", "vertex_credentials": {"type": "service_account"}}), Ok(json!({"model": "m", "vertex_project": "p", "vertex_ai_location": "us-east5", "vertex_credentials": {"type": "service_account"}})), )] + #[case::aws_session_tags_are_projected( + json!({"model": "m", "aws_role_name": "r", "aws_session_tags": [{"Key": "team", "Value": "genai"}]}), + Ok(json!({"model": "m", "aws_role_name": "r", "aws_session_tags": [{"Key": "team", "Value": "genai"}]})), + )] + #[case::malformed_aws_session_tags_are_a_request_error( + json!({"model": "m", "aws_session_tags": [{"Key": "team"}]}), + Err(()), + )] #[case::an_explicit_none_is_absent(json!({"model": "m", "aws_region_name": null}), Ok(json!({"model": "m"})))] #[case::a_wrong_type_is_a_request_error(json!({"model": "m", "aws_region_name": 7}), Err(()))] #[case::a_missing_model_is_a_request_error(json!({"aws_region_name": "eu-central-1"}), Err(()))] diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm/rust/crates/python-bridge/src/routes/messages/mod.rs index 2df94cf911c..fe4be84d2d3 100644 --- a/litellm/rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm/rust/crates/python-bridge/src/routes/messages/mod.rs @@ -1,6 +1,7 @@ mod agentic; mod answer; mod host; +mod model_catalog; mod modify_params; mod pre_request; mod requests_reason; diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/model_catalog.rs b/litellm/rust/crates/python-bridge/src/routes/messages/model_catalog.rs new file mode 100644 index 00000000000..c2100c068be --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/routes/messages/model_catalog.rs @@ -0,0 +1,62 @@ +use std::sync::{Arc, Mutex, PoisonError}; + +use litellm_host_python::json_from_py; +use litellm_llms::{ + anthropic::model_capabilities::bedrock_model_capabilities, + base_llm::messages::context::BedrockCapabilities, +}; +use litellm_model_catalog::Catalog; +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{PyBool, PyMapping}, +}; + +use super::host::MODEL_CAPABILITIES_MODULE; + +type Version = (usize, usize, u64); + +static LIVE: Mutex)>> = Mutex::new(None); + +fn live_catalog(py: Python<'_>) -> PyResult> { + let known = LIVE.lock().unwrap_or_else(PoisonError::into_inner).clone(); + let (version, document): (Version, Option>) = py + .import(MODEL_CAPABILITIES_MODULE)? + .getattr("model_cost_document")? + .call1((known.as_ref().map(|(version, _)| *version),))? + .extract()?; + let Some(document) = document else { + return known + .map(|(_, catalog)| catalog) + .ok_or_else(|| PyValueError::new_err("the model map came back without a document")); + }; + let body = serde_json::to_vec(&json_from_py(&document)?) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let catalog = Arc::new( + py.detach(|| Catalog::parse(&body)) + .map_err(|error| PyValueError::new_err(error.to_string()))? + .value, + ); + *LIVE.lock().unwrap_or_else(PoisonError::into_inner) = Some((version, Arc::clone(&catalog))); + Ok(catalog) +} + +pub(super) fn bedrock_capabilities( + py: Python<'_>, + model: &str, + model_info: Option>, +) -> PyResult { + let model_info = model_info.and_then(|info| info.cast_into::().ok()); + let declared = |name: &str| { + model_info + .as_ref() + .and_then(|info| info.get_item(name).ok()) + .and_then(|value| value.cast_into::().ok()) + .map(|flag| flag.is_true()) + }; + Ok(bedrock_model_capabilities( + &*live_catalog(py)?, + model, + &declared, + )) +} diff --git a/litellm/rust/crates/python-bridge/src/secrets/local.rs b/litellm/rust/crates/python-bridge/src/secrets/local.rs new file mode 100644 index 00000000000..701dc5e6f5b --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/secrets/local.rs @@ -0,0 +1,238 @@ +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use futures_util::future::{BoxFuture, FutureExt, try_join_all}; +use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup}; +use litellm_secret_value::SecretValue; +use litellm_secrets::{ + Error, + source::{SecretSource, Secrets}, +}; + +const ENVIRONMENT_PREFIX: &str = "os.environ/"; +const OIDC_PREFIX: &str = "oidc/"; + +/// Python's `get_secret_str` while no secret manager is readable: names come from the process +/// environment, read in Rust when the route looks them up, and `oidc/` references go to `python`. +pub(super) struct LocalSecrets { + python: Arc, +} + +impl LocalSecrets { + pub(super) fn new(python: Arc) -> Self { + Self { python } + } +} + +fn python_name(name: &str) -> Cow<'_, str> { + if name.starts_with(ENVIRONMENT_PREFIX) { + Cow::Owned(name.replace(ENVIRONMENT_PREFIX, "")) + } else { + Cow::Borrowed(name) + } +} + +fn is_oidc(name: &str) -> bool { + python_name(name).starts_with(OIDC_PREFIX) +} + +fn environment_str(name: &str) -> Option { + let name = python_name(name); + if name.contains(['=', '\0']) { + return None; + } + std::env::var(name.as_ref()) + .ok() + .filter(|value| parse_str_bool(value).is_none()) + .map(SecretValue::new) +} + +impl SecretSource for LocalSecrets { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> BoxFuture<'a, Result, Error>> { + if is_oidc(name) { + return self.python.get_secret_str(name); + } + futures_util::future::ready(Ok(environment_str(name))).boxed() + } + + fn resolve<'a>(&'a self, names: &'a [&str]) -> BoxFuture<'a, Result> { + Box::pin(async move { + let (oidc, environment): (Vec<&str>, Vec<&str>) = + names.iter().copied().partition(|name| is_oidc(name)); + let oidc = try_join_all(oidc.into_iter().map(|name| async move { + self.python + .get_secret_str(name) + .await + .map(|value| (name.to_owned(), value)) + })) + .await? + .into_iter() + .collect(); + Ok(Arc::new(LocalLookup { + environment: environment.into_iter().map(str::to_owned).collect(), + oidc, + }) as Secrets) + }) + } +} + +struct LocalLookup { + environment: HashSet, + oidc: HashMap>, +} + +impl Lookup for LocalLookup { + fn get(&self, name: &str) -> Option { + let value = if self.environment.contains(name) { + environment_str(name) + } else { + self.oidc.get(name).cloned().flatten() + }; + value.map(|value| value.expose().to_owned()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use futures_util::future::{BoxFuture, FutureExt}; + use litellm_secret_value::SecretValue; + use litellm_secrets::{Error, source::SecretSource}; + use rstest::rstest; + + use super::LocalSecrets; + + #[derive(Default)] + struct Recording(Mutex>); + + impl Recording { + fn reads(&self) -> Vec { + self.0.lock().unwrap().clone() + } + } + + impl SecretSource for Recording { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> BoxFuture<'a, Result, Error>> { + self.0.lock().unwrap().push(name.to_owned()); + futures_util::future::ready(Ok(Some(SecretValue::new(format!("python-{name}"))))) + .boxed() + } + } + + fn local() -> (LocalSecrets, Arc) { + let python = Arc::new(Recording::default()); + ( + LocalSecrets::new(Arc::clone(&python) as Arc), + python, + ) + } + + fn set(name: &str, value: Option<&str>) { + match value { + Some(value) => unsafe { std::env::set_var(name, value) }, + None => unsafe { std::env::remove_var(name) }, + } + } + + #[rstest] + #[case::plain( + "LITELLM_LOCAL_PLAIN", + "LITELLM_LOCAL_PLAIN", + Some("value"), + Some("value") + )] + #[case::empty("LITELLM_LOCAL_EMPTY", "LITELLM_LOCAL_EMPTY", Some(""), Some(""))] + #[case::unset("LITELLM_LOCAL_UNSET", "LITELLM_LOCAL_UNSET", None, None)] + #[case::reference( + "LITELLM_LOCAL_REFERENCE", + "os.environ/LITELLM_LOCAL_REFERENCE", + Some("value"), + Some("value") + )] + #[case::every_marker_removed( + "LITELLM_LOCAL_TWICE", + "os.environ/os.environ/LITELLM_LOCAL_TWICE", + Some("value"), + Some("value") + )] + #[case::marker_inside_only_when_prefixed( + "LITELLM_LOCAL_INNER", + "LITELLM_LOCAL_os.environ/INNER", + Some("value"), + None + )] + #[case::boolean_true("LITELLM_LOCAL_TRUE", "LITELLM_LOCAL_TRUE", Some(" TRUE\n"), None)] + #[case::boolean_false("LITELLM_LOCAL_FALSE", "LITELLM_LOCAL_FALSE", Some("False"), None)] + #[case::boolean_like("LITELLM_LOCAL_ONE", "LITELLM_LOCAL_ONE", Some("1"), Some("1"))] + #[case::equals_in_name("LITELLM_LOCAL_EQ", "LITELLM_LOCAL_EQ=x", Some("x=y"), None)] + #[tokio::test] + async fn reads_what_get_secret_str_reads_from_the_environment( + #[case] variable: &str, + #[case] name: &str, + #[case] value: Option<&str>, + #[case] expected: Option<&str>, + ) { + set(variable, value); + let (secrets, python) = local(); + + let read = secrets.get_secret_str(name).await.unwrap(); + let resolved = secrets.resolve(&[name]).await.unwrap().get(name); + set(variable, None); + + assert_eq!(read.as_ref().map(SecretValue::expose), expected); + assert_eq!(resolved.as_deref(), expected); + assert!(python.reads().is_empty()); + } + + #[rstest] + #[case::direct("oidc/env/LITELLM_LOCAL_TOKEN")] + #[case::referenced("os.environ/oidc/env/LITELLM_LOCAL_TOKEN")] + #[tokio::test] + async fn oidc_references_are_left_to_python(#[case] name: &str) { + let (secrets, python) = local(); + + let read = secrets.get_secret_str(name).await.unwrap(); + let resolved = secrets + .resolve(&[name, "LITELLM_LOCAL_BESIDE"]) + .await + .unwrap(); + + let expected = format!("python-{name}"); + assert_eq!( + read.as_ref().map(SecretValue::expose), + Some(expected.as_str()) + ); + assert_eq!(resolved.get(name).as_deref(), Some(expected.as_str())); + assert_eq!(python.reads(), vec![name.to_owned(), name.to_owned()]); + } + + #[tokio::test] + async fn resolved_names_are_read_when_looked_up_and_others_are_absent() { + let name = "LITELLM_LOCAL_LAZY"; + let other = "LITELLM_LOCAL_NOT_RESOLVED"; + set(name, None); + set(other, Some("present")); + let (secrets, python) = local(); + + let resolved = secrets.resolve(&[name]).await.unwrap(); + set(name, Some("set-after-resolve")); + let looked_up = resolved.get(name); + let unrequested = resolved.get(other); + set(name, None); + set(other, None); + + assert_eq!(looked_up.as_deref(), Some("set-after-resolve")); + assert_eq!(unrequested, None); + assert!(python.reads().is_empty()); + } +} diff --git a/litellm/rust/crates/python-bridge/src/secrets/mod.rs b/litellm/rust/crates/python-bridge/src/secrets/mod.rs index c77fd16bebb..dd0b6944b30 100644 --- a/litellm/rust/crates/python-bridge/src/secrets/mod.rs +++ b/litellm/rust/crates/python-bridge/src/secrets/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod callback; pub(crate) mod config; mod error; +mod local; mod mutation; mod operations; mod provider; @@ -9,6 +10,7 @@ pub(crate) mod resolved; pub(crate) mod runtime; pub(crate) mod seeded; mod vault; +mod web_identity; use std::sync::Arc; @@ -16,33 +18,98 @@ use std::sync::Arc; pub(crate) use error::read_error; pub(crate) use error::{ManagerError, python_error}; use litellm_secrets::source::{EnvironmentSecrets, SecretSource}; +use local::LocalSecrets; use pyo3::prelude::*; use python::PythonSecrets; use resolved::ResolvedSecrets; +pub(crate) use web_identity::HostWebIdentityTokens; -use crate::{coercion::FieldSpec, python_settings::PythonSettings}; +use crate::{ + coercion::FieldSpec, + python_settings::{PythonSettings, Snapshot}, +}; const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager"; +const READABLE: FieldSpec = FieldSpec::new("readable", |field| field.schema_bool()); const NATIVE: FieldSpec = FieldSpec::new("native", |field| field.schema_bool()); -/// Where a Rust route reads provider secrets from. Python's `get_secret_str` when the settings -/// projection reports no configured secret manager, otherwise the resolved manager: the native -/// backend when the configured client captures one, the Python callback when it does not. A bare -/// extension module without the litellm package reads the process environment. +#[derive(Debug, PartialEq, Eq)] +enum Selection { + Local, + Native, + Python, +} + +fn select(snapshot: &Snapshot<'_>) -> PyResult { + Ok(if !snapshot.read(&READABLE)? { + Selection::Local + } else if snapshot.read(&NATIVE)? { + Selection::Native + } else { + Selection::Python + }) +} + +/// Where a Rust route reads provider secrets from, decided per call from the settings projection. +/// With no readable secret manager Python's `get_secret_str` only reads the environment, so the +/// route reads it in Rust. A readable manager is the native backend when the configured client +/// captures one, otherwise every read goes through Python's `get_secret_str`. A bare extension +/// module without the litellm package reads the process environment. pub(crate) fn source(py: Python<'_>) -> PyResult> { let Some(snapshot) = PythonSettings::SecretManager.read_or_unset(py)? else { return Ok(Arc::new(EnvironmentSecrets::python_compatible( crate::http::host_client(py, litellm_http::ClientVariant::Provider)?, ))); }; - if snapshot.read(&NATIVE)? { - let context = litellm_host_python::PythonContext::capture(py)?; - let client = crate::http::host_client(py, litellm_http::ClientVariant::Provider)?; - return Ok(Arc::new(ResolvedSecrets::new( - config::read(py)?, - context, - client, - ))); + match select(&snapshot)? { + Selection::Local => Ok(Arc::new(LocalSecrets::new(Arc::new(PythonSecrets::new( + py, + )?)))), + Selection::Native => { + let context = litellm_host_python::PythonContext::capture(py)?; + let client = crate::http::host_client(py, litellm_http::ClientVariant::Provider)?; + Ok(Arc::new(ResolvedSecrets::new( + config::read(py)?, + context, + client, + ))) + } + Selection::Python => Ok(Arc::new(PythonSecrets::new(py)?)), + } +} + +#[cfg(test)] +mod tests { + use pyo3::{prelude::*, types::PyDict}; + use rstest::rstest; + + use super::{Selection, select}; + use crate::python_settings::PythonSettings; + + #[rstest] + #[case::no_manager(false, false, Selection::Local)] + #[case::unreadable_native(false, true, Selection::Local)] + #[case::python_manager(true, false, Selection::Python)] + #[case::native_manager(true, true, Selection::Native)] + fn the_projection_selects_where_secrets_are_read( + #[case] readable: bool, + #[case] native: bool, + #[case] expected: Selection, + ) { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("readable", readable).unwrap(); + locals.set_item("native", native).unwrap(); + let value = py + .eval( + c"__import__('types').SimpleNamespace(readable=readable, native=native)", + None, + Some(&locals), + ) + .unwrap(); + let selected = select(&PythonSettings::SecretManager.snapshot(value)).unwrap(); + assert_eq!(selected, expected); + }); } - Ok(Arc::new(PythonSecrets::new(py)?)) } diff --git a/litellm/rust/crates/python-bridge/src/secrets/seeded.rs b/litellm/rust/crates/python-bridge/src/secrets/seeded.rs index 596a562d89b..9d934d4f1f5 100644 --- a/litellm/rust/crates/python-bridge/src/secrets/seeded.rs +++ b/litellm/rust/crates/python-bridge/src/secrets/seeded.rs @@ -4,8 +4,12 @@ use std::{ }; use futures_util::future::{BoxFuture, FutureExt}; +use litellm_core_utils::settings::Lookup; use litellm_secret_value::SecretValue; -use litellm_secrets::{Error, source::SecretSource}; +use litellm_secrets::{ + Error, + source::{SecretSource, Secrets}, +}; /// A call's secret source that answers the secrets the host already read for the call with the /// values it read, so a secret Python reads once is not read a second time. @@ -49,6 +53,39 @@ impl SecretSource for SeededSecrets { None => self.source.get_secret_str(name), } } + + fn resolve<'a>(&'a self, names: &'a [&str]) -> BoxFuture<'a, Result> { + let (seeded, unseeded) = { + let read = self.read.lock().unwrap_or_else(PoisonError::into_inner); + let (seeded, unseeded): (Vec<&str>, Vec<&str>) = names + .iter() + .copied() + .partition(|name| read.contains_key(*name)); + let seeded: HashMap> = seeded + .into_iter() + .map(|name| (name.to_owned(), read.get(name).cloned().flatten())) + .collect(); + (seeded, unseeded) + }; + Box::pin(async move { + let source = self.source.resolve(&unseeded).await?; + Ok(Arc::new(SeededLookup { seeded, source }) as Secrets) + }) + } +} + +struct SeededLookup { + seeded: HashMap>, + source: Secrets, +} + +impl Lookup for SeededLookup { + fn get(&self, name: &str) -> Option { + match self.seeded.get(name) { + Some(value) => value.as_ref().map(|value| value.expose().to_owned()), + None => self.source.get(name), + } + } } #[cfg(test)] @@ -98,4 +135,49 @@ mod tests { ); assert_eq!(source.0.load(Ordering::SeqCst), 1); } + + #[tokio::test] + async fn resolve_answers_seeded_names_and_leaves_the_rest_to_the_source() { + let source = Arc::new(Counting::default()); + let secrets = SeededSecrets::new(Arc::clone(&source) as Arc); + secrets.seed([ + ( + "AZURE_API_BASE".to_owned(), + Some("http://seeded".to_owned()), + ), + ("UNSET".to_owned(), None), + ]); + + let resolved = secrets + .resolve(&["AZURE_API_BASE", "UNSET", "OTHER"]) + .await + .unwrap(); + + assert_eq!( + resolved.get("AZURE_API_BASE").as_deref(), + Some("http://seeded") + ); + assert_eq!(resolved.get("UNSET"), None); + assert_eq!(resolved.get("OTHER").as_deref(), Some("source-OTHER")); + assert_eq!(resolved.get("NOT_REQUESTED"), None); + assert_eq!(source.0.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn resolve_keeps_a_lazy_source_lazy() { + let name = "LITELLM_SEEDED_LAZY"; + unsafe { std::env::remove_var(name) }; + let python = Arc::new(Counting::default()); + let secrets = SeededSecrets::new(Arc::new(crate::secrets::local::LocalSecrets::new( + Arc::clone(&python) as Arc, + ))); + + let resolved = secrets.resolve(&[name]).await.unwrap(); + unsafe { std::env::set_var(name, "set-after-resolve") }; + let looked_up = resolved.get(name); + unsafe { std::env::remove_var(name) }; + + assert_eq!(looked_up.as_deref(), Some("set-after-resolve")); + assert_eq!(python.0.load(Ordering::SeqCst), 0); + } } diff --git a/litellm/rust/crates/python-bridge/src/secrets/web_identity.rs b/litellm/rust/crates/python-bridge/src/secrets/web_identity.rs new file mode 100644 index 00000000000..3dbe9700538 --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/secrets/web_identity.rs @@ -0,0 +1,30 @@ +use litellm_auth_aws::{WebIdentityTokenFuture, WebIdentityTokenSource}; +use litellm_error::Source; +use litellm_host_python::{PythonContext, attach_blocking}; +use pyo3::prelude::*; + +use super::error::external_error; + +#[derive(Debug)] +pub(crate) struct HostWebIdentityTokens; + +impl WebIdentityTokenSource for HostWebIdentityTokens { + fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a> { + Box::pin(async move { + let context = Python::attach(|py| { + PythonContext::empty(py).map_err(|error| external_error(py, error)) + }) + .map_err(Source::new)?; + let source = match attach_blocking(context, |py| { + super::source(py).map_err(|error| external_error(py, error)) + }) + .await + { + Ok(source) => source, + Err(error) => Err(Python::attach(|py| external_error(py, error))), + } + .map_err(Source::new)?; + source.get_secret_str(reference).await.map_err(Source::new) + }) + } +} diff --git a/litellm/rust/crates/router-types/tests/litellm_params.rs b/litellm/rust/crates/router-types/tests/litellm_params.rs index c5c0d16ee8f..fb564e2d884 100644 --- a/litellm/rust/crates/router-types/tests/litellm_params.rs +++ b/litellm/rust/crates/router-types/tests/litellm_params.rs @@ -126,7 +126,10 @@ fn a_param_of_the_wrong_type_is_rejected(#[case] value: Value) { #[rstest] fn fields_fill_the_model_the_credentials_and_every_group() { let filled: Value = LitellmParams::fields() - .map(|name| (name.to_string(), Value::from(name))) + .map(|name| match name { + AwsParams::SESSION_TAGS => (name.to_string(), json!([{"Key": "k", "Value": "v"}])), + _ => (name.to_string(), Value::from(name)), + }) .collect::>() .into(); let typed = params(filled); diff --git a/litellm/rust/crates/secrets-aws/src/auth.rs b/litellm/rust/crates/secrets-aws/src/auth.rs index 6bec8ad96a1..0e8923dc599 100644 --- a/litellm/rust/crates/secrets-aws/src/auth.rs +++ b/litellm/rust/crates/secrets-aws/src/auth.rs @@ -49,6 +49,8 @@ impl Credentials { profile_name: settings.aws_profile_name.clone(), web_identity_token: settings.aws_web_identity_token.as_ref().cloned(), sts_endpoint: settings.aws_sts_endpoint.clone(), + session_tags: None, + session_policy: None, }, environment, } diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 3e5b9977df6..089d8476d61 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -30,9 +30,14 @@ class Target(str, Enum): MESSAGES = "messages" MESSAGES_PASSTHROUGH = "messages_passthrough" """A deployment's own /v1/messages, which Python's OpenAI-compatible config forwards untranslated.""" + BEDROCK_INVOKE = "bedrock_invoke" + """Claude on Bedrock InvokeModel, an Anthropic Messages body behind the InvokeModel API.""" RESPONSES = "responses" +NATIVE_MESSAGES_TARGETS: Final = frozenset({Target.MESSAGES, Target.MESSAGES_PASSTHROUGH, Target.BEDROCK_INVOKE}) + + class Feature(str, Enum): MOCK_RESPONSE = "mock_response" """mock_response short-circuits before any provider call.""" @@ -202,6 +207,12 @@ RULES: Final[Rules] = ( ), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=MESSAGES_PYTHON_FEATURES), RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED, providers=frozenset({"anthropic", "azure_ai"})), + RouteRule( + Route.MESSAGES, + Rollout.RUST_REQUIRED, + providers=frozenset({"bedrock"}), + targets=frozenset({Target.BEDROCK_INVOKE}), + ), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY, features=RESPONSES_PYTHON_FEATURES), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index 7147e99ac27..10cfa286bd3 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -707,18 +707,26 @@ def _log_superseded_stream_failure( verbose_proxy_logger.error("Error scheduling stream failure logging: %s", e) +def _relayed(head: object) -> bool: + from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead + + return isinstance(head, MessagesStreamHead) and head.relayed + + def stream_failure( logger: Logging, endpoint_type: str, request_body: dict[str, object], chunks: list[bytes], error: Exception, + head: object, ) -> Coroutine[object, object, None]: from litellm._logging import verbose_proxy_logger from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType - verbose_proxy_logger.error("Error in chunk_processor: %s", error) + if not _relayed(head): + verbose_proxy_logger.error("Error in chunk_processor: %s", error) # Python's stream fails before its success handler records `cache_hit`; only a cache hit # stamps it earlier. if logger.model_call_details.get("cache_hit") is not True: diff --git a/litellm/rust_bridge/host/model_capabilities.py b/litellm/rust_bridge/host/model_capabilities.py index 00a00b9e79b..a26dae0ede0 100644 --- a/litellm/rust_bridge/host/model_capabilities.py +++ b/litellm/rust_bridge/host/model_capabilities.py @@ -1,11 +1,14 @@ from __future__ import annotations +import json from collections.abc import Mapping from functools import lru_cache from typing import Final, TypeAlias import litellm -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils import fallback_generalizations +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping +from litellm.litellm_core_utils.get_model_cost_map import FALLBACK_GENERALIZATIONS_KEY from litellm.litellm_core_utils.model_cost_table import ModelCostVersion, model_cost_version @@ -22,6 +25,7 @@ class _Held: return isinstance(other, _Held) and other.value is self.value +_CatalogVersion: TypeAlias = tuple[int, int, int] _Fingerprint: TypeAlias = tuple[str, str | None, _Held, ModelCostVersion, int, tuple[_Held, ...]] @@ -95,3 +99,18 @@ def _capabilities(fingerprint: _Fingerprint, resolved_model: str, provider: str) "supports_mid_conversation_system": supports("supports_mid_conversation_system"), "effort_tiers": {level: tier(level) for level in ("minimal", "low", "medium", "high", "xhigh", "max")}, } + + +def model_cost_document(known: _CatalogVersion | None) -> tuple[_CatalogVersion, Mapping[str, object] | None]: + """The live model map and its active fallback rules as one document, which the native catalog reads, or + ``None`` when the map is still at version ``known``.""" + version: Final = model_cost_version() + current: Final = (version.table, version.size, version.writes) + if current == known: + return current, None + rows: Final = as_str_mapping(litellm.model_cost) or {} # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the model map is an untyped dict of rows + rules: Final = as_object_list(fallback_generalizations.get_fallback_generalization_rules()) or [] # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the rule registry keeps the raw untyped list + document: Final = json.loads( + json.dumps({**rows, FALLBACK_GENERALIZATIONS_KEY: {"rules": list(rules)}}, default=str) + ) + return current, as_str_mapping(document) diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 4a890c8ac63..cda1ab5b7ee 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -9,6 +9,7 @@ import http import http.client import inspect import ipaddress +import json import os import socket import ssl @@ -18,12 +19,17 @@ from typing import Final, TypeAlias, TypeVar, cast # noqa: TID251 # narrows na from urllib.parse import SplitResult, urlsplit import httpx +from aiohttp import ClientPayloadError from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger from litellm.caching.caching_handler import LLMCachingHandler +from litellm.constants import ( + ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS, + ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import exception_mapping_utils from litellm.litellm_core_utils.core_helpers import ( @@ -42,7 +48,7 @@ from litellm.litellm_core_utils.model_latest_aliases import is_model_latest_alia from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout -from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.anthropic.common_utils import ( AnthropicError, AnthropicModelInfo, @@ -70,7 +76,13 @@ from litellm.llms.anthropic.messages.native.handler import ( _with_deployment_output_effort as _deployment_output_effort, # pyright: ignore[reportPrivateUsage] # the handler's own effort rounding ) from litellm.llms.anthropic.messages.native.interceptors import MessagesInterceptor, get_messages_interceptors -from litellm.llms.anthropic.messages.native.utils import normalize_native_messages, prepare_native_messages +from litellm.llms.anthropic.messages.native.utils import ( + continues_message_thread, + message_thread_not_found_body, + message_thread_unsupported_body, + normalize_native_messages, + prepare_native_messages, +) from litellm.llms.anthropic.messages.safeguards import ( SafeguardsEvaluator, StreamedSafeguardResults, @@ -81,6 +93,11 @@ from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enab from litellm.llms.anthropic.wif import get_anthropic_wif_token from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.common_utils import ( + ensure_bedrock_anthropic_messages_tool_names, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict + normalize_custom_field_on_tools, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict + normalize_tool_input_schema_types_for_bedrock_invoke, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict +) from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # its params dict is untyped @@ -152,6 +169,8 @@ class MessagesSettings: ascii_response_headers: bool = False joined_reads: bool = False ascii_request_headers: bool = True + detached_drain_timeout: float | None = None + detached_drain_limit: int | None = None def response(value: Mapping[str, object]) -> AnthropicMessagesResponse: @@ -191,15 +210,100 @@ def _anthropic_metadata(params: Mapping[str, object]) -> Mapping[str, object]: def _body_fields(request: Mapping[str, object]) -> Mapping[str, object]: params: Final = _anthropic_metadata( - known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request)), request) + known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request), _dropped_thread(request)), request) ) if deployment_reasoning_effort_levels(request.get("model_info")) is None: return params return _with_deployment_effort(params, request) +def _thread_provider(request: Mapping[str, object]) -> str | None: + """The provider of a call carrying a thread its Messages config cannot serve, as Python's handler + resolves it, or None when the call carries no thread or its config serves threads.""" + if request.get("thread") is None: + return None + model, provider, _, _ = _requested_provider( + str(request["model"]), optional_str(request.get("custom_llm_provider")), _handler_params(request) + ) + config: Final = native_messages_config( + model, provider, request.get("model_info"), optional_str(request.get("api_base")) + ) + if config is not None and config.supports_message_threads(): + return None + return str(provider) + + +def _dropped_thread(request: Mapping[str, object]) -> tuple[str, ...]: + return () if _thread_provider(request) is None else ("thread",) + + +def _thread_refusal(thread: object, provider: str, drop_params: object) -> AnthropicError | None: + if continues_message_thread(thread): + return AnthropicError( + status_code=404, + message=message_thread_not_found_body( + f"message threads are not supported for provider {provider}, " + "so this thread cannot be continued. Send the full conversation to start over" + ), + ) + if not drops_unsupported_params(drop_params): + return AnthropicError( + status_code=400, + message=message_thread_unsupported_body( + f"message threads are not supported for provider {provider}. " + "Send the request without thread, or set drop_params to serve it without a server-side thread" + ), + ) + verbose_logger.warning( + "Dropping thread for provider %s: message threads are not supported there, " + "so the request is served without a server-side thread", + provider, + ) + return None + + +def _refuse_unsupported_thread(request: Mapping[str, object], asynchronous: bool) -> None: + """Python's handler refuses a thread its provider cannot serve, or drops it under drop_params.""" + provider: Final = _thread_provider(request) + if provider is None: + return + refusal: Final = _thread_refusal(request.get("thread"), provider, request.get("drop_params")) + if refusal is None: + return + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) + try: + raise refusal + except AnthropicError as provider_error: + _exception_type( + model=str(request["model"]), + custom_llm_provider=provider, + original_exception=provider_error, + extra_kwargs={name: value for name, value in request.items() if name not in _ENTRY_PARAMS}, + ) + raise + + +def _shape_bedrock_tools(tools: object, model: str) -> None: + """Python's Bedrock invoke transform shapes the caller's own tool dicts in place, so its logs show them shaped.""" + if not isinstance(tools, list): + return + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( + AmazonAnthropicClaudeMessagesConfig, + ) + + body: Final[dict[str, object]] = {"tools": tools} # mutable-ok: the Bedrock transform helpers shape it in place + AmazonAnthropicClaudeMessagesConfig()._remove_ttl_from_cache_control( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # the transform's own in-place step + anthropic_messages_request=body, model=model + ) + normalize_custom_field_on_tools(body) + normalize_tool_input_schema_types_for_bedrock_invoke(body) + ensure_bedrock_anthropic_messages_tool_names(body) + + def body_params(request: Mapping[str, object], asynchronous: bool, hooked: bool = False) -> Mapping[str, object]: fields: Final = _body_fields(request) + if _resolved_provider(request) == "bedrock": + _shape_bedrock_tools(fields.get("tools"), str(request["model"]).removeprefix("bedrock/")) if not detectors.cache_control_injection(NativeCall(args=(), kwargs=request, base={})): return {**fields, "messages": request["messages"]} if hooked else fields messages, system, tools = _prompt_cached(request, asynchronous, hooked) @@ -493,12 +597,18 @@ def answer(owner: MessagesInterceptor, request: Mapping[str, object], stream: bo ) -def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: +def stream_hidden_params(headers: Sequence[tuple[str, str]], relayed: bool = False) -> Mapping[str, object]: + """The stream's head. `relayed` says the stream runs through Python's native Messages relay, + `BaseAnthropicMessagesStreamingIterator.async_sse_wrapper`, rather than its pass-through.""" from litellm.llms.anthropic.messages.native.streaming_iterator import ( anthropic_messages_stream_hidden_params, ) - return MessagesStreamHead(anthropic_messages_stream_hidden_params(callbacks_legacy_python.wire_headers(headers))) + head: Final = MessagesStreamHead( + anthropic_messages_stream_hidden_params(callbacks_legacy_python.wire_headers(headers)) + ) + head.relayed = relayed + return head async def _unrequested_stream_delivered( @@ -557,6 +667,7 @@ def unrequested_stream( _UPSTREAM_ARGS: Final = TypeAdapter(tuple[int, str]) _UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]]) +_INVOCATION_METRICS: Final = "amazon-bedrock-invocationMetrics" _BODY_CUT: Final = TypeAdapter(tuple[int, int | None, bool]) _ERROR_CLASS: TypeAlias = Callable[..., Exception] _validate_metadata: Final = cast( # cast-ok: the handler's untyped metadata validator @@ -601,12 +712,18 @@ def _rejection(error: Exception, request: Mapping[str, object], request_provider provider_rejection: Final = getattr(error, "messages_provider_rejection", None) if isinstance(provider_rejection, str): _log_resolved_call(request, request_provider) - _flag_azure_ai_call(request, request_provider, asynchronous) - return _public_provider_error( - _config_error(400, provider_rejection, (), request, request_provider), error, request, request_provider + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) + rejection: Final = _provider_rejection( + provider_rejection, + getattr(error, "messages_provider_bad_request", False) is True, + request, + request_provider, ) + if isinstance(rejection, litellm.BadRequestError): + return rejection + return _public_provider_error(rejection, error, request, request_provider) if getattr(error, "messages_unsupported_param", False): - _flag_azure_ai_call(request, request_provider, asynchronous) + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) return litellm.UnsupportedParamsError(message=str(error)) return litellm.BadRequestError( message=str(error), @@ -615,6 +732,23 @@ def _rejection(error: Exception, request: Mapping[str, object], request_provider ) +def _provider_rejection( + text: str, bad_request: bool, request: Mapping[str, object], request_provider: str +) -> Exception: + """The Bedrock invoke config raises its own `BadRequestError`, which other configs' HTTP handler + maps into their error class by its text.""" + if not bad_request: + return _config_error(400, text, (), request, request_provider) + raised: Final = litellm.BadRequestError( + message=text, + model=str(request["model"]).removeprefix(f"{request_provider}/"), + llm_provider=request_provider, + ) + if request_provider == "bedrock": + return raised + return _config_error(400, str(raised), (), request, request_provider) + + def _timeout_text(phase: str, request: Mapping[str, object]) -> str: """`litellm.Timeout`'s text as Python's HTTP client raises it for the same timeout.""" seconds: Final = request.get("timeout") @@ -652,21 +786,115 @@ def _body_cut(received: int, expected: int | None, reset: bool) -> httpx.Transpo f"Not enough data to satisfy content length header (received {received} of {expected} bytes)." ) ) - text: Final = f"Response payload is not completed: {payload!r}" - if not reset: - return httpx.ReadError(text) - return httpx.ReadError(f"{text}. {ConnectionResetError(errno.ECONNRESET, os.strerror(errno.ECONNRESET))!r}") + completed: Final = f"Response payload is not completed: {payload!r}" + text: Final = ( + f"{completed}. {ConnectionResetError(errno.ECONNRESET, os.strerror(errno.ECONNRESET))!r}" + if reset + else completed + ) + payload_error: Final = ClientPayloadError(text) + payload_error.__cause__ = payload + read_error: Final = httpx.ReadError(text) + read_error.__cause__ = payload_error + read_error.__context__ = payload_error + return read_error + + +@dataclass(frozen=True, slots=True) +class _ProviderCall: + config: BaseAnthropicMessagesConfig + model: str + params: dict[str, object] # mutable-ok: the config hooks take a params dict + + +def _provider_call(request: Mapping[str, object]) -> _ProviderCall | None: + """The config, model and params Python's handler hands its provider config for the call.""" + model, provider, _, _ = _requested_provider( + str(request["model"]), optional_str(request.get("custom_llm_provider")), _handler_params(request) + ) + config: Final = native_messages_config( + model, provider, request.get("model_info"), optional_str(request.get("api_base")) + ) + if config is None: + return None + return _ProviderCall(config=config, model=model, params=dict(_python_handler_params(request, provider))) + + +def _complete_url(call: _ProviderCall, request: Mapping[str, object]) -> str: + return call.config.get_complete_url( # pyright: ignore[reportUnknownMemberType] # the config's untyped params dicts + api_base=optional_str(request.get("api_base")), + api_key=optional_str(request.get("api_key")), + model=call.model, + optional_params=call.params, + litellm_params=call.params, + stream=request.get("stream") is True, + ) + + +def _bedrock_url(request: Mapping[str, object]) -> str | None: + """The URL Python's bedrock Messages config builds for the call.""" + call: Final = _provider_call(request) + return None if call is None else _complete_url(call, request) + + +def _signing_failure(kind: str) -> Exception | None: + """The exception Python's AWS signing raises for a credential failure whose outcome needs no STS, + IMDS or secret manager call. botocore's environment provider raises `PartialCredentialsError` + for an access key without its secret, where Rust's default chain finds nothing.""" + from botocore.exceptions import NoCredentialsError + + from litellm.llms.bedrock.base_aws_llm import AwsAuthError + + if kind == "web_identity_reference": + return AwsAuthError(status_code=400, message="Invalid web identity token reference.") + if kind == "missing_web_identity_token": + return AwsAuthError(status_code=401, message="OIDC token could not be retrieved from secret manager.") + if kind == "no_credentials" and not os.environ.get("AWS_ACCESS_KEY_ID"): + return NoCredentialsError() + return None + + +def _bedrock_credential_failure(error: Exception, request: Mapping[str, object]) -> Exception | None: + """The exception Python's bedrock config raises while signing a call whose credentials Rust + could not acquire, from botocore's `NoCredentialsError` to an STS or `AwsAuthError` failure.""" + kind: Final = getattr(error, "messages_credential_failure", None) + if not isinstance(kind, str): + return None + known: Final = _signing_failure(kind) + if known is not None: + return _raised(known) + call: Final = _provider_call(request) + if call is None: + return None + try: + call.config.sign_request( # pyright: ignore[reportUnknownMemberType] # the config's untyped params dicts + headers={}, + optional_params=call.params, + request_data={}, + api_base=_complete_url(call, request), + api_key=optional_str(request.get("api_key")), + model=call.model, + stream=request.get("stream") is True, + fake_stream=False, + ) + except Exception as python_failure: # noqa: BLE001 # the exception Python's signing raises is the answer + return python_failure + return None def _sent_base(request: Mapping[str, object], request_provider: str, secrets_read: _SecretsRead) -> str | None: """The base the call was sent to: azure_ai's falls back to the `AZURE_API_BASE` the host read - to send it, and anthropic's to its default.""" + to send it, anthropic's to its default and bedrock's to the URL its config builds.""" api_base: Final = request.get("api_base") + if isinstance(api_base, str) and (api_base or request_provider != "azure_ai"): + return api_base if request_provider == "anthropic": - return api_base if isinstance(api_base, str) else anthropic_pass_through_api_base() - if request_provider == "azure_ai" and not api_base: + return anthropic_pass_through_api_base() + if request_provider == "azure_ai": return dict(secrets_read).get("AZURE_API_BASE") - return api_base if isinstance(api_base, str) else None + if request_provider == "bedrock": + return _bedrock_url(request) + return None def _connect_failure( @@ -869,6 +1097,58 @@ def _raised_like_python(error: _RaisedT, request_provider: str) -> _RaisedT: return error if request_provider in _MAIN_NATIVE_ERROR_PROVIDERS else _raised(error) +def _chunk_parser_failure(text: str) -> Exception: + try: + json.loads(text).pop(_INVOCATION_METRICS, {}) + except Exception as error: # noqa: BLE001 # the chunk parser's own failure is what Python raised + return error + return ValueError(text) + + +def _json_failure(text: str) -> Exception: + try: + json.loads(text) + except json.JSONDecodeError as error: + return error + return ValueError(text) + + +def bedrock_stream_raise(kind: str, arguments: Sequence[object]) -> Exception: + """The exception Python's Bedrock stream decoder raised unmapped for a body it could not parse.""" + from botocore import eventstream + + raises: Final[Mapping[str, Callable[..., Exception]]] = { + "ChecksumMismatch": eventstream.ChecksumMismatch, + "InvalidHeadersLength": eventstream.InvalidHeadersLength, + "InvalidPayloadLength": eventstream.InvalidPayloadLength, + "DuplicateHeader": eventstream.DuplicateHeader, + "InvalidJson": _json_failure, + "NotAnObject": _chunk_parser_failure, + } + return _raise_with(raises[kind], arguments) + + +def _raise_with(factory: Callable[..., Exception], arguments: Sequence[object]) -> Exception: + return factory(*arguments) + + +def stream_failure(error: BaseException, request: Mapping[str, object], request_provider: str) -> BaseException: + """Python's Bedrock Messages stream raises a failure after its head with the deployment's debug info.""" + if request_provider in _MAIN_NATIVE_ERROR_PROVIDERS or not isinstance(error, Exception): + return error + return _with_deployment_debug_info(error, request, request_provider) + + +def _bedrock_stream_error(error: Exception, request: Mapping[str, object]) -> Exception | None: + """Python's Bedrock stream decoder raises its `BedrockError` as built, unmapped.""" + try: + status, text = _UPSTREAM_ARGS.validate_python(error.args) + headers: Final = _UPSTREAM_HEADERS.validate_python(getattr(error, "headers", None)) + except ValidationError: + return None + return _config_error(status, text, headers, request, "bedrock") + + def map_failure( error: Exception, request: Mapping[str, object], @@ -878,6 +1158,10 @@ def map_failure( ) -> Exception: """Raises what Python's Messages handler raises for the same failure. `secrets_read` are the secrets the host read to send the call, such as azure_ai's `AZURE_API_BASE`.""" + if request_provider == "bedrock" and getattr(error, "messages_stream_failure", False): + stream_error: Final = _bedrock_stream_error(error, request) + if stream_error is not None: + return stream_error if getattr(error, "messages_timeout", None) == "Body": # Python reads a stream's body outside its HTTP client's error mapping, so the transport's # own timeout reaches the caller unmapped. @@ -915,6 +1199,10 @@ def map_failure( ), request_provider, ) + credential_failure: Final = _bedrock_credential_failure(error, request) if request_provider == "bedrock" else None + if credential_failure is not None: + _log_resolved_call(request, request_provider) + return map_token_failure(credential_failure, request, request_provider, asynchronous) provider_error: Final = _provider_error(error, request, request_provider, secrets_read or []) if provider_error is None: return failures.map_native_failure( @@ -950,6 +1238,20 @@ def failure_provider(request: Mapping[str, object]) -> str: return _resolved_provider(request) or "anthropic" +def filters_beta_headers(model: str, custom_llm_provider: str | None, api_base: str | None) -> bool: + """Whether the call's provider config filters the caller's `anthropic-beta` before Python's client encodes it.""" + config: Final = native_messages_config( + model.removeprefix(f"{custom_llm_provider}/"), custom_llm_provider, None, api_base + ) + return config is not None and config.should_filter_anthropic_beta_headers() + + +def record_guardrail(public: Mapping[str, object], request: Mapping[str, object]) -> None: + logger: Final = _logger(request) + if logger is not None: + logger.record_native_bedrock_guardrail(public) + + def _scope_provider_specific_header(request: Mapping[str, object]) -> None: scoped: Final = request.get("provider_specific_header") if scoped is None: @@ -1009,10 +1311,15 @@ def settings( drops_unsupported_params=drops_unsupported_params(kwargs.get("drop_params")), ascii_response_headers=aiohttp, joined_reads=aiohttp, + detached_drain_timeout=ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS if asynchronous else None, + detached_drain_limit=ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS if asynchronous else None, ) - return asdict( - configured if request is None else _with_call_settings(configured, request, kwargs.get("litellm_logging_obj")) - ) + if request is None: + return asdict(configured) + call_settings: Final = _with_call_settings(configured, request, kwargs.get("litellm_logging_obj")) + if call_settings.mock_response is None and not call_settings.mcp_gateway: + _refuse_unsupported_thread(request, asynchronous) + return asdict(call_settings) def _flag_messages_call(logger: object, asynchronous: bool) -> None: @@ -1021,13 +1328,6 @@ def _flag_messages_call(logger: object, asynchronous: bool) -> None: logger.model_call_details.setdefault("litellm_params", {})[CallTypes.aanthropic_messages.value] = asynchronous -def _flag_azure_ai_call(request: Mapping[str, object], request_provider: str, asynchronous: bool) -> None: - """Python's handler flags the call before it gets the credential or refuses a param. By the owner's ruling - anthropic keeps main's unflagged failure.""" - if request_provider == "azure_ai": - _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) - - @dataclass(frozen=True, slots=True) class _Answered: value: object @@ -1296,7 +1596,7 @@ def map_token_failure( ) -> Exception: """Python's handler maps a litellm exception raised while it gets the call's credential, and lets any other exception through as raised.""" - _flag_azure_ai_call(request, request_provider, asynchronous) + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) if not isinstance(error, BaseLLMException): return error return _public_provider_error(error, error, request, request_provider) @@ -1518,13 +1818,14 @@ def unencodable_header(value: bytes, received: bool) -> Exception: def _sent_url(request: Mapping[str, object], url: str) -> str: """The native URL builder trims the caller's api_base; Python sends it as given.""" api_base: Final = request.get("api_base") - if ( - not isinstance(api_base, str) - or api_base == api_base.strip() - or not _unsendable_api_base(api_base) - or _resolved_provider(request) != "anthropic" - ): + if not isinstance(api_base, str) or api_base == api_base.strip() or not _unsendable_api_base(api_base): return url + provider: Final = _resolved_provider(request) + if provider not in ("anthropic", "bedrock"): + return url + if provider == "bedrock": + trimmed: Final = f"{api_base.strip().rstrip('/')}/" + return join_api_base(api_base, url.removeprefix(trimmed)) if url.startswith(trimmed) else url try: return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url except ValueError: # a base urlsplit cannot split keeps the URL the native builder made diff --git a/litellm/rust_bridge/host/settings.py b/litellm/rust_bridge/host/settings.py index 895e7994f71..77a4a55bf4b 100644 --- a/litellm/rust_bridge/host/settings.py +++ b/litellm/rust_bridge/host/settings.py @@ -31,6 +31,8 @@ class ProviderDefaults: enable_azure_ad_token_refresh: object api_key: object azure_key: object + drop_params: object + bedrock_request_metadata_fields: object @dataclass(frozen=True, slots=True) @@ -145,6 +147,8 @@ def provider_defaults() -> ProviderDefaults: enable_azure_ad_token_refresh=litellm.enable_azure_ad_token_refresh, api_key=litellm.api_key, azure_key=litellm.azure_key, + drop_params=litellm.drop_params, + bedrock_request_metadata_fields=litellm.bedrock_request_metadata_fields, ) diff --git a/litellm/rust_bridge/messages/entrypoints.py b/litellm/rust_bridge/messages/entrypoints.py index cc708914df5..56772b6b3a7 100644 --- a/litellm/rust_bridge/messages/entrypoints.py +++ b/litellm/rust_bridge/messages/entrypoints.py @@ -14,6 +14,7 @@ class MessagesStreamHead(dict[str, object]): agentic: Callable[[execution.AsyncStream], object] | None = None deliver: Callable[[object], Awaitable[object]] | None = None python_types: bool = False + relayed: bool = False cached: object | None = None cache_writer: Callable[[object], object] | None = None diff --git a/tests/test_litellm_rust/messages/test_bedrock.py b/tests/test_litellm_rust/messages/test_bedrock.py new file mode 100644 index 00000000000..6f2df6d908c --- /dev/null +++ b/tests/test_litellm_rust/messages/test_bedrock.py @@ -0,0 +1,713 @@ +import asyncio +import base64 +import binascii +import json +import logging +import struct +import uuid +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Iterator, Mapping, Sequence +from typing import Final, cast # noqa: TID251 # narrows the untyped Messages entries to their call shape + +import pytest +from botocore.exceptions import NoCredentialsError, PartialCredentialsError +from pydantic import TypeAdapter + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.anthropic.messages.native import handler as python_messages +from litellm.llms.bedrock.base_aws_llm import AwsAuthError +from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_EVENTS, MESSAGES_RESPONSE, RATE_LIMITED +from tests.test_litellm_rust.support.isolation import rebound +from tests.test_litellm_rust.support.recording_server import RecordedRequest, RecordingServer, ResponseSpec, relaxed + +pytestmark = pytest.mark.requires_rust_extension + +BEDROCK_MODEL: Final = "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" +AWS: Final = { + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-east-1", +} +GUARDRAIL_TRACE: Final[dict[str, object]] = { + "guardrail": { + "input": {"gr-1": {"invocationMetrics": {"usage": {"contentPolicyUnits": 1, "topicPolicyUnits": 1}}}}, + "outputs": [], + } +} + + +@pytest.fixture +def server(server: RecordingServer) -> Iterator[RecordingServer]: + yield from relaxed(server) + + +@pytest.fixture(autouse=True) +def no_ambient_bedrock_auth(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("AWS_BEARER_TOKEN_BEDROCK", "AWS_BEDROCK_RUNTIME_ENDPOINT", "AWS_SESSION_TOKEN", "AWS_PROFILE"): + monkeypatch.delenv(name, raising=False) + + +_OBJECT: Final = TypeAdapter(dict[str, object]) +_RUST_ENTRY: Final = cast( # cast-ok: the public entry's call shape, which dispatch sends to Rust + Callable[..., Awaitable[object]], litellm.anthropic.messages.acreate +) +_PYTHON_ENTRY: Final = cast( # cast-ok: the Python handler's call shape, which no dispatch reroutes + Callable[..., Awaitable[object]], python_messages.anthropic_messages +) + + +async def _rust(server: RecordingServer, **kwargs: object) -> object: + return await _RUST_ENTRY(**_arguments(server, kwargs)) + + +async def _python(server: RecordingServer, **kwargs: object) -> object: + return await _PYTHON_ENTRY(**_arguments(server, kwargs)) + + +def _arguments(server: RecordingServer, kwargs: Mapping[str, object]) -> dict[str, object]: + return { + "model": BEDROCK_MODEL, + "messages": list(MESSAGES), + "max_tokens": 64, + "api_base": server.base_url, + "litellm_call_id": str(uuid.uuid4()), + **AWS, + **kwargs, + } + + +def _signed_headers(request: RecordedRequest) -> str: + authorization: Final = request.headers["authorization"] + assert authorization.startswith("AWS4-HMAC-SHA256 ") + return authorization.split("SignedHeaders=", 1)[1].split(",", 1)[0] + + +def _public(response: object) -> object: + return {name: value for name, value in _OBJECT.validate_python(response).items() if name != "_hidden_params"} + + +@pytest.mark.asyncio +async def test_bedrock_invoke_sends_and_answers_what_python_does(server: RecordingServer) -> None: + server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + + python_response: Final = await _python(server) + rust_response: Final = await _rust(server) + + python_sent, rust_sent = server.requests + assert rust_sent.path == python_sent.path + assert rust_sent.body == python_sent.body + assert _signed_headers(rust_sent) == _signed_headers(python_sent) + assert _public(rust_response) == _public(python_response) + + +@pytest.mark.asyncio +async def test_bedrock_invoke_upstream_failure_raises_the_bedrock_exception_python_raises( + server: RecordingServer, +) -> None: + server.default_response = ResponseSpec(body=RATE_LIMITED, status=429) + + with pytest.raises(litellm.RateLimitError) as python_raised: + await _python(server) + with pytest.raises(litellm.RateLimitError) as rust_raised: + await _rust(server) + + assert str(rust_raised.value) == str(python_raised.value) + assert rust_raised.value.llm_provider == python_raised.value.llm_provider == "bedrock" # pyright: ignore[reportUnknownMemberType] # the public error leaves llm_provider untyped + assert rust_raised.value.status_code == python_raised.value.status_code == 429 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("thread", "error_class"), + ( + pytest.param({"type": "create"}, litellm.BadRequestError, id="create"), + pytest.param({"type": "continue", "previous_message_id": "msg_1"}, litellm.NotFoundError, id="continue"), + ), +) +async def test_bedrock_invoke_refuses_a_thread_as_python_does( + server: RecordingServer, thread: Mapping[str, str], error_class: type[Exception] +) -> None: + with pytest.raises(error_class) as python_raised: + await _python(server, thread=dict(thread)) + with pytest.raises(error_class) as rust_raised: + await _rust(server, thread=dict(thread)) + + assert str(rust_raised.value) == str(python_raised.value) + assert server.requests == [] + + +@pytest.mark.asyncio +async def test_bedrock_invoke_drops_a_thread_under_drop_params_as_python_does(server: RecordingServer) -> None: + server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + + await _python(server, thread={"type": "create"}, drop_params=True) + await _rust(server, thread={"type": "create"}, drop_params=True) + + python_sent, rust_sent = server.requests + assert "thread" not in _OBJECT.validate_python(rust_sent.body) + assert rust_sent.body == python_sent.body + + +class _GuardrailBilling(CustomLogger): + def __init__(self) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # CustomLogger's untyped init + self.billed: list[object] = [] + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.billed.append(_OBJECT.validate_python(kwargs.get("standard_logging_object")).get("guardrail_information")) + + async def wait_for(self, count: int) -> None: + async with asyncio.timeout(5): + while len(self.billed) < count: + await asyncio.sleep(0.01) + + +@pytest.mark.asyncio +async def test_bedrock_invoke_bills_the_guardrail_trace_as_python_does(server: RecordingServer) -> None: + server.default_response = ResponseSpec(body={**MESSAGES_RESPONSE, "amazon-bedrock-trace": GUARDRAIL_TRACE}) + guardrail: Final = {"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}} + billing: Final = _GuardrailBilling() + + with rebound(litellm, "callbacks", [billing]): + await _python(server, **guardrail) + await billing.wait_for(1) + await _rust(server, **guardrail) + await billing.wait_for(2) + + python_sent, rust_sent = server.requests + python_billed, rust_billed = billing.billed + assert python_billed + assert rust_billed == python_billed + assert {name: value for name, value in rust_sent.headers.items() if name.startswith("x-amzn-bedrock-")} == { + name: value for name, value in python_sent.headers.items() if name.startswith("x-amzn-bedrock-") + } + + +def _aws_header(name: str, value: str) -> bytes: + return bytes((len(name),)) + name.encode() + b"\x07" + struct.pack("!H", len(value)) + value.encode() + + +def _wire_frame(message_type: str, kind: str, payload: bytes) -> bytes: + headers: Final = ( + _aws_header(":exception-type" if message_type == "exception" else ":event-type", kind) + + _aws_header(":content-type", "application/json") + + _aws_header(":message-type", message_type) + ) + prelude: Final = struct.pack("!II", 12 + len(headers) + len(payload) + 4, len(headers)) + prelude_crc: Final = struct.pack("!I", binascii.crc32(prelude) & 0xFFFFFFFF) + message: Final = prelude + prelude_crc + headers + payload + return message + struct.pack("!I", binascii.crc32(message) & 0xFFFFFFFF) + + +def _chunk_frame(text: bytes) -> bytes: + return _wire_frame("event", "chunk", json.dumps({"bytes": base64.b64encode(text).decode()}).encode()) + + +def _aws_frame(event: Mapping[str, object]) -> bytes: + return _chunk_frame(json.dumps(event).encode()) + + +def _event_stream(final: Mapping[str, object]) -> bytes: + *head, (_, stop) = MESSAGES_EVENTS + events: Final = (*(data for _, data in head), {**_OBJECT.validate_python(stop), **final}) + return b"".join(_aws_frame(event) for event in events) + + +def _sse_events(chunks: Sequence[bytes]) -> tuple[object, ...]: + frames: Final = b"".join(chunks).decode().split("\n\n") + return tuple(json.loads(frame.split("data: ", 1)[1]) for frame in frames if frame) + + +async def _drain(response: object) -> tuple[bytes, ...]: + stream: Final = cast(AsyncIterator[bytes], response) # cast-ok: a streamed Messages answer yields SSE bytes + return tuple([chunk async for chunk in stream]) + + +@pytest.mark.asyncio +async def test_bedrock_invoke_stream_bills_the_guardrail_trace_of_message_stop_as_python_does( + server: RecordingServer, +) -> None: + server.default_response = ResponseSpec( + body=_event_stream({"amazon-bedrock-trace": GUARDRAIL_TRACE}), + headers={"content-type": "application/vnd.amazon.eventstream"}, + ) + guardrail: Final = {"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}} + billing: Final = _GuardrailBilling() + + with rebound(litellm, "callbacks", [billing]): + python_chunks: Final = await _drain(await _python(server, stream=True, **guardrail)) + await billing.wait_for(1) + rust_chunks: Final = await _drain(await _rust(server, stream=True, **guardrail)) + await billing.wait_for(2) + + python_billed, rust_billed = billing.billed + assert python_billed + assert rust_billed == python_billed + assert _sse_events(rust_chunks)[-1] == _sse_events(python_chunks)[-1] + + +@pytest.fixture +def no_aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_ROLE_NAME", "AWS_WEB_IDENTITY_TOKEN_FILE"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("AWS_CONFIG_FILE", "/nonexistent") + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", "/nonexistent") + monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_aws_credentials") +@pytest.mark.parametrize( + ("credentials", "raised"), + ( + pytest.param({}, NoCredentialsError, id="no-credentials"), + pytest.param( + {"aws_web_identity_token": "os.environ/AWS_WEB_IDENTITY_TOKEN"}, AwsAuthError, id="environment-reference" + ), + pytest.param( + {"aws_web_identity_token": "bedrock-web-identity-token-not-set"}, AwsAuthError, id="unresolved-token" + ), + ), +) +async def test_bedrock_invoke_credential_failure_raises_what_python_raises( + server: RecordingServer, credentials: Mapping[str, str], raised: type[Exception] +) -> None: + web_identity: Final = ( + {"aws_role_name": "arn:aws:iam::123456789012:role/bedrock", "aws_session_name": "litellm-session"} + if credentials + else {} + ) + unsigned: Final = {"aws_access_key_id": None, "aws_secret_access_key": None, **credentials, **web_identity} + + with pytest.raises(raised) as python_raised: + await _python(server, **unsigned) + with pytest.raises(raised) as rust_raised: + await _rust(server, **unsigned) + + assert type(rust_raised.value) is type(python_raised.value) + assert str(rust_raised.value) == str(python_raised.value) + assert getattr(rust_raised.value, "status_code", None) == getattr(python_raised.value, "status_code", None) + assert server.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_aws_credentials") +async def test_bedrock_invoke_access_key_without_its_secret_raises_what_python_raises( + server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIDEXAMPLE") + unsigned: Final = {"aws_access_key_id": None, "aws_secret_access_key": None} + + with pytest.raises(PartialCredentialsError) as python_raised: + await _python(server, **unsigned) + with pytest.raises(PartialCredentialsError) as rust_raised: + await _rust(server, **unsigned) + + assert str(rust_raised.value) == str(python_raised.value) + assert server.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deployment", + ( + pytest.param( + {"model_id": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123"}, + id="model-id-arn", + ), + pytest.param( + { + "model": "bedrock/invoke/arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + }, + id="invoke-arn", + ), + pytest.param( + {"model": "bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0", "aws_region_name": None}, + id="invoke-region-path", + ), + pytest.param( + { + "model": "bedrock/invoke/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "aws_region_name": None, + }, + id="invoke-region-path-arn", + ), + pytest.param({"stream": False, "drop_params": True, "temperature": 0.2}, id="sampling"), + ), +) +async def test_bedrock_invoke_sends_the_url_and_body_python_sends( + server: RecordingServer, deployment: Mapping[str, object] +) -> None: + server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + + await _python(server, **deployment) + await _rust(server, **deployment) + + python_sent, rust_sent = server.requests + assert rust_sent.path == python_sent.path + assert rust_sent.body == python_sent.body + + +def _signing(request: RecordedRequest) -> tuple[object, ...]: + authorization: Final = request.headers["authorization"] + return ( + request.path, + authorization.split(", Signature=", 1)[0].split("/", 2)[2], + request.headers.get("x-amz-security-token"), + "x-amz-date" in request.headers, + "date" in request.headers, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller_headers", + ( + pytest.param( + { + "x-amz-date": "20200101T000000Z", + "x-amz-security-token": "caller-token", + "date": "Wed, 01 Jan 2020 00:00:00 GMT", + }, + id="all", + ), + pytest.param({"x-amz-security-token": "caller-token"}, id="security-token"), + pytest.param({"x-amz-date": "20200101T000000Z"}, id="amz-date"), + pytest.param({"Date": "Wed, 01 Jan 2020 00:00:00 GMT"}, id="date"), + ), +) +@pytest.mark.parametrize("session_token", (None, "deployment-session"), ids=("static", "session")) +async def test_bedrock_invoke_signs_caller_amz_headers_as_python_does( + server: RecordingServer, caller_headers: Mapping[str, str], session_token: str | None +) -> None: + server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + deployment: Final = { + "model": "bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0", + "aws_region_name": None, + "aws_session_token": session_token, + "extra_headers": dict(caller_headers), + } + + await _python(server, **deployment) + await _rust(server, **deployment) + + python_sent, rust_sent = server.requests + assert _signing(rust_sent) == _signing(python_sent) + + +@pytest.mark.asyncio +async def test_bedrock_invoke_refuses_an_invalid_guardrail_config_as_python_does(server: RecordingServer) -> None: + invalid: Final = {"guardrailConfig": {"guardrailVersion": "1"}} + + with pytest.raises(litellm.BadRequestError) as python_raised: + await _python(server, **invalid) + with pytest.raises(litellm.BadRequestError) as rust_raised: + await _rust(server, **invalid) + + assert str(rust_raised.value) == str(python_raised.value) + assert type(rust_raised.value) is type(python_raised.value) + assert getattr(rust_raised.value, "status_code", None) == getattr(python_raised.value, "status_code", None) + assert server.requests == [] + + +@pytest.mark.asyncio +async def test_bedrock_invoke_refuses_a_web_search_tool_as_python_does(server: RecordingServer) -> None: + web_search: Final = {"tools": [{"type": "web_search_20250305", "name": "web_search"}]} + + with pytest.raises(litellm.BadRequestError) as python_raised: + await _python(server, **web_search) + with pytest.raises(litellm.BadRequestError) as rust_raised: + await _rust(server, **web_search) + + assert type(rust_raised.value) is type(python_raised.value) + assert str(rust_raised.value) == str(python_raised.value) + assert rust_raised.value.llm_provider == python_raised.value.llm_provider # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the public error's provider + assert rust_raised.value.model == python_raised.value.model # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the public error's model + assert server.requests == [] + assert getattr(rust_raised.value, "litellm_debug_info", None) == getattr( + python_raised.value, "litellm_debug_info", None + ) + + +_EVENTS: Final = tuple(data for _, data in MESSAGES_EVENTS) + + +class _Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # CustomLogger's untyped init + self.logged: list[tuple[str, dict[str, object]]] = [] + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.logged.append(("success", _OBJECT.validate_python(kwargs.get("standard_logging_object")))) + + async def async_log_failure_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.logged.append(("failure", _OBJECT.validate_python(kwargs.get("standard_logging_object")))) + + async def wait_for(self, count: int) -> None: + async with asyncio.timeout(5): + while len(self.logged) < count: + await asyncio.sleep(0.01) + + +def _billed(logged: tuple[str, Mapping[str, object]]) -> tuple[str, object, object]: + kind, record = logged + response: Final = record.get("response") + return ( + kind, + {name: value for name, value in _OBJECT.validate_python(response).items() if name != "created"} + if isinstance(response, dict) + else response, + record.get("error_str"), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "events", + ( + pytest.param(_EVENTS, id="complete"), + pytest.param(_EVENTS[:5], id="no-message-stop"), + pytest.param(_EVENTS[:3], id="no-message-delta"), + pytest.param( + (*_EVENTS[:2], {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hé"}}), + id="non-ascii", + ), + ), +) +async def test_bedrock_invoke_stream_relays_the_frames_and_bills_what_python_does( + server: RecordingServer, events: Sequence[Mapping[str, object]] +) -> None: + server.default_response = ResponseSpec(body=b"".join(_aws_frame(event) for event in events)) + recorder: Final = _Recorder() + + with rebound(litellm, "callbacks", [recorder]): + python_chunks: Final = await _drain(await _python(server, stream=True)) + await recorder.wait_for(1) + rust_chunks: Final = await _drain(await _rust(server, stream=True)) + await recorder.wait_for(2) + + assert rust_chunks == python_chunks + python_logged, rust_logged = recorder.logged + assert _billed(rust_logged) == _billed(python_logged) + + +_STREAM_HEAD: Final = b"".join(_aws_frame(event) for event in _EVENTS[:3]) + + +async def _failed_stream( + entry: Callable[..., Awaitable[object]], server: RecordingServer +) -> tuple[tuple[bytes, ...], Exception]: + chunks: Final[list[bytes]] = [] + try: + async for chunk in cast( + AsyncIterator[bytes], await entry(server, stream=True) + ): # cast-ok: a streamed Messages answer yields SSE bytes + chunks.append(chunk) + except Exception as error: # noqa: BLE001 # each lane's failure class is what the test compares + return tuple(chunks), error + raise AssertionError("the stream ended without a failure") + + +def _raised(error: BaseException | None) -> tuple[object, ...]: + if error is None: + return (None,) + return ( + type(error), + str(error), + getattr(error, "status_code", None), + getattr(error, "litellm_debug_info", None), + *_raised(error.__cause__), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response", + ( + pytest.param( + ResponseSpec(body=_STREAM_HEAD + _wire_frame("event", "internalServerException", b'{"message": "boom"}')), + id="error-event-mid-stream", + ), + pytest.param( + ResponseSpec(body=_wire_frame("exception", "throttlingException", b'{"message": "slow"}')), + id="exception-frame-first", + ), + pytest.param(ResponseSpec(body=b""), id="empty-body"), + pytest.param(ResponseSpec(body=_wire_frame("event", "somethingNew", b'{"message": "x"}')), id="unknown-event"), + pytest.param(ResponseSpec(body=MESSAGES_RESPONSE), id="json-body"), + pytest.param(ResponseSpec(body=_STREAM_HEAD + _chunk_frame(b'{"type": ')), id="invalid-json-chunk"), + pytest.param(ResponseSpec(body=_STREAM_HEAD + _chunk_frame(b"[1]")), id="list-chunk"), + pytest.param(ResponseSpec(body=_STREAM_HEAD + _STREAM_HEAD, body_chunks=2, cut_after=1), id="cut-body"), + ), +) +async def test_bedrock_invoke_stream_failure_raises_and_logs_what_python_does( + server: RecordingServer, response: ResponseSpec +) -> None: + server.default_response = response + recorder: Final = _Recorder() + + with rebound(litellm, "callbacks", [recorder]): + python_chunks, python_error = await _failed_stream(_python, server) + await recorder.wait_for(1) + rust_chunks, rust_error = await _failed_stream(_rust, server) + await recorder.wait_for(2) + + assert rust_chunks == python_chunks + assert _raised(rust_error) == _raised(python_error) + python_logged, rust_logged = recorder.logged + assert _billed(rust_logged) == _billed(python_logged) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tools", + ( + pytest.param( + [ + { + "name": "a", + "description": "A", + "input_schema": {"type": "object"}, + "custom": {"defer_loading": True}, + }, + { + "name": "b", + "description": "B", + "input_schema": {"type": "object"}, + "custom": {"defer_loading": "no"}, + }, + ], + id="custom-field", + ), + pytest.param( + [ + { + "type": "custom", + "name": "n", + "input_schema": {"type": "custom", "properties": {"o": {"type": "custom"}}}, + } + ], + id="custom-schema-type", + ), + pytest.param( + [{"name": " ", "input_schema": {"type": "object"}}, {"name": "", "input_schema": {"type": "object"}}], + id="unnamed", + ), + ), +) +async def test_bedrock_invoke_logs_the_tools_python_shaped( + server: RecordingServer, tools: Sequence[Mapping[str, object]] +) -> None: + server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + recorder: Final = _Recorder() + python_tools: Final = json.loads(json.dumps(tools)) + rust_tools: Final = json.loads(json.dumps(tools)) + + with rebound(litellm, "callbacks", [recorder]): + await _python(server, tools=python_tools) + await recorder.wait_for(1) + await _rust(server, tools=rust_tools) + await recorder.wait_for(2) + + (_, python_logged), (_, rust_logged) = recorder.logged + assert rust_tools == python_tools != tools + assert _OBJECT.validate_python(rust_logged["model_parameters"])["tools"] == python_tools + assert _OBJECT.validate_python(python_logged["model_parameters"])["tools"] == python_tools + + +class _Warnings(logging.Handler): + def __init__(self) -> None: + super().__init__(logging.WARNING) + self.messages: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +_SCHEMA: Final = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + "additionalProperties": False, + }, +} +_NATIVE_OUTPUT_REJECTED: Final = ResponseSpec( + body={"message": "output_config.format: Extra inputs are not permitted"}, status=400 +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_params", "rejected"), + ( + pytest.param({"output_config": {"format": _SCHEMA}}, False, id="native-output"), + pytest.param({"output_config": {"format": _SCHEMA}}, True, id="native-output-rejected"), + pytest.param( + {"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", "output_format": _SCHEMA}, + False, + id="inlined-schema", + ), + pytest.param( + {"max_tokens": 1000, "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}}, + False, + id="clear-thinking", + ), + pytest.param({"service_tier": "auto"}, False, id="service-tier"), + ), +) +async def test_bedrock_invoke_warns_what_python_warns( + server: RecordingServer, request_params: Mapping[str, object], rejected: bool +) -> None: + server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + warnings: Final = _Warnings() + logger: Final = logging.getLogger("LiteLLM") + logger.addHandler(warnings) + try: + if rejected: + server.enqueue(_NATIVE_OUTPUT_REJECTED) + await _python(server, **request_params) + python_warnings: Final = list(warnings.messages) + warnings.messages.clear() + if rejected: + server.enqueue(_NATIVE_OUTPUT_REJECTED) + await _rust(server, **request_params) + finally: + logger.removeHandler(warnings) + + assert warnings.messages == python_warnings + + +@pytest.mark.asyncio +async def test_bedrock_invoke_stream_bills_the_terminal_usage_after_the_caller_detaches( + server: RecordingServer, +) -> None: + start, *middle, (_, delta), stop = MESSAGES_EVENTS + events: Final = (start[1], *(data for _, data in middle), {**delta, "usage": {"output_tokens": 412}}, stop[1]) + server.default_response = ResponseSpec( + body=b"".join(_aws_frame(event) for event in events), body_chunks=len(events), chunk_delay=0.1 + ) + recorder: Final = _Recorder() + + with rebound(litellm, "callbacks", [recorder]): + for entry, count in ((_python, 1), (_rust, 2)): + stream = cast( + AsyncGenerator[bytes], await entry(server, stream=True) + ) # cast-ok: a streamed Messages answer is an async generator + await anext(stream) + await stream.aclose() + await recorder.wait_for(count) + + python_logged, rust_logged = recorder.logged + assert _billed(rust_logged) == _billed(python_logged) + assert ( + _OBJECT.validate_python(_OBJECT.validate_python(rust_logged[1]["response"])["usage"])["completion_tokens"] + == 412 + ) diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 1df83c5649c..b7c755efd82 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -12,9 +12,9 @@ from litellm.exceptions import UnsupportedParamsError from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger from litellm.llms.anthropic.messages.native import handler as python_messages -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.anthropic.messages.native import interceptors as messages_interceptors from litellm.llms.anthropic.messages.native.interceptors import MessagesInterceptor +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.messages import dispatch from litellm.messages.dispatch import ( _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch @@ -256,6 +256,48 @@ def test_only_requests_a_provider_messages_config_serves_reach_a_messages_target assert result is (native_result if passes_through else python_result) +BEDROCK_INVOKE_MODEL: Final = "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + + +@pytest.mark.parametrize( + ("model", "target"), + ( + pytest.param(BEDROCK_INVOKE_MODEL, Target.BEDROCK_INVOKE, id="invoke"), + pytest.param( + "bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", Target.BEDROCK_INVOKE, id="explicit-invoke" + ), + pytest.param( + "bedrock/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0", Target.BEDROCK_INVOKE, id="regional" + ), + pytest.param("bedrock/claude_platform/claude-sonnet-4-5", Target.MESSAGES, id="claude-platform"), + pytest.param("bedrock/mantle/anthropic.claude-sonnet-5", Target.MESSAGES, id="mantle"), + pytest.param("bedrock_mantle/anthropic.claude-sonnet-5", Target.MESSAGES, id="bedrock-mantle"), + pytest.param( + "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", Target.CHAT_COMPLETIONS, id="converse" + ), + pytest.param("bedrock/us.meta.llama3-2-11b-instruct-v1:0", Target.CHAT_COMPLETIONS, id="non-claude"), + ), +) +def test_only_claude_on_bedrock_invoke_model_reaches_a_bedrock_invoke_target_rule(model: str, target: Target) -> None: + python_result: Final = response(model) + native_result: Final = response(model) + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: fallback + return python_result + + result: Final = _DISPATCH.run( + (16, MESSAGES, model), + {}, + python=python, + binding=messages_binding(lambda request: native_result), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=(RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED, targets=frozenset({Target.BEDROCK_INVOKE})),), + ) + + assert dispatch.messages_target(model, None, None, None) is target + assert result is (native_result if target is Target.BEDROCK_INVOKE else python_result) + + DANGEROUS_TOOL_USE: Final = [{"type": "dangerous_tool_use", "classifier_context": {"permission_mode": "auto"}}] CLASSIFIER_SETTINGS: Final = {"safeguards_classifier_model": "gpt-oss-120b"} @@ -756,6 +798,49 @@ async def test_anthropic_runs_on_rust_without_the_opt_in( assert shipped_native == ["async" if asynchronous else "sync"] +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +async def test_bedrock_invoke_runs_on_rust_without_the_opt_in( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool +) -> None: + monkeypatch.setenv("LITELLM_RUST", "0") + monkeypatch.setattr(litellm, "callbacks", []) + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback + pytest.fail("bedrock invoke has no Python Messages path") + + with rebound(dispatch, "_PYTHON_MESSAGES", python), rebound(dispatch, "_PYTHON_AMESSAGES", python): + result: Final = await _public(asynchronous, BEDROCK_INVOKE_MODEL, {}) + + assert isinstance(result, dict) + assert shipped_native == ["async" if asynchronous else "sync"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model", + ( + pytest.param("bedrock/claude_platform/claude-sonnet-4-5", id="claude-platform"), + pytest.param("bedrock/mantle/anthropic.claude-sonnet-5", id="mantle"), + pytest.param("bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", id="converse"), + ), +) +async def test_other_bedrock_messages_keep_python( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], model: str +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + python_result: Final = response(model) + + async def apython(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + return python_result + + with rebound(dispatch, "_PYTHON_AMESSAGES", apython): + result: Final = await _public(True, model, {}) + + assert result is python_result + assert shipped_native == [] + + @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) @pytest.mark.parametrize( @@ -982,6 +1067,8 @@ async def test_azure_ai_runs_on_rust_only_where_its_claude_config_serves_the_cal False, id="azure-ai-non-claude-passthrough", ), + pytest.param(BEDROCK_INVOKE_MODEL, {}, True, id="bedrock-invoke"), + pytest.param("bedrock/mantle/anthropic.claude-sonnet-5", {}, False, id="bedrock-mantle"), pytest.param("deepseek/deepseek-v4-flash", {}, False, id="deepseek"), ), ) diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index a8da1ad512d..215def14c90 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -5739,7 +5739,10 @@ _BEDROCK_INVOKE_REPLY: Final = MappingProxyType( async def test_a_bedrock_claude_deployment_declaring_the_breakpoint_flag_keeps_bedrocks_own_cache_marks( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str, expected_system: list[object] ) -> None: + from litellm.llms.anthropic.messages.native import handler as python_messages + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) bedrock: Final = respx_mock.post(host="bedrock-runtime.us-east-1.amazonaws.com").mock( side_effect=lambda request: Response( 200, diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index a17e74a6eef..0c1340f1fbd 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -2051,7 +2051,7 @@ def _bedrock_invoke_recording(bodies: list[dict[str, object]]) -> AsyncHTTPHandl async def _opus_4_8_messages_with_top_k(bodies: list[dict[str, object]], **deployment: object) -> object: - return await messages.acreate( + return await python_messages.anthropic_messages( max_tokens=16, messages=[{"role": "user", "content": "hi"}], model=_OPUS_4_8_ON_BEDROCK_INVOKE, @@ -2593,7 +2593,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): } # Call the function with AWS credentials - await litellm.anthropic.messages.acreate( + await python_messages.anthropic_messages( messages=[{"role": "user", "content": "Hello, test credentials"}], model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", max_tokens=100, @@ -2648,7 +2648,7 @@ async def test_anthropic_messages_bedrock_dynamic_region(): test_region = "us-east-1" # Call anthropic.messages.acreate with aws_region_name - response = await litellm.anthropic.messages.acreate( + response = await python_messages.anthropic_messages( messages=[{"role": "user", "content": "Hello, test region"}], model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", max_tokens=100, diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index d43cc771185..5a5653019d9 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -23,6 +23,7 @@ from botocore.exceptions import ClientError import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.llms.bedrock.chat.converse_handler import BedrockConverseLLM from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.rust_bridge import configuration @@ -501,6 +502,7 @@ async def test_live_anthropic_messages_stream_through_the_router_fails_at_the_st respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=slow_upstream_answer()) router: Final = _router_with_a_stream_timeout_shorter_than_the_upstreams_first_byte( "bedrock/invoke/anthropic.claude-sonnet-4-5-v1:0" @@ -773,6 +775,7 @@ async def test_failed_stream_on_each_router_surface_reports_the_timeout_it_was_b monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=slow_upstream_answer(upstream_answer)) router: Final = _bedrock_router(model, timeout=UPSTREAM_FIRST_BYTE_SECONDS / 2, stream_timeout=STREAM_TIMEOUT_SECONDS) diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index b224fa151cf..351542a2a3c 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -47,6 +47,13 @@ from litellm.llms.custom_httpx.stream_close import aclose_if_supported REASONING_BODY_ADAPTER: Final = TypeAdapter(dict[str, object]) +@pytest.fixture(autouse=True) +def _python_messages(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.native import handler as python_messages + + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) + + @pytest.mark.asyncio async def test_bedrock_sse_wrapper_encodes_dict_chunks(): """Verify that `bedrock_sse_wrapper` converts dictionary chunks to properly formatted Server-Sent Events and forwards non-dict chunks unchanged.""" diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index b15bd9326a7..3207af4f512 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -1386,7 +1386,7 @@ _BATCH_FILE_ID: Final = "s3://batch-inputs/litellm-bedrock-files/batch.jsonl" _BEDROCK_ENTRY_POINTS: Final[tuple[tuple[str, Callable[[], Awaitable[object]], httpx.Response], ...]] = ( ( "anthropic_messages", - lambda: litellm.anthropic.messages.acreate( + lambda: python_messages.anthropic_messages( model="bedrock/invoke/us.anthropic.claude-sonnet-5-5", messages=[{"role": "user", "content": "hi"}], max_tokens=16, diff --git a/tests/unit/rust_bridge/host/routes/test_messages.py b/tests/unit/rust_bridge/host/routes/test_messages.py index 06baa0bd67d..ed4afd2f495 100644 --- a/tests/unit/rust_bridge/host/routes/test_messages.py +++ b/tests/unit/rust_bridge/host/routes/test_messages.py @@ -20,7 +20,11 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.caching.llm_caching_handler import LLMClientCache -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import ( + ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS, + ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS, + DEFAULT_REQUEST_TIMEOUT_SECONDS, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.model_latest_aliases import get_model_latest_aliases @@ -72,6 +76,17 @@ def test_settings_join_buffered_stream_reads_only_on_the_aiohttp_transport( assert messages_host.settings({})["joined_reads"] is joined +@pytest.mark.parametrize(("asynchronous", "drains"), [(True, True), (False, False)]) +def test_settings_drain_a_detached_stream_only_on_an_async_call(asynchronous: bool, drains: bool) -> None: + projected: Final = messages_host.settings({}, asynchronous=asynchronous) + + assert (projected["detached_drain_timeout"], projected["detached_drain_limit"]) == ( + (ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS, ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS) + if drains + else (None, None) + ) + + def test_settings_project_caller_configuration_without_resolving_a_model(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "drop_params", False) monkeypatch.setattr(litellm, "reasoning_auto_summary", True) @@ -96,6 +111,8 @@ def test_settings_project_caller_configuration_without_resolving_a_model(monkeyp "ascii_response_headers": True, "joined_reads": True, "ascii_request_headers": True, + "detached_drain_timeout": None, + "detached_drain_limit": None, } @@ -219,7 +236,22 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl assert additional["request-id"] == additional["llm_provider-request-id"] +@pytest.mark.parametrize("relayed", [False, True]) +def test_stream_hidden_params_carry_whether_the_native_relay_serves_the_stream(relayed: bool) -> None: + from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead + + head: Final = messages_host.stream_hidden_params((), relayed) + + assert isinstance(head, MessagesStreamHead) + assert head.relayed is relayed + assert "relayed" not in head + + _ADAPTIVE_OPUS: Final = "anthropic/claude-opus-4-7" +_BEDROCK_INVOKE: Final = "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" +_AWS: Final = MappingProxyType( + {"aws_access_key_id": "test-access-key", "aws_secret_access_key": "test-secret-key", "aws_region_name": "us-east-1"} +) @pytest.mark.parametrize( @@ -248,6 +280,18 @@ def test_azure_ad_token_is_the_one_python_mints( assert messages_host.azure_ad_token(arguments) == expected +@pytest.mark.parametrize( + ("model", "provider", "expected"), + [ + pytest.param("anthropic/claude-sonnet-4-5", "anthropic", False, id="anthropic"), + pytest.param("azure_ai/claude-sonnet-4-5", "azure_ai", True, id="azure_ai"), + pytest.param(_BEDROCK_INVOKE, "bedrock", True, id="bedrock_invoke"), + ], +) +def test_filters_beta_headers_is_what_the_provider_config_says(model: str, provider: str, expected: bool) -> None: + assert messages_host.filters_beta_headers(model, provider, None) is expected + + @pytest.mark.parametrize( ("call", "expected"), [ @@ -255,6 +299,13 @@ def test_azure_ad_token_is_the_one_python_mints( pytest.param( {"model": "claude-sonnet-4-5", "custom_llm_provider": "azure_ai"}, "azure_ai", id="explicit_provider" ), + pytest.param({"model": _BEDROCK_INVOKE}, "bedrock", id="bedrock_prefix"), + pytest.param( + {"model": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", "custom_llm_provider": "bedrock"}, + "bedrock", + id="bedrock_explicit_provider", + ), + pytest.param({"model": "claude-sonnet-4-5"}, "anthropic", id="anthropic_model"), pytest.param({"model": "not-a-known-model"}, "anthropic", id="unresolved"), ], ) @@ -2118,6 +2169,21 @@ def test_a_provider_config_rejection_maps_as_the_python_http_handler_maps_it() - assert logger.model_call_details["custom_llm_provider"] == "anthropic" +def test_a_bad_request_another_config_raises_maps_by_its_text_as_the_python_http_handler_maps_it() -> None: + text: Final = "Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'." + rejected: Final = ValueError(text) + rejected.messages_request_error = True # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + rejected.messages_provider_rejection = text # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + rejected.messages_provider_bad_request = True # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + + mapped: Final = messages_host.map_failure( + rejected, {"model": "anthropic/claude-sonnet-4-5", "messages": []}, "anthropic" + ) + + assert type(mapped) is litellm.BadRequestError + assert str(mapped) == f"litellm.BadRequestError: AnthropicException - litellm.BadRequestError: {text}" + + def test_short_circuits_needs_a_replaced_web_search_short_circuit(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger @@ -2300,11 +2366,13 @@ _PARAMS: Final = TypeAdapter(dict[str, object]) [ pytest.param("azure_ai", True, True, id="azure_ai_async"), pytest.param("azure_ai", False, False, id="azure_ai_sync"), - pytest.param("anthropic", True, None, id="anthropic_keeps_main"), + pytest.param("anthropic", True, True, id="anthropic_async"), + pytest.param("anthropic", False, False, id="anthropic_sync"), + pytest.param("bedrock", True, True, id="bedrock_async"), ], ) def test_a_token_failure_logs_the_call_flag_python_s_handler_set_first( - provider: str, asynchronous: bool, flagged: bool | None + provider: str, asynchronous: bool, flagged: bool ) -> None: logger: Final = Logging( model=f"{provider}/claude-sonnet-4-5", @@ -2333,11 +2401,13 @@ def test_a_token_failure_logs_the_call_flag_python_s_handler_set_first( [ pytest.param("azure_ai", True, True, id="azure_ai_async"), pytest.param("azure_ai", False, False, id="azure_ai_sync"), - pytest.param("anthropic", True, None, id="anthropic_keeps_main"), + pytest.param("anthropic", True, True, id="anthropic_async"), + pytest.param("anthropic", False, False, id="anthropic_sync"), + pytest.param("bedrock", True, True, id="bedrock_async"), ], ) def test_an_unsupported_param_logs_the_call_flag_python_s_handler_set_first( - provider: str, asynchronous: bool, flagged: bool | None + provider: str, asynchronous: bool, flagged: bool ) -> None: logger: Final = Logging( model=f"{provider}/claude-sonnet-4-5", @@ -2682,3 +2752,337 @@ def test_a_base_python_cannot_split_keeps_the_native_client(model: str) -> None: None, [], ) + + +@pytest.mark.parametrize( + ("status", "body", "error_class", "text"), + [ + ( + 429, + '{"message": "Too many requests"}', + litellm.RateLimitError, + 'litellm.RateLimitError: BedrockException - {"message": "Too many requests"}', + ), + ( + 400, + '{"message": "prompt is too long: 300000 tokens > 200000 maximum"}', + litellm.ContextWindowExceededError, + "litellm.ContextWindowExceededError: litellm.BadRequestError: BedrockException: Context Window Error - " + '{"message": "prompt is too long: 300000 tokens > 200000 maximum"}', + ), + ( + 404, + '{"message": "model not found"}', + litellm.NotFoundError, + 'litellm.NotFoundError: BedrockException - {"message": "model not found"}', + ), + ], + ids=("throttled", "context-window", "not-found"), +) +def test_a_bedrock_upstream_status_maps_to_the_bedrock_exception_python_raises( + status: int, body: str, error_class: type[Exception], text: str +) -> None: + request: Final[dict[str, object]] = {"model": _BEDROCK_INVOKE, "messages": [], **_AWS} + upstream: Final = ValueError(status, body) + upstream.headers = [("content-type", "application/json"), ("x-amzn-requestid", "req-1")] # pyright: ignore[reportAttributeAccessIssue] # set by the native host + + mapped: Final = messages_host.map_failure(upstream, request, messages_host.failure_provider(request)) + + assert type(mapped) is error_class + assert str(mapped) == text + assert mapped.status_code == status # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the public error's status + assert mapped.llm_provider == "bedrock" # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the public error's provider + assert mapped.model == _BEDROCK_INVOKE # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the public error's model + + +def _ssl_configuration(_ssl_verify: object) -> str: + return "" + + +@pytest.mark.parametrize( + ("aiohttp", "endpoint", "native", "text"), + [ + ( + True, + {"aws_bedrock_runtime_endpoint": "http://127.0.0.1:9"}, + f"could not reach the provider: {_TRANSPORT} (Connect): tcp connect error: Connection refused (os error 111)", + "Cannot connect to host 127.0.0.1:9 ssl: [Connect call failed ('127.0.0.1', 9)]", + ), + ( + True, + {}, + f"could not reach the provider: {_TRANSPORT} (Connect): dns error: error resolving DNS: " + "failed to lookup address information: Name or service not known", + "Cannot connect to host bedrock-runtime.us-east-1.amazonaws.com:443 ssl: [Name or service not known]", + ), + ( + False, + {}, + f"could not reach the provider: {_TRANSPORT} (Connect): tcp connect error: Connection refused (os error 111)", + "All connection attempts failed", + ), + ], + ids=("aiohttp-runtime-endpoint", "aiohttp-default-host", "httpx"), +) +def test_a_bedrock_connect_failure_names_the_host_python_reaches( + monkeypatch: pytest.MonkeyPatch, aiohttp: bool, endpoint: Mapping[str, str], native: str, text: str +) -> None: + from litellm.rust_bridge.host import http_transport + + monkeypatch.delenv("AWS_BEDROCK_RUNTIME_ENDPOINT", raising=False) + monkeypatch.setattr(http_transport, "uses_aiohttp_transport", lambda: aiohttp) + monkeypatch.setattr(messages_host, "get_ssl_configuration", _ssl_configuration) + + mapped: Final = messages_host.map_failure( + RuntimeError(native), {"model": _BEDROCK_INVOKE, "messages": [], **_AWS, **endpoint}, "bedrock" + ) + + assert type(mapped) is litellm.ServiceUnavailableError + assert mapped.status_code == 503 + assert mapped.llm_provider == "bedrock" # pyright: ignore[reportUnknownMemberType] # the public error leaves llm_provider untyped + assert str(mapped) == f"litellm.ServiceUnavailableError: BedrockException - {text}" + + +@pytest.fixture +def no_aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_PROFILE", + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_ROLE_NAME", + "AWS_WEB_IDENTITY_TOKEN_FILE", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("AWS_CONFIG_FILE", "/nonexistent") + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", "/nonexistent") + monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true") + + +def _credential_failure(kind: str) -> RuntimeError: + native: Final = RuntimeError("provider authentication failed: could not load AWS default chain credentials") + native.messages_credential_failure = kind # pyright: ignore[reportAttributeAccessIssue] # set by the native host + return native + + +@pytest.mark.usefixtures("no_aws_credentials") +@pytest.mark.parametrize("kind", ("no_credentials", "replay")) +def test_a_bedrock_credential_failure_raises_what_python_signing_raises(kind: str) -> None: + from botocore.exceptions import NoCredentialsError + + mapped: Final = messages_host.map_failure( + _credential_failure(kind), {"model": _BEDROCK_INVOKE, "messages": [], "aws_region_name": "us-east-1"}, "bedrock" + ) + + assert type(mapped) is NoCredentialsError + assert str(mapped) == "Unable to locate credentials" + assert mapped.__traceback__ is not None + + +@pytest.mark.usefixtures("no_aws_credentials") +def test_an_access_key_without_its_secret_raises_the_partial_credentials_error_python_signing_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from botocore.exceptions import PartialCredentialsError + + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-access-key") + + mapped: Final = messages_host.map_failure( + _credential_failure("no_credentials"), + {"model": _BEDROCK_INVOKE, "messages": [], "aws_region_name": "us-east-1"}, + "bedrock", + ) + + assert type(mapped) is PartialCredentialsError + assert str(mapped) == "Partial credentials found in env, missing: AWS_SECRET_ACCESS_KEY" + + +@pytest.mark.usefixtures("no_aws_credentials") +@pytest.mark.parametrize( + ("token", "kind", "status", "text"), + [ + ( + "os.environ/AWS_WEB_IDENTITY_TOKEN_REFERENCE", + "web_identity_reference", + 400, + "Invalid web identity token reference.", + ), + ( + "oidc-secret-that-is-not-set", + "missing_web_identity_token", + 401, + "OIDC token could not be retrieved from secret manager.", + ), + ], + ids=("environment-reference", "unresolved-token"), +) +@pytest.mark.parametrize("replayed", (False, True), ids=("typed", "replayed")) +def test_a_bedrock_web_identity_failure_raises_the_aws_auth_error_python_raises( + token: str, kind: str, status: int, text: str, replayed: bool +) -> None: + from litellm.llms.bedrock.base_aws_llm import AwsAuthError + + request: Final[dict[str, object]] = { + "model": _BEDROCK_INVOKE, + "messages": [], + "aws_region_name": "us-east-1", + "aws_web_identity_token": token, + "aws_role_name": "arn:aws:iam::123456789012:role/bedrock", + "aws_session_name": "litellm-session", + } + + mapped: Final = messages_host.map_failure(_credential_failure("replay" if replayed else kind), request, "bedrock") + + assert type(mapped) is AwsAuthError + assert mapped.status_code == status + assert str(mapped) == text + + +def test_a_bedrock_credential_failure_python_would_sign_through_keeps_the_native_mapping() -> None: + mapped: Final = messages_host.map_failure( + _credential_failure("replay"), {"model": _BEDROCK_INVOKE, "messages": [], **_AWS}, "bedrock" + ) + + assert isinstance(mapped, litellm.APIConnectionError | litellm.InternalServerError) + assert mapped.llm_provider == "bedrock" # pyright: ignore[reportUnknownMemberType] # the public error leaves llm_provider untyped + + +@pytest.mark.usefixtures("no_aws_credentials") +def test_a_credential_failure_text_without_the_marker_keeps_the_native_mapping() -> None: + native: Final = RuntimeError("credential acquisition failed: no credentials in the default chain") + + mapped: Final = messages_host.map_failure( + native, {"model": _BEDROCK_INVOKE, "messages": [], "aws_region_name": "us-east-1"}, "bedrock" + ) + + assert isinstance(mapped, litellm.APIConnectionError | litellm.InternalServerError) + assert mapped.llm_provider == "bedrock" # pyright: ignore[reportUnknownMemberType] # the public error leaves llm_provider untyped + + +@pytest.mark.parametrize( + ("thread", "error_class", "status", "text"), + [ + ( + {"type": "create"}, + litellm.BadRequestError, + 400, + 'litellm.BadRequestError: BedrockException - {"type": "error", "error": {"type": "invalid_request_error", ' + '"message": "message threads are not supported for provider bedrock. Send the request without thread, ' + 'or set drop_params to serve it without a server-side thread", ' + '"details": {"error_code": "thread_unsupported_request"}}}', + ), + ( + {"type": "continue", "previous_message_id": "msg_1"}, + litellm.NotFoundError, + 404, + 'litellm.NotFoundError: BedrockException - {"type": "error", "error": {"type": "not_found_error", ' + '"message": "message threads are not supported for provider bedrock, so this thread cannot be continued. ' + 'Send the full conversation to start over", "details": {"error_code": "thread_not_found"}}}', + ), + ], + ids=("create", "continue"), +) +def test_settings_refuse_a_thread_bedrock_cannot_serve_as_the_python_handler_does( + monkeypatch: pytest.MonkeyPatch, + thread: Mapping[str, str], + error_class: type[Exception], + status: int, + text: str, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = {**_CALL, "model": _BEDROCK_INVOKE, "thread": dict(thread), **_AWS} + + with pytest.raises(error_class) as raised: + messages_host.settings({}, request, asynchronous=True) + + assert str(raised.value) == text + assert raised.value.status_code == status # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the public error's status + assert raised.value.llm_provider == "bedrock" # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the public error's provider + + +def test_a_thread_bedrock_cannot_serve_is_dropped_under_drop_params() -> None: + request: Final = {**_CALL, "model": _BEDROCK_INVOKE, "thread": {"type": "create"}, "drop_params": True, **_AWS} + + messages_host.settings({}, request, asynchronous=True) + + assert "thread" not in messages_host.body_params(request, asynchronous=True) + + +def test_a_bedrock_thread_continue_is_not_found_even_under_drop_params() -> None: + request: Final = { + **_CALL, + "model": _BEDROCK_INVOKE, + "thread": {"type": "continue", "previous_message_id": "msg_1"}, + "drop_params": True, + **_AWS, + } + + with pytest.raises(litellm.NotFoundError): + messages_host.settings({}, request, asynchronous=True) + + +@pytest.mark.parametrize( + ("api_base", "native", "sent"), + [ + ( + " https://bedrock.example ", + "https://bedrock.example/model/m/invoke", + " https://bedrock.example /model/m/invoke", + ), + ("https://bedrock.example", "https://bedrock.example/model/m/invoke", "https://bedrock.example/model/m/invoke"), + ], + ids=("untrimmed", "clean"), +) +def test_a_bedrock_call_sends_the_url_python_joins_onto_the_callers_api_base( + api_base: str, native: str, sent: str +) -> None: + assert messages_host._sent_url({"model": _BEDROCK_INVOKE, "api_base": api_base}, native) == sent # pyright: ignore[reportPrivateUsage] # the route's own URL fix-up + + +def test_record_guardrail_bills_the_bedrock_trace_on_the_calls_logger() -> None: + logger: Final = Logging( + model=_BEDROCK_INVOKE, + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="guardrail", + function_id="guardrail", + ) + logger.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # the logger's untyped dict params + kwargs={}, + model=_BEDROCK_INVOKE, + optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}}, + litellm_params={"aws_region_name": "us-east-1"}, + custom_llm_provider="bedrock", + ) + trace: Final[dict[str, object]] = { + "guardrail": { + "input": {"gr-1": {"invocationMetrics": {"usage": {"contentPolicyUnits": 1}}}}, + "outputs": [], + } + } + + messages_host.record_guardrail( + {"id": "msg_1", "type": "message", "content": [], "amazon-bedrock-trace": trace}, + {"model": _BEDROCK_INVOKE, "litellm_logging_obj": logger}, + ) + + metadata: Final = logger.model_call_details["litellm_params"]["metadata"] # pyright: ignore[reportAny] # the logger's untyped details + assert metadata["standard_logging_guardrail_information"] + + +def test_a_bedrock_web_search_rejection_raises_the_bad_request_the_invoke_config_raises() -> None: + text: Final = "Bedrock does not support Anthropic's server-side web search tool" + native: Final = ValueError(text) + native.messages_request_error = True # pyright: ignore[reportAttributeAccessIssue] # set by the native host + native.messages_provider_rejection = text # pyright: ignore[reportAttributeAccessIssue] # set by the native host + native.messages_provider_bad_request = True # pyright: ignore[reportAttributeAccessIssue] # set by the native host + + mapped: Final = messages_host.map_failure(native, {"model": _BEDROCK_INVOKE, "messages": [], **_AWS}, "bedrock") + + assert type(mapped) is litellm.BadRequestError + assert str(mapped) == f"litellm.BadRequestError: {text}" + assert mapped.llm_provider == "bedrock" # pyright: ignore[reportUnknownMemberType] # the public error leaves llm_provider untyped + assert mapped.model == "us.anthropic.claude-sonnet-4-5-20250929-v1:0" # pyright: ignore[reportUnknownMemberType] # the public error leaves model untyped diff --git a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py index 7143a550f76..0962a1ae5ea 100644 --- a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py @@ -471,9 +471,19 @@ def test_stream_head_is_stamped_in_place_with_the_call_hidden_params() -> None: @pytest.mark.asyncio -async def test_a_native_stream_failure_logs_like_the_python_chunk_processor(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize( + ("relayed", "logs"), + [ + pytest.param(False, True, id="pass_through_logs_like_the_chunk_processor"), + pytest.param(True, False, id="the_native_relay_does_not"), + ], +) +async def test_a_native_stream_failure_logs_like_the_python_chunk_processor( + monkeypatch: pytest.MonkeyPatch, relayed: bool, logs: bool +) -> None: from litellm._logging import verbose_proxy_logger from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler + from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead logged: Final[list[tuple[object, ...]]] = [] scheduled: Final[list[Mapping[str, object]]] = [] @@ -487,9 +497,12 @@ async def test_a_native_stream_failure_logs_like_the_python_chunk_processor(monk logger.model_call_details["cache_hit"] = False cut: Final = httpx.ReadError("cut") - await legacy.stream_failure(logger, "anthropic", {}, [b"event: ping\n\n"], cut) + head: Final = MessagesStreamHead() + head.relayed = relayed - assert logged == [("Error in chunk_processor: %s", cut)] + await legacy.stream_failure(logger, "anthropic", {}, [b"event: ping\n\n"], cut, head) + + assert logged == ([("Error in chunk_processor: %s", cut)] if logs else []) assert "cache_hit" not in logger.model_call_details assert scheduled[0]["exception"] is cut diff --git a/tests/unit/rust_bridge/host/test_model_capabilities.py b/tests/unit/rust_bridge/host/test_model_capabilities.py index 501730431a7..8a2d13755d7 100644 --- a/tests/unit/rust_bridge/host/test_model_capabilities.py +++ b/tests/unit/rust_bridge/host/test_model_capabilities.py @@ -1,9 +1,12 @@ +import datetime from typing import Final import pytest import litellm -from litellm.rust_bridge.host.model_capabilities import anthropic_model_capabilities +from litellm.litellm_core_utils.fallback_generalizations import get_fallback_generalization_rules +from litellm.litellm_core_utils.model_cost_table import changed_in_place +from litellm.rust_bridge.host.model_capabilities import anthropic_model_capabilities, model_cost_document pytestmark = pytest.mark.usefixtures("local_model_cost_map") @@ -155,3 +158,47 @@ def test_the_cache_key_holds_the_entries_it_names(monkeypatch: pytest.MonkeyPatc assert any(h.value is entry for h in held[5]) assert held != same_content + + +def test_model_cost_document_carries_the_live_rows_and_the_active_rules(monkeypatch: pytest.MonkeyPatch) -> None: + _flag_model(monkeypatch, "claude-test-document", supports_tool_search=True) + changed_in_place() + + version, document = model_cost_document(None) + + assert document is not None + assert document["claude-test-document"]["supports_tool_search"] is True + assert document["fallback_generalizations"] == {"rules": list(get_fallback_generalization_rules())} + assert model_cost_document(version) == (version, None) + + +def test_model_cost_document_carries_registered_rows_with_non_json_values(monkeypatch: pytest.MonkeyPatch) -> None: + registered_at: Final = datetime.datetime(2026, 11, 15, tzinfo=datetime.timezone.utc) + monkeypatch.setitem( + litellm.model_cost, + "bedrock/invoke/claude-test-registered", + {"litellm_provider": "bedrock", "mode": "chat", "supports_tool_search": True, "created_at": registered_at}, + ) + changed_in_place() + + _, document = model_cost_document(None) + + assert document is not None + assert document["bedrock/invoke/claude-test-registered"] == { + "litellm_provider": "bedrock", + "mode": "chat", + "supports_tool_search": True, + "created_at": str(registered_at), + } + + +def test_model_cost_document_moves_with_a_write(monkeypatch: pytest.MonkeyPatch) -> None: + version, _ = model_cost_document(None) + + _flag_model(monkeypatch, "claude-test-document-write", supports_tool_search=True) + changed_in_place() + moved, document = model_cost_document(version) + + assert moved != version + assert document is not None + assert "claude-test-document-write" in document diff --git a/tests/unit/rust_bridge/host/test_settings.py b/tests/unit/rust_bridge/host/test_settings.py index c50a6758727..2282708eb95 100644 --- a/tests/unit/rust_bridge/host/test_settings.py +++ b/tests/unit/rust_bridge/host/test_settings.py @@ -230,6 +230,8 @@ def test_provider_defaults_read_the_litellm_globals(monkeypatch: pytest.MonkeyPa monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", True) monkeypatch.setattr(litellm, "api_key", "module-key") monkeypatch.setattr(litellm, "azure_key", "azure-module-key") + monkeypatch.setattr(litellm, "drop_params", True) + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ["team"]) assert settings.provider_defaults() == settings.ProviderDefaults( vertex_project="configured-project", @@ -237,4 +239,6 @@ def test_provider_defaults_read_the_litellm_globals(monkeypatch: pytest.MonkeyPa enable_azure_ad_token_refresh=True, api_key="module-key", azure_key="azure-module-key", + drop_params=True, + bedrock_request_metadata_fields=["team"], ) diff --git a/tests/unit/rust_bridge/messages/test_secrets.py b/tests/unit/rust_bridge/messages/test_secrets.py index 5ee89fa7794..297237d4bde 100644 --- a/tests/unit/rust_bridge/messages/test_secrets.py +++ b/tests/unit/rust_bridge/messages/test_secrets.py @@ -10,7 +10,7 @@ import pytest import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.llms.anthropic.messages.native.handler import anthropic_messages -from litellm.rust_bridge.host import settings +from litellm.rust_bridge.host import secret_manager, settings from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES from litellm.rust_bridge.public_call import NativeCall from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem @@ -19,6 +19,23 @@ from tests.test_litellm_rust.support.recording_server import ResponseSpec, recor pytestmark = pytest.mark.usefixtures("local_model_cost_map") +BEDROCK_MODEL: Final = "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" +SECRET_NAMES: Final = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_API_BASE", + "ANTHROPIC_BASE_URL", + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_BEDROCK_RUNTIME_ENDPOINT", + "AWS_REGION_NAME", + "AWS_REGION", + "AWS_DEFAULT_REGION", +) +CREDENTIALS: Final = { + MESSAGES_MODEL: ("ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "x-api-key", "{}"), + BEDROCK_MODEL: ("AWS_BEARER_TOKEN_BEDROCK", "AWS_BEDROCK_RUNTIME_ENDPOINT", "authorization", "Bearer {}"), +} + class Messages(Protocol): def __call__(self) -> Awaitable[object]: ... @@ -46,13 +63,13 @@ class _ManagedSecrets(CustomSecretManager): return self.values.get(secret_name) -def _native_request() -> NativeCall: - supplied: Final = _public_kwargs() +def _native_request(model: str = MESSAGES_MODEL) -> NativeCall: + supplied: Final = _public_kwargs(model) return NativeCall( args=(), kwargs=supplied, base={ - "model": MESSAGES_MODEL, + "model": model, "messages": MESSAGES, "max_tokens": 8, "stream": None, @@ -64,18 +81,18 @@ def _native_request() -> NativeCall: ) -def _public_kwargs() -> dict[str, object]: - return {"model": MESSAGES_MODEL, "messages": [dict(message) for message in MESSAGES], "max_tokens": 8} +def _public_kwargs(model: str = MESSAGES_MODEL) -> dict[str, object]: + return {"model": model, "messages": [dict(message) for message in MESSAGES], "max_tokens": 8} async def _python_messages() -> object: return await anthropic_messages(**_public_kwargs()) -async def _rust_amessages() -> object: +async def _rust_amessages(model: str = MESSAGES_MODEL) -> object: route: Final = NATIVE_AMESSAGES.load() assert route is not None - return await route(_native_request()) + return await route(_native_request(model)) @pytest.fixture(params=(_python_messages, _rust_amessages), ids=("python-async", "rust-async")) @@ -104,3 +121,52 @@ async def test_secret_manager_supplies_the_anthropic_key_and_base( assert len(server.requests) == 1 assert server.requests[0].headers["x-api-key"] == "vault-key" + + +def _recorded_secret_reads(monkeypatch: pytest.MonkeyPatch) -> list[str]: + reads: Final[list[str]] = [] + read_secret_str: Final = secret_manager.read_secret_str + + def recording(name: str) -> str | None: + reads.append(name) + return read_secret_str(name) + + monkeypatch.setattr(secret_manager, "read_secret_str", recording) + return reads + + +@pytest.mark.parametrize("model", tuple(CREDENTIALS), ids=("anthropic", "bedrock")) +async def test_environment_secrets_are_read_without_calling_python_until_a_manager_is_configured( + monkeypatch: pytest.MonkeyPatch, model: str +) -> None: + for name in SECRET_NAMES: + monkeypatch.delenv(name, raising=False) + key_name, base_name, header, shown = CREDENTIALS[model] + monkeypatch.setenv(key_name, "env-key") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + monkeypatch.setattr(litellm, "secret_manager_client", None) + monkeypatch.setattr(litellm, "_key_management_system", None) + monkeypatch.setattr(litellm, "_key_management_settings", None) + reads: Final = _recorded_secret_reads(monkeypatch) + with recording_service() as server: + server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + server.expected_requests = 2 + monkeypatch.setenv(base_name, server.base_url) + + await _rust_amessages(model) + unmanaged_reads: Final = tuple(reads) + + monkeypatch.setattr(litellm, "secret_manager_client", _ManagedSecrets({key_name: "vault-key"})) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + configured: Final = settings.secret_manager + monkeypatch.setattr(settings, "secret_manager", lambda: replace(configured(), native=False)) + + await _rust_amessages(model) + + assert unmanaged_reads == () + assert key_name in reads + assert [request.headers[header] for request in server.requests] == [ + shown.format("env-key"), + shown.format("vault-key"), + ] diff --git a/tests/unit/rust_bridge/test_catalog.py b/tests/unit/rust_bridge/test_catalog.py index 56fcad14851..d3cc99a03b9 100644 --- a/tests/unit/rust_bridge/test_catalog.py +++ b/tests/unit/rust_bridge/test_catalog.py @@ -146,7 +146,37 @@ def test_empty_constraints_match_nothing() -> None: ) assert catalog.decision(RouteContext(Route.OCR), rules) is Decision.PYTHON - assert ( - catalog.decision(RouteContext(Route.OCR, features=frozenset({Feature.STREAMING})), rules) - is Decision.PYTHON + assert catalog.decision(RouteContext(Route.OCR, features=frozenset({Feature.STREAMING})), rules) is Decision.PYTHON + + +@pytest.mark.parametrize( + ("context", "expected"), + ( + (RouteContext(Route.MESSAGES, provider="bedrock", target=Target.BEDROCK_INVOKE), Decision.RUST_REQUIRED), + (RouteContext(Route.MESSAGES, provider="bedrock", target=Target.MESSAGES), Decision.PYTHON), + (RouteContext(Route.MESSAGES, provider="bedrock", target=Target.CHAT_COMPLETIONS), Decision.PYTHON), + (RouteContext(Route.MESSAGES, provider="anthropic", target=Target.BEDROCK_INVOKE), Decision.PYTHON), + ( + RouteContext( + Route.MESSAGES, + provider="bedrock", + target=Target.BEDROCK_INVOKE, + features=frozenset({Feature.STREAMING}), + ), + Decision.PYTHON, + ), + ), +) +def test_a_provider_and_target_rule_admits_only_bedrock_invoke(context: RouteContext, expected: Decision) -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.STREAMING})), + RouteRule( + Route.MESSAGES, + Rollout.RUST_REQUIRED, + providers=frozenset({"bedrock"}), + targets=frozenset({Target.BEDROCK_INVOKE}), + ), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), ) + + assert catalog.decision(context, rules) is expected diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index 3fafb9e0f04..befa8682a3c 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -4,7 +4,8 @@ import asyncio import json import os from datetime import datetime, timedelta -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping +from contextlib import contextmanager from typing import Final from unittest.mock import AsyncMock, Mock @@ -23,6 +24,7 @@ from litellm.types.llms.openai import AllMessageValues import time from unittest.mock import patch, MagicMock +from tests.test_litellm_rust.support.recording_server import ResponseSpec, recording_service from tests.unit.llms.bedrock.slow_upstream import ( CONVERSE_ANSWER, STREAM_TIMEOUT_SECONDS, @@ -320,12 +322,26 @@ async def _streamed_messages_text(router: Router) -> str: return "".join(event["delta"]["text"] for event in events if event["type"] == "content_block_delta") +@contextmanager +def _slow_upstream(respx_mock: respx.MockRouter, served_by_rust: bool, answer: bytes = b"") -> Iterator[str | None]: + if not served_by_rust: + respx_mock.post(url__regex=r"https://(bedrock-runtime\.us-east-1\.amazonaws\.com|api\.openai\.com)/.*").mock( + side_effect=slow_upstream_answer(answer) + ) + yield None + return + with recording_service() as server: + server.expected_requests = None + server.default_response = ResponseSpec(body=answer, delay=UPSTREAM_FIRST_BYTE_SECONDS) + yield server.base_url + + @pytest.mark.asyncio @pytest.mark.parametrize( - ("model", "answer", "streamed_text"), + ("model", "answer", "streamed_text", "served_by_rust"), [ - ("bedrock/converse/anthropic.claude-sonnet-4-5-v1:0", CONVERSE_ANSWER, _streamed_chat_text), - ("bedrock/invoke/anthropic.claude-sonnet-4-5-v1:0", _INVOKE_ANSWER, _streamed_messages_text), + ("bedrock/converse/anthropic.claude-sonnet-4-5-v1:0", CONVERSE_ANSWER, _streamed_chat_text, False), + ("bedrock/invoke/anthropic.claude-sonnet-4-5-v1:0", _INVOKE_ANSWER, _streamed_messages_text, True), ], ids=["chat_completions", "anthropic_messages"], ) @@ -333,32 +349,32 @@ async def test_stream_a_deployment_hook_answers_in_one_piece_gets_the_longer_lim model: str, answer: bytes, streamed_text: Callable[[Router], Awaitable[str]], + served_by_rust: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(litellm, "callbacks", [_AnswersStreamsInOnePiece()]) - respx_mock.post(url__regex=r"https://bedrock-runtime\.us-east-1\.amazonaws\.com/.*").mock( - side_effect=slow_upstream_answer(answer) - ) - router: Final = Router( - model_list=[ - { - "model_name": "claude", - "litellm_params": { - "model": model, - "aws_access_key_id": "fake", - "aws_secret_access_key": "fake", - "aws_region_name": "us-east-1", - "timeout": STREAM_TIMEOUT_SECONDS, - "stream_timeout": UPSTREAM_FIRST_BYTE_SECONDS * 2, - }, - } - ], - num_retries=0, - ) + with _slow_upstream(respx_mock, served_by_rust, answer) as api_base: + router: Final = Router( + model_list=[ + { + "model_name": "claude", + "litellm_params": { + "model": model, + "api_base": api_base, + "aws_access_key_id": "fake", + "aws_secret_access_key": "fake", + "aws_region_name": "us-east-1", + "timeout": STREAM_TIMEOUT_SECONDS, + "stream_timeout": UPSTREAM_FIRST_BYTE_SECONDS * 2, + }, + } + ], + num_retries=0, + ) - assert await streamed_text(router) == "hi" + assert await streamed_text(router) == "hi" class _FailureDetails(CustomLogger): @@ -392,46 +408,46 @@ async def _drain_native_responses_stream(router: Router) -> None: @pytest.mark.asyncio @pytest.mark.parametrize( - ("model", "drain"), + ("model", "drain", "served_by_rust"), [ - ("bedrock/invoke/anthropic.claude-sonnet-4-5-v1:0", _drain_messages_stream), - ("openai/gpt-4.1", _drain_native_responses_stream), + ("bedrock/invoke/anthropic.claude-sonnet-4-5-v1:0", _drain_messages_stream, True), + ("openai/gpt-4.1", _drain_native_responses_stream, False), ], ids=["anthropic_messages", "responses"], ) async def test_native_live_stream_cut_at_the_stream_timeout_is_not_blamed_on_the_deployment( model: str, drain: Callable[[Router], Awaitable[None]], + served_by_rust: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) failure_details: Final = _FailureDetails() monkeypatch.setattr(litellm, "callbacks", [failure_details]) - respx_mock.post(url__regex=r"https://(bedrock-runtime\.us-east-1\.amazonaws\.com|api\.openai\.com)/.*").mock( - side_effect=slow_upstream_answer() - ) - router: Final = Router( - model_list=[ - { - "model_name": "native", - "litellm_params": { - "model": model, - "api_key": "fake", - "aws_access_key_id": "fake", - "aws_secret_access_key": "fake", - "aws_region_name": "us-east-1", - }, - } - ], - timeout=UPSTREAM_FIRST_BYTE_SECONDS * 2, - stream_timeout=STREAM_TIMEOUT_SECONDS, - num_retries=0, - ) + with _slow_upstream(respx_mock, served_by_rust) as api_base: + router: Final = Router( + model_list=[ + { + "model_name": "native", + "litellm_params": { + "model": model, + "api_base": api_base, + "api_key": "fake", + "aws_access_key_id": "fake", + "aws_secret_access_key": "fake", + "aws_region_name": "us-east-1", + }, + } + ], + timeout=UPSTREAM_FIRST_BYTE_SECONDS * 2, + stream_timeout=STREAM_TIMEOUT_SECONDS, + num_retries=0, + ) - with pytest.raises(litellm.Timeout): - await drain(router) - await asyncio.wait_for(failure_details.logged.wait(), timeout=10) + with pytest.raises(litellm.Timeout): + await drain(router) + await asyncio.wait_for(failure_details.logged.wait(), timeout=10) started: Final = failure_details.details.get("api_call_start_time") assert isinstance(started, datetime)