mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(messages): serve bedrock invoke messages on rust only (#51330)
This commit is contained in:
parent
d0bf848d91
commit
8d200a4f98
114 changed files with 11703 additions and 1468 deletions
|
|
@ -6,6 +6,9 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele
|
|||
from litellm.exceptions import BadRequestError, UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.anthropic.messages.native import handler as main
|
||||
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import (
|
||||
AmazonAnthropicClaudeMessagesConfig,
|
||||
)
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.rust_bridge import catalog, detectors
|
||||
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
|
||||
|
|
@ -105,8 +108,11 @@ def _route_target(
|
|||
api_base: str | None,
|
||||
compacting: bool,
|
||||
) -> Target:
|
||||
if main._provider_messages_config(model, provider, api_base) is not None: # pyright: ignore[reportPrivateUsage] # the handler's own config lookup
|
||||
return Target.MESSAGES
|
||||
provider_config: Final = main._provider_messages_config(model, provider, api_base) # pyright: ignore[reportPrivateUsage] # the handler's own config lookup
|
||||
if provider_config is not None:
|
||||
return (
|
||||
Target.BEDROCK_INVOKE if type(provider_config) is AmazonAnthropicClaudeMessagesConfig else Target.MESSAGES
|
||||
)
|
||||
if main._deployment_passes_through_anthropic_messages(model_info): # pyright: ignore[reportPrivateUsage] # the handler's own passthrough opt-in
|
||||
return Target.MESSAGES_PASSTHROUGH
|
||||
if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
|||
snapshot_tuning_baselines,
|
||||
tuning_limit_violation,
|
||||
)
|
||||
from litellm.rust_bridge.catalog import Target
|
||||
from litellm.rust_bridge.catalog import NATIVE_MESSAGES_TARGETS, Target
|
||||
from litellm.types.utils import (
|
||||
PRICING_OVERRIDES_KEY,
|
||||
CallTypes,
|
||||
|
|
@ -16130,7 +16130,6 @@ def _count_context_management(
|
|||
|
||||
|
||||
_GATEWAY_EDIT_ROUTES: Final = frozenset({Target.CHAT_COMPLETIONS, Target.RESPONSES})
|
||||
_MESSAGES_ROUTES: Final = frozenset({Target.MESSAGES, Target.MESSAGES_PASSTHROUGH})
|
||||
|
||||
|
||||
def _messages_route(deployment: _TokenCountDeployment, request_model: str) -> Target:
|
||||
|
|
@ -16669,7 +16668,7 @@ async def _count_request_tokens(
|
|||
|
||||
local_view: Final = (
|
||||
None
|
||||
if route not in _MESSAGES_ROUTES or context_edits is None or request_messages is None
|
||||
if route not in NATIVE_MESSAGES_TARGETS or context_edits is None or request_messages is None
|
||||
else await _bridged_request_view(
|
||||
request_messages,
|
||||
system,
|
||||
|
|
|
|||
64
litellm/rust/Cargo.lock
generated
64
litellm/rust/Cargo.lock
generated
|
|
@ -258,6 +258,8 @@ checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319"
|
|||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-sdk-sso",
|
||||
"aws-sdk-ssooidc",
|
||||
"aws-sdk-sts",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
|
|
@ -269,11 +271,14 @@ dependencies = [
|
|||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"hex",
|
||||
"http 1.4.2",
|
||||
"sha1 0.10.7",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -428,6 +433,58 @@ dependencies = [
|
|||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "1.111.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-observability",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "1.113.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-observability",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.108.0"
|
||||
|
|
@ -3455,6 +3512,7 @@ dependencies = [
|
|||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures-util",
|
||||
"litellm-auth-types",
|
||||
"litellm-error",
|
||||
|
|
@ -3467,6 +3525,7 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"url",
|
||||
|
|
@ -4004,6 +4063,7 @@ dependencies = [
|
|||
"litellm-auth",
|
||||
"litellm-cache-memory",
|
||||
"litellm-cache-response",
|
||||
"litellm-core-utils",
|
||||
"litellm-error",
|
||||
"litellm-gateway-auth",
|
||||
"litellm-host",
|
||||
|
|
@ -4269,6 +4329,7 @@ dependencies = [
|
|||
"litellm-inference-testing",
|
||||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-python-compat",
|
||||
"litellm-router-types",
|
||||
"litellm-secret-value",
|
||||
"litellm-secrets",
|
||||
|
|
@ -4391,6 +4452,7 @@ dependencies = [
|
|||
"aws-smithy-types",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"crc32fast",
|
||||
"data-url",
|
||||
"fancy-regex 0.19.2",
|
||||
"futures-util",
|
||||
|
|
@ -4420,6 +4482,7 @@ dependencies = [
|
|||
"thiserror 2.0.19",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"url",
|
||||
]
|
||||
|
||||
|
|
@ -4488,6 +4551,7 @@ name = "litellm-python-bridge"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-credential-types",
|
||||
"aws-sdk-secretsmanager",
|
||||
"azure_core",
|
||||
"bytes",
|
||||
|
|
|
|||
|
|
@ -16,12 +16,13 @@ litellm-secret-value.workspace = true
|
|||
serde.workspace = true
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
sha2.workspace = true
|
||||
chrono.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
url.workspace = true
|
||||
veil.workspace = true
|
||||
|
||||
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] }
|
||||
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process", "sso"] }
|
||||
aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] }
|
||||
aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] }
|
||||
aws-sigv4 = "1.5.1"
|
||||
|
|
@ -33,6 +34,7 @@ litellm-secrets.workspace = true
|
|||
aws-smithy-types = "1.6.4"
|
||||
aws-runtime = { version = "1.9.4", default-features = false }
|
||||
rstest.workspace = true
|
||||
tempfile = "3"
|
||||
litellm-http = { workspace = true, features = ["test-support"] }
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1 @@
|
|||
{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream","bedrock:CountTokens","bedrock:Rerank","bedrock:Retrieve","bedrock:ListKnowledgeBases","bedrock:InvokeAgent","bedrock:ApplyGuardrail","bedrock:GetGuardrail","bedrock:ListGuardrails"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"BedrockAgentCoreLiteLLM","Effect":"Allow","Action":["bedrock-agentcore:InvokeAgentRuntime","bedrock-agentcore:InvokeAgentRuntimeForUser","bedrock-agentcore:InvokeGateway"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformLiteLLM","Effect":"Allow","Action":["aws-external-anthropic:CreateInference","aws-external-anthropic:CreateBatchInference","aws-external-anthropic:CancelBatchInference","aws-external-anthropic:DeleteBatchInference","aws-external-anthropic:CountTokens","aws-external-anthropic:Get*","aws-external-anthropic:List*"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformOutboundIdentityLiteLLM","Effect":"Allow","Action":["sts:GetWebIdentityToken","sts:TagGetWebIdentityToken"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringEquals":{"aws:CalledViaLast":"aws-external-anthropic.amazonaws.com"}}},{"Sid":"BedrockMantleLiteLLM","Effect":"Allow","Action":["bedrock-mantle:CreateInference","bedrock-mantle:CountTokens"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]}
|
||||
|
|
@ -42,5 +42,6 @@ pub const SIGV4_COMPUTED_HEADER_NAMES: &[&str] = &[
|
|||
pub const BEDROCK_SERVICE: &str = "bedrock";
|
||||
pub const DEFAULT_SESSION_NAME_PREFIX: &str = "litellm-session";
|
||||
pub const DEFAULT_BEDROCK_REGION: &str = "us-west-2";
|
||||
pub const BEDROCK_RUNTIME_ENDPOINT_TEMPLATE: &str =
|
||||
"https://bedrock-runtime.{region}.amazonaws.com";
|
||||
pub const BEDROCK_SESSION_POLICY: &str = include_str!("bedrock_session_policy.json");
|
||||
pub const WEB_IDENTITY_SESSION_SECONDS: i32 = 3600;
|
||||
pub const AWS4_AUTHORIZATION_PREFIX: &str = "AWS4-HMAC-SHA256";
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use aws_credential_types::provider::error::CredentialsError;
|
||||
use litellm_error::Source;
|
||||
|
||||
#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)]
|
||||
|
|
@ -10,6 +11,14 @@ pub enum Error {
|
|||
#[source]
|
||||
source: Source,
|
||||
},
|
||||
#[error(
|
||||
"the AWS web identity token must name a secret or an oidc/ provider, not an environment variable"
|
||||
)]
|
||||
WebIdentityTokenReference,
|
||||
#[error("the AWS web identity token reference resolved to no token")]
|
||||
MissingWebIdentityToken,
|
||||
#[error("could not read the AWS web identity token")]
|
||||
WebIdentityToken(#[source] Source),
|
||||
#[error("AWS web identity response had no credentials")]
|
||||
MissingWebIdentityCredentials,
|
||||
#[error("AWS web identity response had an invalid expiration")]
|
||||
|
|
@ -49,6 +58,20 @@ pub enum SigningStage {
|
|||
}
|
||||
|
||||
impl Error {
|
||||
pub fn is_empty_default_chain(&self) -> bool {
|
||||
let Self::Credentials {
|
||||
flow: CredentialFlow::DefaultChain,
|
||||
source,
|
||||
} = self
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
matches!(
|
||||
source.downcast_ref::<CredentialsError>(),
|
||||
Some(CredentialsError::CredentialsNotLoaded(_))
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn credentials<E: std::error::Error + Send + Sync + 'static>(
|
||||
flow: CredentialFlow,
|
||||
) -> impl FnOnce(E) -> Self {
|
||||
|
|
@ -73,3 +96,40 @@ impl From<Error> for litellm_auth_types::Error {
|
|||
Self::provider(error)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::empty_default_chain(
|
||||
CredentialFlow::DefaultChain,
|
||||
CredentialsError::not_loaded("no providers in chain provided credentials"),
|
||||
true
|
||||
)]
|
||||
#[case::failed_default_chain(
|
||||
CredentialFlow::DefaultChain,
|
||||
CredentialsError::provider_error("IMDS answered 500"),
|
||||
false
|
||||
)]
|
||||
#[case::empty_profile(
|
||||
CredentialFlow::Profile,
|
||||
CredentialsError::not_loaded("the profile has no credentials"),
|
||||
false
|
||||
)]
|
||||
fn only_a_default_chain_that_loaded_nothing_is_empty(
|
||||
#[case] flow: CredentialFlow,
|
||||
#[case] failure: CredentialsError,
|
||||
#[case] empty: bool,
|
||||
) {
|
||||
assert_eq!(
|
||||
Error::credentials(flow)(failure).is_empty_default_chain(),
|
||||
empty
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_web_identity_failure_is_not_an_empty_default_chain() {
|
||||
assert!(!Error::MissingWebIdentityToken.is_empty_default_chain());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@ mod aws;
|
|||
pub mod constants;
|
||||
mod error;
|
||||
mod provider;
|
||||
mod region;
|
||||
mod signer;
|
||||
|
||||
pub use aws::*;
|
||||
pub use aws_credential_types::Credentials;
|
||||
pub use error::{CredentialFlow, Error, SigningStage};
|
||||
pub use provider::AwsCredentialsProvider;
|
||||
pub use signer::SigV4Signer;
|
||||
pub use region::{AwsPartition, aws_partition, bedrock_runtime_endpoint};
|
||||
pub use signer::{CallerSigningHeaders, SigV4Signer};
|
||||
|
|
|
|||
239
litellm/rust/crates/auth-aws/src/region.rs
Normal file
239
litellm/rust/crates/auth-aws/src/region.rs
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
use std::path::PathBuf;
|
||||
|
||||
use crate::constants::AWS_DEFAULT_REGION;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct AwsPartition {
|
||||
pub partition: &'static str,
|
||||
pub dns_suffix: &'static str,
|
||||
}
|
||||
|
||||
const COMMERCIAL_PARTITION: AwsPartition = AwsPartition {
|
||||
partition: "aws",
|
||||
dns_suffix: "amazonaws.com",
|
||||
};
|
||||
|
||||
const PARTITIONS_BY_REGION_PREFIX: [(&str, AwsPartition); 6] = [
|
||||
(
|
||||
"cn-",
|
||||
AwsPartition {
|
||||
partition: "aws-cn",
|
||||
dns_suffix: "amazonaws.com.cn",
|
||||
},
|
||||
),
|
||||
(
|
||||
"us-gov-",
|
||||
AwsPartition {
|
||||
partition: "aws-us-gov",
|
||||
dns_suffix: "amazonaws.com",
|
||||
},
|
||||
),
|
||||
(
|
||||
"us-isob-",
|
||||
AwsPartition {
|
||||
partition: "aws-iso-b",
|
||||
dns_suffix: "sc2s.sgov.gov",
|
||||
},
|
||||
),
|
||||
(
|
||||
"us-isof-",
|
||||
AwsPartition {
|
||||
partition: "aws-iso-f",
|
||||
dns_suffix: "csp.hci.ic.gov",
|
||||
},
|
||||
),
|
||||
(
|
||||
"us-iso-",
|
||||
AwsPartition {
|
||||
partition: "aws-iso",
|
||||
dns_suffix: "c2s.ic.gov",
|
||||
},
|
||||
),
|
||||
(
|
||||
"eu-isoe-",
|
||||
AwsPartition {
|
||||
partition: "aws-iso-e",
|
||||
dns_suffix: "cloud.adc-e.uk",
|
||||
},
|
||||
),
|
||||
];
|
||||
|
||||
pub fn aws_partition(region: &str) -> AwsPartition {
|
||||
PARTITIONS_BY_REGION_PREFIX
|
||||
.iter()
|
||||
.find(|(prefix, _)| region.starts_with(prefix))
|
||||
.map_or(COMMERCIAL_PARTITION, |(_, partition)| *partition)
|
||||
}
|
||||
|
||||
pub fn bedrock_runtime_endpoint(region: &str) -> String {
|
||||
format!(
|
||||
"https://bedrock-runtime.{region}.{}",
|
||||
aws_partition(region).dns_suffix
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn session_region(
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
process_env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
env_lookup(AWS_DEFAULT_REGION)
|
||||
.filter(|region| !region.is_empty())
|
||||
.or_else(|| profile_region(process_env))
|
||||
}
|
||||
|
||||
fn profile_region(process_env: &dyn Fn(&str) -> Option<String>) -> Option<String> {
|
||||
let profile = process_env("AWS_PROFILE")
|
||||
.or_else(|| process_env("AWS_DEFAULT_PROFILE"))
|
||||
.filter(|profile| !profile.is_empty())
|
||||
.unwrap_or_else(|| "default".to_string());
|
||||
let home = process_env("HOME");
|
||||
let path = match process_env("AWS_CONFIG_FILE").filter(|path| !path.is_empty()) {
|
||||
Some(path) => match (path.strip_prefix("~/"), &home) {
|
||||
(Some(rest), Some(home)) => PathBuf::from(home).join(rest),
|
||||
_ => PathBuf::from(path),
|
||||
},
|
||||
None => PathBuf::from(home?).join(".aws").join("config"),
|
||||
};
|
||||
config_file_region(&std::fs::read_to_string(path).ok()?, &profile)
|
||||
}
|
||||
|
||||
fn config_file_region(contents: &str, profile: &str) -> Option<String> {
|
||||
let named = format!("profile {profile}");
|
||||
let selects = |section: &str| section == named || (profile == "default" && section == profile);
|
||||
let mut selected = false;
|
||||
for line in contents.lines().map(str::trim) {
|
||||
if line.is_empty() || line.starts_with('#') || line.starts_with(';') {
|
||||
continue;
|
||||
}
|
||||
if let Some(section) = line
|
||||
.strip_prefix('[')
|
||||
.and_then(|line| line.strip_suffix(']'))
|
||||
{
|
||||
selected = selects(§ion.split_whitespace().collect::<Vec<_>>().join(" "));
|
||||
continue;
|
||||
}
|
||||
if selected
|
||||
&& let Some((key, value)) = line.split_once('=')
|
||||
&& key.trim() == "region"
|
||||
{
|
||||
return Some(value.trim().to_string()).filter(|region| !region.is_empty());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(crate) fn sts_endpoint_region(endpoint: &str) -> Option<String> {
|
||||
let url = url::Url::parse(endpoint).ok()?;
|
||||
let labels: Vec<&str> = url.host_str()?.split('.').collect();
|
||||
(0..labels.len()).find_map(|index| {
|
||||
let ["sts" | "sts-fips", region, rest @ ..] = &labels[index..] else {
|
||||
return None;
|
||||
};
|
||||
let rest = rest.join(".");
|
||||
let valid_region = !region.is_empty()
|
||||
&& region
|
||||
.chars()
|
||||
.all(|char| char.is_ascii_lowercase() || char.is_ascii_digit() || char == '-');
|
||||
(valid_region
|
||||
&& (rest.starts_with("amazonaws.com") || rest.starts_with("vpce.amazonaws.com")))
|
||||
.then(|| region.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::commercial("us-east-1", "https://bedrock-runtime.us-east-1.amazonaws.com")]
|
||||
#[case::china("cn-north-1", "https://bedrock-runtime.cn-north-1.amazonaws.com.cn")]
|
||||
#[case::gov_cloud("us-gov-west-1", "https://bedrock-runtime.us-gov-west-1.amazonaws.com")]
|
||||
#[case::iso("us-iso-east-1", "https://bedrock-runtime.us-iso-east-1.c2s.ic.gov")]
|
||||
#[case::iso_b(
|
||||
"us-isob-east-1",
|
||||
"https://bedrock-runtime.us-isob-east-1.sc2s.sgov.gov"
|
||||
)]
|
||||
#[case::iso_f(
|
||||
"us-isof-south-1",
|
||||
"https://bedrock-runtime.us-isof-south-1.csp.hci.ic.gov"
|
||||
)]
|
||||
#[case::iso_e(
|
||||
"eu-isoe-west-1",
|
||||
"https://bedrock-runtime.eu-isoe-west-1.cloud.adc-e.uk"
|
||||
)]
|
||||
fn the_runtime_endpoint_uses_the_partition_dns_suffix(
|
||||
#[case] region: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(bedrock_runtime_endpoint(region), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::china("cn-northwest-1", "aws-cn")]
|
||||
#[case::gov_cloud("us-gov-east-1", "aws-us-gov")]
|
||||
#[case::commercial("eu-central-1", "aws")]
|
||||
fn a_region_names_its_partition(#[case] region: &str, #[case] expected: &str) {
|
||||
assert_eq!(aws_partition(region).partition, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::regional("https://sts.eu-west-1.amazonaws.com", Some("eu-west-1"))]
|
||||
#[case::fips("https://sts-fips.us-east-2.amazonaws.com/", Some("us-east-2"))]
|
||||
#[case::china("https://sts.cn-north-1.amazonaws.com.cn", Some("cn-north-1"))]
|
||||
#[case::vpc_endpoint(
|
||||
"https://vpce-0abc-1234.sts.ap-south-1.vpce.amazonaws.com",
|
||||
Some("ap-south-1")
|
||||
)]
|
||||
#[case::global("https://sts.amazonaws.com", None)]
|
||||
#[case::custom_host("https://sts.internal.example.com", None)]
|
||||
#[case::no_scheme("sts.eu-west-1.amazonaws.com", None)]
|
||||
fn the_sts_region_is_read_from_a_regional_endpoint(
|
||||
#[case] endpoint: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(sts_endpoint_region(endpoint).as_deref(), expected);
|
||||
}
|
||||
|
||||
const CONFIG: &str = "[default]\nregion = eu-west-3\n\n[profile work]\noutput = json\nregion=ap-northeast-1\n\n[profile empty]\nregion =\n";
|
||||
|
||||
#[rstest]
|
||||
#[case::default_profile("default", Some("eu-west-3"))]
|
||||
#[case::named_profile("work", Some("ap-northeast-1"))]
|
||||
#[case::blank_region("empty", None)]
|
||||
#[case::unknown_profile("missing", None)]
|
||||
fn the_profile_region_comes_from_its_config_section(
|
||||
#[case] profile: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(config_file_region(CONFIG, profile).as_deref(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default_region_wins(Some("sa-east-1"), None, Some("sa-east-1"))]
|
||||
#[case::blank_default_region_reads_the_profile(Some(""), Some("work"), Some("ap-northeast-1"))]
|
||||
#[case::profile_from_the_environment(None, Some("work"), Some("ap-northeast-1"))]
|
||||
#[case::default_profile(None, None, Some("eu-west-3"))]
|
||||
fn the_session_region_reads_the_default_region_then_the_profile(
|
||||
#[case] default_region: Option<&str>,
|
||||
#[case] profile: Option<&str>,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let file = tempfile::NamedTempFile::new().unwrap();
|
||||
std::fs::write(file.path(), CONFIG).unwrap();
|
||||
let config = file.path().to_string_lossy().into_owned();
|
||||
let env = |name: &str| {
|
||||
(name == AWS_DEFAULT_REGION)
|
||||
.then(|| default_region.map(str::to_string))
|
||||
.flatten()
|
||||
};
|
||||
let process_env = |name: &str| match name {
|
||||
"AWS_CONFIG_FILE" => Some(config.clone()),
|
||||
"AWS_PROFILE" => profile.map(str::to_string),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
assert_eq!(session_region(&env, &process_env).as_deref(), expected);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +1,18 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use aws_credential_types::Credentials;
|
||||
use reqwest::{Request, header::HeaderMap};
|
||||
use aws_sigv4::sign::v4;
|
||||
use chrono::{DateTime, Utc};
|
||||
use reqwest::{
|
||||
Request,
|
||||
header::{HeaderMap, HeaderName, HeaderValue},
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::{
|
||||
AwsAuthService, AwsCredentialSource, Error, SigningScope, aws_signature_headers,
|
||||
is_sigv4_computed_header, sign_headers,
|
||||
AwsAuthService, AwsCredentialSource, Error, SigningScope, SigningStage, aws_signature_headers,
|
||||
constants::AWS4_AUTHORIZATION_PREFIX, is_sigv4_computed_header, sign_headers,
|
||||
};
|
||||
use litellm_http::outbound::RequestSigner;
|
||||
|
||||
|
|
@ -15,6 +22,14 @@ pub struct SigV4Signer {
|
|||
service: &'static str,
|
||||
credentials: Credentials,
|
||||
clock: fn() -> SystemTime,
|
||||
caller_headers: CallerSigningHeaders,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum CallerSigningHeaders {
|
||||
#[default]
|
||||
Dropped,
|
||||
Botocore,
|
||||
}
|
||||
|
||||
impl SigV4Signer {
|
||||
|
|
@ -24,6 +39,7 @@ impl SigV4Signer {
|
|||
service,
|
||||
credentials,
|
||||
clock: SystemTime::now,
|
||||
caller_headers: CallerSigningHeaders::Dropped,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -31,6 +47,18 @@ impl SigV4Signer {
|
|||
Self { clock, ..self }
|
||||
}
|
||||
|
||||
pub fn with_caller_headers(self, caller_headers: CallerSigningHeaders) -> Self {
|
||||
Self {
|
||||
caller_headers,
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
fn keeps_caller_security_token(&self) -> bool {
|
||||
self.caller_headers == CallerSigningHeaders::Botocore
|
||||
&& self.credentials.session_token().is_none()
|
||||
}
|
||||
|
||||
pub async fn resolve(
|
||||
auth: &AwsAuthService,
|
||||
region: String,
|
||||
|
|
@ -38,48 +66,246 @@ impl SigV4Signer {
|
|||
credentials: AwsCredentialSource,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<Self, Error> {
|
||||
Ok(Self::new(
|
||||
region,
|
||||
service,
|
||||
credentials.resolve(auth, env_lookup).await?,
|
||||
))
|
||||
let credentials = credentials
|
||||
.resolve(auth, ®ion, service, env_lookup)
|
||||
.await?;
|
||||
Ok(Self::new(region, service, credentials))
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestSigner for SigV4Signer {
|
||||
fn sign(&self, request: &Request) -> Result<HeaderMap, litellm_http::Error> {
|
||||
if let Some(name) = request
|
||||
.headers()
|
||||
.keys()
|
||||
.find(|name| is_sigv4_computed_header(name.as_str()))
|
||||
{
|
||||
return Err(litellm_http::Error::computed_header(name.as_str()));
|
||||
const CALLER_HEADERS_THE_SIGNER_REPLACES: &[&str] = &["x-amz-date", "x-amz-security-token", "date"];
|
||||
|
||||
const CALLER_DATE_HEADERS: &[&str] = &["x-amz-date", "date"];
|
||||
|
||||
const SECURITY_TOKEN: &str = "x-amz-security-token";
|
||||
|
||||
const BOTOCORE_UNSIGNED_HEADERS: &[&str] = &["expect", "user-agent", "x-amzn-trace-id"];
|
||||
|
||||
fn signable(headers: &HeaderMap, keep_security_token: bool) -> HeaderMap {
|
||||
aws_signature_headers(headers)
|
||||
.iter()
|
||||
.filter(|(name, _)| {
|
||||
!is_sigv4_computed_header(name.as_str())
|
||||
|| (keep_security_token && name.as_str() == SECURITY_TOKEN)
|
||||
})
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn hex_sha256(bytes: &[u8]) -> String {
|
||||
Sha256::digest(bytes)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn quote_path(path: &str) -> String {
|
||||
path.bytes()
|
||||
.map(|byte| match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' | b'/' => {
|
||||
char::from(byte).to_string()
|
||||
}
|
||||
other => format!("%{other:02X}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn canonical_uri(path: &str) -> String {
|
||||
if path.is_empty() {
|
||||
return "/".to_string();
|
||||
}
|
||||
let segments = path.split('/').fold(Vec::new(), |mut kept, segment| {
|
||||
match segment {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
kept.pop();
|
||||
}
|
||||
other => kept.push(other),
|
||||
}
|
||||
kept
|
||||
});
|
||||
let leading = if path.starts_with('/') { "/" } else { "" };
|
||||
let trailing = if path.ends_with('/') && !segments.is_empty() {
|
||||
"/"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
quote_path(&format!("{leading}{}{trailing}", segments.join("/")))
|
||||
}
|
||||
|
||||
fn canonical_query(query: Option<&str>) -> String {
|
||||
let mut pairs: Vec<(&str, &str)> = query
|
||||
.filter(|query| !query.is_empty())
|
||||
.map(|query| {
|
||||
query
|
||||
.split('&')
|
||||
.map(|pair| pair.split_once('=').unwrap_or((pair, "")))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
pairs.sort_unstable();
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(key, value)| format!("{key}={value}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("&")
|
||||
}
|
||||
|
||||
fn canonical_host(url: &reqwest::Url) -> String {
|
||||
let host = url.host_str().unwrap_or_default();
|
||||
match url.port() {
|
||||
Some(port) => format!("{host}:{port}"),
|
||||
None => host.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn header_pair(name: &str, value: &str) -> Result<(HeaderName, HeaderValue), Error> {
|
||||
Ok((
|
||||
HeaderName::from_bytes(name.as_bytes()).map_err(Error::signing(SigningStage::Output))?,
|
||||
HeaderValue::from_str(value).map_err(Error::signing(SigningStage::Output))?,
|
||||
))
|
||||
}
|
||||
|
||||
fn sign_with_date_header(
|
||||
request: &Request,
|
||||
body: &[u8],
|
||||
headers: &HeaderMap,
|
||||
scope: SigningScope<'_>,
|
||||
) -> Result<HeaderMap, Error> {
|
||||
let time: DateTime<Utc> = scope.time.into();
|
||||
let amz_date = time.format("%Y%m%dT%H%M%SZ").to_string();
|
||||
let http_date = time.format("%a, %d %b %Y %H:%M:%S -0000").to_string();
|
||||
let session_token = scope.credentials.session_token();
|
||||
let mut canonical: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
for (name, value) in headers {
|
||||
if BOTOCORE_UNSIGNED_HEADERS.contains(&name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let value = value.to_str().map_err(|_| Error::NonAsciiHeader {
|
||||
name: name.to_string(),
|
||||
})?;
|
||||
canonical
|
||||
.entry(name.as_str().to_string())
|
||||
.or_default()
|
||||
.push(value.split_whitespace().collect::<Vec<_>>().join(" "));
|
||||
}
|
||||
canonical.insert("date".into(), vec![http_date.clone()]);
|
||||
if let Some(token) = session_token {
|
||||
canonical.insert(SECURITY_TOKEN.into(), vec![token.to_string()]);
|
||||
}
|
||||
canonical
|
||||
.entry("host".into())
|
||||
.or_insert_with(|| vec![canonical_host(request.url())]);
|
||||
let payload = headers
|
||||
.get("x-amz-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map_or_else(|| hex_sha256(body), str::to_string);
|
||||
let canonical_headers: String = canonical
|
||||
.iter()
|
||||
.map(|(name, values)| format!("{name}:{}\n", values.join(",")))
|
||||
.collect();
|
||||
let signed_headers = canonical.keys().cloned().collect::<Vec<_>>().join(";");
|
||||
let canonical_request = format!(
|
||||
"{}\n{}\n{}\n{canonical_headers}\n{signed_headers}\n{payload}",
|
||||
request.method().as_str(),
|
||||
canonical_uri(request.url().path()),
|
||||
canonical_query(request.url().query()),
|
||||
);
|
||||
let credential_scope = format!(
|
||||
"{}/{}/{}/aws4_request",
|
||||
&amz_date[..8],
|
||||
scope.region,
|
||||
scope.service
|
||||
);
|
||||
let string_to_sign = format!(
|
||||
"{AWS4_AUTHORIZATION_PREFIX}\n{amz_date}\n{credential_scope}\n{}",
|
||||
hex_sha256(canonical_request.as_bytes())
|
||||
);
|
||||
let signing_key = v4::generate_signing_key(
|
||||
scope.credentials.secret_access_key(),
|
||||
scope.time,
|
||||
scope.region,
|
||||
scope.service,
|
||||
);
|
||||
let signature = v4::calculate_signature(signing_key, string_to_sign.as_bytes());
|
||||
let authorization = format!(
|
||||
"{AWS4_AUTHORIZATION_PREFIX} Credential={}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}",
|
||||
scope.credentials.access_key_id()
|
||||
);
|
||||
[
|
||||
Some(("date", http_date.as_str())),
|
||||
session_token.map(|token| (SECURITY_TOKEN, token)),
|
||||
Some(("authorization", authorization.as_str())),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|(name, value)| header_pair(name, value))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn overrides_the_signature(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get(reqwest::header::AUTHORIZATION)
|
||||
.is_some_and(|value| {
|
||||
!value
|
||||
.as_bytes()
|
||||
.starts_with(AWS4_AUTHORIZATION_PREFIX.as_bytes())
|
||||
})
|
||||
}
|
||||
|
||||
impl RequestSigner for SigV4Signer {
|
||||
fn owned_headers(&self) -> &'static [&'static str] {
|
||||
if self.keeps_caller_security_token() {
|
||||
CALLER_DATE_HEADERS
|
||||
} else {
|
||||
CALLER_HEADERS_THE_SIGNER_REPLACES
|
||||
}
|
||||
}
|
||||
|
||||
fn sign(&self, request: &Request) -> Result<HeaderMap, litellm_http::Error> {
|
||||
let body = match request.body() {
|
||||
None => &[],
|
||||
Some(body) => body
|
||||
.as_bytes()
|
||||
.ok_or_else(litellm_http::Error::unsignable_body)?,
|
||||
};
|
||||
sign_headers(
|
||||
request.method().as_str(),
|
||||
request.url().as_str(),
|
||||
body,
|
||||
&aws_signature_headers(request.headers()),
|
||||
SigningScope {
|
||||
region: &self.region,
|
||||
service: self.service,
|
||||
credentials: &self.credentials,
|
||||
time: (self.clock)(),
|
||||
},
|
||||
)
|
||||
let headers = signable(request.headers(), self.keeps_caller_security_token());
|
||||
let scope = SigningScope {
|
||||
region: &self.region,
|
||||
service: self.service,
|
||||
credentials: &self.credentials,
|
||||
time: (self.clock)(),
|
||||
};
|
||||
let signed = match self.caller_headers == CallerSigningHeaders::Botocore
|
||||
&& request.headers().contains_key(reqwest::header::DATE)
|
||||
{
|
||||
true => sign_with_date_header(request, body, &headers, scope),
|
||||
false => sign_headers(
|
||||
request.method().as_str(),
|
||||
request.url().as_str(),
|
||||
body,
|
||||
&headers,
|
||||
scope,
|
||||
),
|
||||
}
|
||||
.map_err(|error| match error {
|
||||
Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error),
|
||||
Error::ExpiredCredentials
|
||||
| Error::Credentials { .. }
|
||||
| Error::WebIdentityTokenReference
|
||||
| Error::MissingWebIdentityToken
|
||||
| Error::WebIdentityToken(_)
|
||||
| Error::MissingWebIdentityCredentials
|
||||
| Error::WebIdentityExpiration(_)
|
||||
| Error::Signing { .. } => litellm_http::Error::signature(error),
|
||||
})?;
|
||||
Ok(match overrides_the_signature(request.headers()) {
|
||||
true => signed
|
||||
.iter()
|
||||
.filter(|(name, _)| *name != reqwest::header::AUTHORIZATION)
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
.collect(),
|
||||
false => signed,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -176,22 +402,166 @@ mod tests {
|
|||
assert!(original.contains("/us-east-1/textract/aws4_request"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_forwarded_computed_header_is_refused_instead_of_sent_twice() {
|
||||
let error = signed(
|
||||
fn sent(headers: &[(&str, &str)], credentials: Credentials) -> Request {
|
||||
signed(
|
||||
json(
|
||||
URL,
|
||||
&[("authorization".into(), "Bearer caller".into())],
|
||||
&headers
|
||||
.iter()
|
||||
.map(|(name, value)| (name.to_string(), value.to_string()))
|
||||
.collect::<Vec<_>>(),
|
||||
&json_value!({}),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
&signer("textract"),
|
||||
&SigV4Signer::new("us-east-1".into(), "bedrock", credentials).with_clock(fixed_clock),
|
||||
)
|
||||
.unwrap_err();
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
assert!(error.is_computed_header());
|
||||
assert!(error.to_string().contains("authorization"), "{error}");
|
||||
fn header<'a>(request: &'a Request, name: &str) -> Vec<&'a str> {
|
||||
request
|
||||
.headers()
|
||||
.get_all(name)
|
||||
.iter()
|
||||
.map(|value| value.to_str().unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::caller_date("date", None)]
|
||||
#[case::caller_amz_date("x-amz-date", Some("20231114T221320Z"))]
|
||||
#[case::caller_security_token_without_a_session("x-amz-security-token", None)]
|
||||
fn a_header_the_signer_computes_is_dropped_from_the_callers_copy(
|
||||
#[case] name: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let request = sent(
|
||||
&[(name, "caller-value")],
|
||||
Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
header(&request, name),
|
||||
expected.into_iter().collect::<Vec<_>>()
|
||||
);
|
||||
assert!(header(&request, "authorization")[0].starts_with("AWS4-HMAC-SHA256 "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_session_token_replaces_the_callers_security_token() {
|
||||
let request = sent(
|
||||
&[("X-Amz-Security-Token", "caller-value")],
|
||||
Credentials::new(
|
||||
"AKIDEXAMPLE",
|
||||
"secret",
|
||||
Some("session".into()),
|
||||
None,
|
||||
"test",
|
||||
),
|
||||
);
|
||||
|
||||
assert_eq!(header(&request, "x-amz-security-token"), ["session"]);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::bearer_overrides_the_signature("Bearer gateway", "Bearer gateway")]
|
||||
#[case::a_stale_signature_is_replaced(
|
||||
"AWS4-HMAC-SHA256 Credential=stale",
|
||||
"AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;host;x-amz-date"
|
||||
)]
|
||||
fn a_callers_authorization_overrides_the_signature_unless_it_is_sigv4(
|
||||
#[case] caller: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let request = sent(
|
||||
&[("Authorization", caller)],
|
||||
Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"),
|
||||
);
|
||||
let authorization = header(&request, "authorization");
|
||||
|
||||
assert_eq!(authorization.len(), 1);
|
||||
assert_eq!(
|
||||
authorization[0].split(", Signature=").next(),
|
||||
Some(expected)
|
||||
);
|
||||
assert_eq!(header(&request, "x-amz-date"), ["20231114T221320Z"]);
|
||||
}
|
||||
|
||||
const BEDROCK_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke";
|
||||
const ARN_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A1%3Ainference-profile%2Fx/invoke?b=2&a=1";
|
||||
const CALLER_SIGNING_HEADERS: &[(&str, &str)] = &[
|
||||
("date", "Wed, 01 Jan 2020 00:00:00 GMT"),
|
||||
("x-amz-date", "20200101T000000Z"),
|
||||
("x-amz-security-token", "caller-token"),
|
||||
];
|
||||
const CALLER_TOKEN: &[(&str, &str)] = &[("x-amz-security-token", "caller-token")];
|
||||
const CALLER_DATE: &[(&str, &str)] = &[("Date", "Wed, 01 Jan 2020 00:00:00 GMT")];
|
||||
const HTTP_DATE: &str = "Tue, 14 Nov 2023 22:13:20 -0000";
|
||||
const AMZ_DATE: &str = "20231114T221320Z";
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::date_and_caller_token(URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "1b3652d523d5296b3904a80981e936a201167da9cb7bd200c2da8171dadb2c08")]
|
||||
#[case::caller_token_only(URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "0fbd1a8e98efbfbcdb6fa9d665fe89ffffcdd8a7656865d63def21a5344d02dd")]
|
||||
#[case::date_only(URL, CALLER_DATE, None, (Some(HTTP_DATE), None, None), "content-type;date;host", "cccdac7b04f20ebfe37a156c4e806a14e6c0a4b0aad11ca67e25dca337237085")]
|
||||
#[case::session_replaces_the_caller_token_under_a_date(URL, CALLER_SIGNING_HEADERS, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "3406b8200cae95c8b66037dc5a9a838e1fe4da94b57b89fada3c05cde8a2c15c")]
|
||||
#[case::session_replaces_the_caller_token(URL, CALLER_TOKEN, Some("session"), (None, Some(AMZ_DATE), Some("session")), "content-type;host;x-amz-date;x-amz-security-token", "d75425952d9f5ef7465a6ef3acabf15908c0edfd8ce2529cc3f0f8d2e9232e76")]
|
||||
#[case::colon_in_the_model_path(BEDROCK_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "47d75a0dcb00499a025fc0f9f010f6dce89c7661eb54cb97c3a293e9c585b153")]
|
||||
#[case::encoded_arn_and_query(ARN_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "8fb7d97a0ad5df9d1c3bbfc55897266059a5a2f35a1aaed9687f370047e78363")]
|
||||
#[case::encoded_arn_and_query_without_a_date(ARN_URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "f5eb993af90c7896203bf6b3b85c0087087f712a0634e44ac0a04c9d87d6fe12")]
|
||||
#[case::encoded_arn_and_query_with_a_session(ARN_URL, CALLER_DATE, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "15e8d5a9f64eb64366363f951db64de2a747959573a8cda9b5faa0fcbabc713c")]
|
||||
fn botocore_caller_headers_sign_like_botocore(
|
||||
#[case] url: &str,
|
||||
#[case] caller: &[(&str, &str)],
|
||||
#[case] session: Option<&str>,
|
||||
#[case] sent: (Option<&str>, Option<&str>, Option<&str>),
|
||||
#[case] signed_headers: &str,
|
||||
#[case] signature: &str,
|
||||
) {
|
||||
let request = signed(
|
||||
json(
|
||||
url,
|
||||
&caller
|
||||
.iter()
|
||||
.map(|(name, value)| (name.to_string(), value.to_string()))
|
||||
.collect::<Vec<_>>(),
|
||||
&json_value!({}),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
&SigV4Signer::new(
|
||||
"us-east-1".into(),
|
||||
"bedrock",
|
||||
Credentials::new(
|
||||
"AKIDEXAMPLE",
|
||||
"secret",
|
||||
session.map(str::to_string),
|
||||
None,
|
||||
"test",
|
||||
),
|
||||
)
|
||||
.with_clock(fixed_clock)
|
||||
.with_caller_headers(CallerSigningHeaders::Botocore),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
header(&request, "date"),
|
||||
header(&request, "x-amz-date"),
|
||||
header(&request, "x-amz-security-token"),
|
||||
header(&request, "authorization"),
|
||||
),
|
||||
(
|
||||
sent.0.into_iter().collect::<Vec<_>>(),
|
||||
sent.1.into_iter().collect::<Vec<_>>(),
|
||||
sent.2.into_iter().collect::<Vec<_>>(),
|
||||
vec![format!(
|
||||
"AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders={signed_headers}, Signature={signature}"
|
||||
)
|
||||
.as_str()],
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -52,6 +52,6 @@ pub use credential::{
|
|||
};
|
||||
pub use error::{Error, ErrorDetail};
|
||||
pub use http::CredentialPlacement;
|
||||
pub use params::{AwsParams, ParamKind, ParamSpec, VertexParams};
|
||||
pub use params::{AwsParams, AwsSessionTag, ParamKind, ParamSpec, VertexParams};
|
||||
pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
|
||||
pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
|
||||
|
|
|
|||
|
|
@ -21,6 +21,14 @@ const fn spec(
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct AwsSessionTag {
|
||||
#[serde(rename = "Key")]
|
||||
pub key: String,
|
||||
#[serde(rename = "Value")]
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
/// The `aws_*` fields of Python's `GenericLiteLLMParams`.
|
||||
#[derive(Redact, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AwsParams {
|
||||
|
|
@ -50,11 +58,15 @@ pub struct AwsParams {
|
|||
pub aws_external_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub aws_bedrock_runtime_endpoint: Option<String>,
|
||||
#[serde(default)]
|
||||
pub aws_session_tags: Option<Vec<AwsSessionTag>>,
|
||||
}
|
||||
|
||||
impl AwsParams {
|
||||
pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN";
|
||||
pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
|
||||
pub const DEFAULT_REGION_ENV: &str = "AWS_DEFAULT_REGION";
|
||||
pub const SESSION_TAGS: &str = "aws_session_tags";
|
||||
|
||||
pub const ACCESS_KEY_ID: ParamSpec = spec(
|
||||
"access key id",
|
||||
|
|
@ -144,6 +156,7 @@ impl AwsParams {
|
|||
Self::SPECS
|
||||
.iter()
|
||||
.flat_map(|spec| spec.wire.iter().copied())
|
||||
.chain([Self::SESSION_TAGS])
|
||||
}
|
||||
|
||||
/// The environment names the specs read, for hosts that resolve secrets up front.
|
||||
|
|
@ -153,6 +166,7 @@ impl AwsParams {
|
|||
.iter()
|
||||
.flat_map(|spec| spec.env.iter().copied())
|
||||
.chain([
|
||||
AwsParams::DEFAULT_REGION_ENV,
|
||||
AwsParams::ROLE_ARN_ENV,
|
||||
AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV,
|
||||
])
|
||||
|
|
@ -205,6 +219,9 @@ impl AwsParams {
|
|||
aws_sts_endpoint: value("aws_sts_endpoint"),
|
||||
aws_external_id: value("aws_external_id"),
|
||||
aws_bedrock_runtime_endpoint: value("aws_bedrock_runtime_endpoint"),
|
||||
aws_session_tags: optional_params
|
||||
.get(Self::SESSION_TAGS)
|
||||
.and_then(|tags| serde_json::from_value(tags.clone()).ok()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
mod aws;
|
||||
mod vertex;
|
||||
|
||||
pub use aws::AwsParams;
|
||||
pub use aws::{AwsParams, AwsSessionTag};
|
||||
pub use vertex::VertexParams;
|
||||
|
||||
/// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads
|
||||
|
|
|
|||
|
|
@ -1,11 +1,16 @@
|
|||
use litellm_auth_types::AwsParams;
|
||||
use rstest::rstest;
|
||||
use serde_json::{Map, Value};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
#[rstest]
|
||||
fn every_named_param_roundtrips_without_loss() {
|
||||
let filled: Map<String, Value> = AwsParams::fields()
|
||||
.map(|name| (name.to_string(), Value::from(format!("value-of-{name}"))))
|
||||
.map(|name| match name {
|
||||
AwsParams::SESSION_TAGS => {
|
||||
(name.to_string(), json!([{"Key": "team", "Value": "genai"}]))
|
||||
}
|
||||
_ => (name.to_string(), Value::from(format!("value-of-{name}"))),
|
||||
})
|
||||
.collect();
|
||||
let typed = AwsParams::from_optional_params(&filled);
|
||||
let serialized = serde_json::to_value(&typed).unwrap();
|
||||
|
|
|
|||
|
|
@ -159,6 +159,7 @@
|
|||
"endpoint_type",
|
||||
"request_body",
|
||||
"chunks",
|
||||
"error"
|
||||
"error",
|
||||
"head"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ struct PassThroughStream {
|
|||
|
||||
/// What the Messages stream iterator keeps for its end-of-stream billing.
|
||||
struct DeliveredStream {
|
||||
head: Py<PyAny>,
|
||||
chunks: Py<PyList>,
|
||||
first_chunk: Option<Py<PyAny>>,
|
||||
}
|
||||
|
|
@ -65,22 +66,14 @@ pub struct LegacyLogging {
|
|||
/// When the provider request went out, until its window is recorded (Messages only).
|
||||
api_started: Option<f64>,
|
||||
host_answered: bool,
|
||||
/// Whether the call's Messages stream starts only on its first read, as Python's iterator
|
||||
/// does. Only `azure_ai`: by the owner's ruling `anthropic` keeps main's billing of an
|
||||
/// unread stream.
|
||||
/// Whether the call's stream starts only on its first read, as Python's Messages iterator
|
||||
/// does.
|
||||
stream_starts_on_first_read: bool,
|
||||
/// Set once the route sends through the host's own client, whose timed `post` already
|
||||
/// reports the exchange.
|
||||
host_client: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
/// The one provider whose unread Messages stream bills nothing, as Python's does.
|
||||
const STREAM_STARTS_ON_FIRST_READ_PROVIDER: &str = "azure_ai";
|
||||
|
||||
/// The one provider whose exchange through the host's client is timed only by that client, as
|
||||
/// Python's is. By the owner's ruling `anthropic` keeps main's second report.
|
||||
const HOST_CLIENT_TIMED_PROVIDER: &str = "azure_ai";
|
||||
|
||||
fn retry_reason(reason: &RetryReason) -> Value {
|
||||
match reason {
|
||||
RetryReason::ThinkingSignature => serde_json::json!({"kind": "thinking_signature"}),
|
||||
|
|
@ -504,6 +497,7 @@ impl LegacyLogging {
|
|||
),
|
||||
&stream.chunks,
|
||||
error,
|
||||
&stream.head,
|
||||
),
|
||||
);
|
||||
match scheduled {
|
||||
|
|
@ -675,8 +669,7 @@ impl LegacyLogging {
|
|||
)?;
|
||||
if self.operation == LoggingOperation::Messages
|
||||
&& self.asynchronous
|
||||
&& !(context.custom_llm_provider == HOST_CLIENT_TIMED_PROVIDER
|
||||
&& self.host_client.load(Ordering::Acquire))
|
||||
&& !self.host_client.load(Ordering::Acquire)
|
||||
&& self.times_provider_exchange(py)?
|
||||
{
|
||||
self.api_started = Some(epoch_seconds());
|
||||
|
|
@ -729,8 +722,7 @@ impl LegacyLogging {
|
|||
) -> PyResult<HookStep<Self, ()>> {
|
||||
use litellm_host::interceptors::{Cost, ResultSource};
|
||||
|
||||
self.stream_starts_on_first_read =
|
||||
facts.provider.provider == STREAM_STARTS_ON_FIRST_READ_PROVIDER;
|
||||
self.stream_starts_on_first_read = self.operation == LoggingOperation::Messages;
|
||||
self.cache_key = match &facts.source {
|
||||
ResultSource::Provider => None,
|
||||
ResultSource::Cache { key } => Some(key.clone()),
|
||||
|
|
@ -940,6 +932,7 @@ impl LegacyLogging {
|
|||
),
|
||||
)?;
|
||||
self.stream = Some(DeliveredStream {
|
||||
head: head.clone_ref(py),
|
||||
chunks: PyList::empty(py).unbind(),
|
||||
first_chunk: None,
|
||||
});
|
||||
|
|
@ -995,6 +988,7 @@ impl PythonOwned for LegacyLogging {
|
|||
visit.call(&self.response)?;
|
||||
visit.call(&self.error)?;
|
||||
if let Some(stream) = &self.stream {
|
||||
visit.call(&stream.head)?;
|
||||
visit.call(&stream.chunks)?;
|
||||
visit.call(&stream.first_chunk)?;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ FAKES = {
|
|||
'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record(
|
||||
'stream_success', list(chunks)
|
||||
),
|
||||
'stream_failure': lambda logger, endpoint_type, request_body, chunks, error: logger.record('stream_failure', error),
|
||||
'stream_failure': lambda logger, endpoint_type, request_body, chunks, error, head: logger.record('stream_failure', error),
|
||||
}
|
||||
assert FAKES.keys() == CONTRACT.keys(), sorted(FAKES.keys() ^ CONTRACT.keys())
|
||||
for name, fake in FAKES.items():
|
||||
|
|
|
|||
|
|
@ -336,6 +336,8 @@ impl Settings {
|
|||
web_identity_token: secret(params, "s3_aws_web_identity_token"),
|
||||
sts_endpoint: text(params, "s3_aws_sts_endpoint"),
|
||||
external_id: None,
|
||||
session_tags: None,
|
||||
session_policy: None,
|
||||
},
|
||||
use_team_prefix: truthy_param("s3_use_team_prefix"),
|
||||
use_key_prefix: truthy_param("s3_use_key_prefix"),
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ serde_json.workspace = true
|
|||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-cache-memory.workspace = true
|
||||
futures-util.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util"] }
|
||||
|
|
|
|||
|
|
@ -324,8 +324,8 @@ mod tests {
|
|||
#[rstest]
|
||||
#[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)]
|
||||
#[case::declined(
|
||||
Error::Route(InferenceError::from(litellm_http::Error::computed_header(
|
||||
"authorization"
|
||||
Error::Route(InferenceError::Declined(litellm_core_utils::decline::Decline::new(
|
||||
"unrecognized request parameter"
|
||||
))),
|
||||
StatusCode::NOT_IMPLEMENTED
|
||||
)]
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ fn project(
|
|||
.chain(tools)
|
||||
.collect();
|
||||
Ok(MessagesCall {
|
||||
body: messages_body(body, deployment.custom_llm_provider.as_deref())?,
|
||||
body: messages_body(body)?,
|
||||
api_key: deployment.api_key.clone(),
|
||||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
|
|
|
|||
|
|
@ -16,15 +16,15 @@ Boundary tests exercise behavior with a supplied lifecycle binding without impor
|
|||
|
||||
`HookChain` composes Python runtime hooks in order. Each argument, wire-request and response transformation feeds its result to the next hook. After all argument transformations, the driver calls `arguments_prepared` on every hook in order. Retained callback views must adopt that dictionary before later policy hooks can mutate or reject it. SDK policy is supplied by bridge composition as a hook, never a separate driver phase or parameter. Hooks implement only the stages they need. Default stages preserve the supplied values
|
||||
|
||||
Terminal notifications share the selected response or exception. An ordinary notification error is reported as unraisable and does not skip the next hook or replace the selected outcome. Preparation, interception and transformation errors stop the chain. Cancellation stops all further hook dispatch. Suspensions stay inline in the existing driver, and the chain traverses retained event values for GC
|
||||
Terminal notifications share the selected response or exception. An ordinary notification error is reported as unraisable and does not skip the next hook or replace the selected outcome. Preparation, interception and transformation errors stop the chain. Cancellation stops all further hook dispatch, except on a stream whose head asks for a detached drain: the first cancellation during a read keeps reading to the end, delivers every chunk and the success, then raises the cancellation, and a second cancellation stops it. Suspensions stay inline in the existing driver, and the chain traverses retained event values for GC
|
||||
|
||||
## Existing runtime invariants
|
||||
|
||||
- Keep this crate the CPython runtime adapter and nothing more: Serde marshalling, interpreter detachment, tokio/asyncio glue, the `Execution` handle, the call driver and the `PythonBinding`, `PythonHostCalls` and `PythonOwned` traits, and the `PythonRuntime` specialization of `host::hooks::CallHooks`
|
||||
- No LiteLLM domain dependencies beyond `litellm-host`: no route types, no `Logging` policy, no public API registration, no cdylib build features
|
||||
- `PythonCallHooks` only constrains the shared call-stage interface to `PythonRuntime` and Python ownership. It must not redeclare the stages
|
||||
- `PythonCallEvent` is a specialization of the shared `CallEvent`, never a separately defined lifecycle. The driver emits `Succeeded` or `Failed` exactly once and never dispatches callbacks after a cancellation. Which Python objects consume those events is the legacy adapter's business
|
||||
- `CallOptions` can publish snapshots independently of callback delivery. Terminal snapshots follow completed hook dispatch, and cancellation never calls a Python callback. For Python-driven calls, leave the machine's observation publisher unset so the driver is the sole publisher of intercepted provider-response snapshots
|
||||
- `PythonCallEvent` is a specialization of the shared `CallEvent`, never a separately defined lifecycle. The driver emits `Succeeded` or `Failed` exactly once and never dispatches callbacks after a cancellation, apart from the detached drain above. Which Python objects consume those events is the legacy adapter's business
|
||||
- `CallOptions` can publish snapshots independently of callback delivery. Terminal snapshots follow completed hook dispatch, and cancellation never calls a Python callback outside a detached drain. For Python-driven calls, leave the machine's observation publisher unset so the driver is the sole publisher of intercepted provider-response snapshots
|
||||
- `PythonBinding::decode_request` receives the keyword view returned by `prepare_arguments` and updated by `arguments_prepared`, not the caller's dict. A binding that decodes from it inherits all composed argument rewrites
|
||||
- A native failure, including one a host op returns as `InvokeError::Native`, is classified exactly once through the binding's `map_error`. A Python exception raised inside the call, and a failure in `prepare_arguments` or `transform_response`, is raised as is
|
||||
- A failing `map_error` is raised with the native error's text as its `__context__`, never swallowed
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ use litellm_host::{
|
|||
lifecycle::{CallEvent, ExecutionEvent, FailureOrigin, Timing, epoch_seconds},
|
||||
machine::{HostFailure, Machine, MachineStep},
|
||||
observation::ObservationSender,
|
||||
protocol::{HostRequest, InterceptRequest, Protocol, Reply, StreamDelivery},
|
||||
protocol::{HostRequest, InterceptRequest, Protocol, Reply, StreamDelivery, UnobservedTail},
|
||||
};
|
||||
|
||||
use crate::PythonHostCalls;
|
||||
|
|
@ -133,6 +133,9 @@ where
|
|||
native: NativeMachine<M>,
|
||||
start: Option<StartMachine<ProtocolOf<H>, M>>,
|
||||
streaming: bool,
|
||||
detached: bool,
|
||||
drains: bool,
|
||||
unobserved_tail: Option<UnobservedTail>,
|
||||
}
|
||||
|
||||
impl<H, M, L> CallBody<H, M, L>
|
||||
|
|
@ -238,6 +241,13 @@ where
|
|||
NativePoll::Ready(result) => result,
|
||||
NativePoll::Suspend(awaitable) => match self.caller.await_python(awaitable).await {
|
||||
Ok(_) => self.native.take_result()?,
|
||||
Err(error) if self.drains_after(&error) => {
|
||||
self.detached = true;
|
||||
Python::attach(|py| {
|
||||
self.parts().interrupted = Some(error.into_value(py));
|
||||
});
|
||||
continue;
|
||||
}
|
||||
Err(error) if self.parts().interrupted.is_none() => {
|
||||
interruption = Some(self.interrupt(error));
|
||||
continue;
|
||||
|
|
@ -260,6 +270,14 @@ where
|
|||
}
|
||||
}
|
||||
|
||||
fn drains_after(&self, error: &PyErr) -> bool {
|
||||
self.drains
|
||||
&& self.streaming
|
||||
&& !self.detached
|
||||
&& self.parts().interrupted.is_none()
|
||||
&& Python::attach(|py| is_cancellation(py, error))
|
||||
}
|
||||
|
||||
fn interrupt(&self, error: PyErr) -> HostFailure<ErrorOf<H>> {
|
||||
Python::attach(|py| {
|
||||
let cancelled = is_cancellation(py, &error);
|
||||
|
|
@ -320,6 +338,8 @@ where
|
|||
}
|
||||
HostRequest::Stream(StreamDelivery::Open(head, reply)) => {
|
||||
self.streaming = true;
|
||||
self.drains = ProtocolOf::<H>::detached_drain(&head).is_some();
|
||||
self.unobserved_tail = ProtocolOf::<H>::unobserved_tail(&head);
|
||||
let head = {
|
||||
let mut parts = self.parts();
|
||||
let Parts { binding, hooks, .. } = &mut *parts;
|
||||
|
|
@ -333,16 +353,28 @@ where
|
|||
Ok(())
|
||||
}
|
||||
HostRequest::Stream(StreamDelivery::Chunk(chunk, reply)) => {
|
||||
let observed = !self
|
||||
.unobserved_tail
|
||||
.as_ref()
|
||||
.is_some_and(UnobservedTail::begun);
|
||||
let chunk = {
|
||||
let mut parts = self.parts();
|
||||
let Parts { binding, hooks, .. } = &mut *parts;
|
||||
Python::attach(|py| {
|
||||
let chunk = binding.encode_chunk(py, chunk)?;
|
||||
hooks.on_stream_chunk(py, &chunk)?;
|
||||
if observed {
|
||||
hooks.on_stream_chunk(py, &chunk)?;
|
||||
}
|
||||
Ok::<_, PyErr>(chunk)
|
||||
})?
|
||||
};
|
||||
reply.send(self.caller.yield_chunk(chunk).await);
|
||||
if self.detached {
|
||||
reply.send(std::ops::ControlFlow::Break(()));
|
||||
return Ok(());
|
||||
}
|
||||
let flow = self.caller.yield_chunk(chunk).await;
|
||||
self.detached = flow.is_break();
|
||||
reply.send(flow);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -384,7 +416,14 @@ where
|
|||
HostedCompletion::Complete(response) => response,
|
||||
HostedCompletion::StreamEnded | HostedCompletion::Detached => {
|
||||
let none = Python::attach(|py| py.None());
|
||||
return self.succeeded(none).await;
|
||||
let succeeded = self.succeeded(none).await;
|
||||
let cancelled = self.parts().interrupted.take();
|
||||
return match cancelled {
|
||||
Some(cancelled) => Err(Python::attach(|py| {
|
||||
PyErr::from_value(cancelled.into_bound(py).into_any())
|
||||
})),
|
||||
None => succeeded,
|
||||
};
|
||||
}
|
||||
};
|
||||
let encoded = {
|
||||
|
|
@ -645,6 +684,9 @@ where
|
|||
native: NativeMachine::new(asynchronous),
|
||||
start: Some(Box::new(start)),
|
||||
streaming: false,
|
||||
detached: false,
|
||||
drains: false,
|
||||
unobserved_tail: None,
|
||||
}
|
||||
.run()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,13 +67,14 @@ where
|
|||
let state = Arc::clone(self.state.as_ref().ok_or(Error::MissingState)?);
|
||||
let future = async move {
|
||||
let mut state = state.lock().await;
|
||||
let result = match interruption {
|
||||
Some(failure) => state
|
||||
let result = match (interruption, state.result.take()) {
|
||||
(Some(failure), _) => state
|
||||
.machine
|
||||
.interrupt(failure)
|
||||
.await
|
||||
.map(MachineStep::Complete),
|
||||
None => state.machine.resume().await,
|
||||
(None, Some(settled)) => settled,
|
||||
(None, None) => state.machine.resume().await,
|
||||
};
|
||||
state.result = Some(result);
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -23,8 +23,9 @@ use pyo3::{
|
|||
use rstest::rstest;
|
||||
use support::{
|
||||
HookScript, Interpreter, Log, MachineFactory, OpScript, Synthetic, binding, call_options,
|
||||
failing_machine, handed, hooks, interpreter, lifecycle_module, open_stream, read_all,
|
||||
run_composed, run_scripted, settled, success_machine,
|
||||
draining_streaming_machine, failing_machine, handed, hooks, interpreter, lifecycle_module,
|
||||
open_stream, open_stream_from, read_all, run_composed, run_scripted, settled, success_machine,
|
||||
tailed_streaming_machine,
|
||||
};
|
||||
|
||||
fn observed(capacity: usize) -> (CallOptions, impl FnMut() -> Vec<CallEvent>) {
|
||||
|
|
@ -403,6 +404,151 @@ fn a_stream_carries_its_head_before_the_first_chunk(
|
|||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn chunks_on_the_unobserved_tail_reach_the_caller_without_the_stream_hooks(
|
||||
#[from(interpreter)] _python: Interpreter,
|
||||
#[values(false, true)] asynchronous: bool,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let log = Log::default();
|
||||
let stream = open_stream_from(
|
||||
py,
|
||||
tailed_streaming_machine(),
|
||||
hooks(&log, HookScript::Plain),
|
||||
call_options(asynchronous),
|
||||
);
|
||||
assert_eq!(
|
||||
read_all(py, &stream, asynchronous),
|
||||
["first", "second", "trailer"]
|
||||
);
|
||||
assert_eq!(
|
||||
log.entries(),
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"opened",
|
||||
"delivered",
|
||||
"delivered",
|
||||
"succeeded:None",
|
||||
"adapter.close"
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_draining_stream_reports_every_chunk_to_the_stream_hooks_after_the_caller_closes(
|
||||
#[from(interpreter)] _python: Interpreter,
|
||||
#[values(false, true)] asynchronous: bool,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let log = Log::default();
|
||||
let stream = open_stream_from(
|
||||
py,
|
||||
draining_streaming_machine(true, std::time::Duration::ZERO),
|
||||
hooks(&log, HookScript::Plain),
|
||||
call_options(asynchronous),
|
||||
);
|
||||
if asynchronous {
|
||||
for method in ["__anext__", "aclose"] {
|
||||
let stop = stream
|
||||
.call_method0(method)
|
||||
.unwrap()
|
||||
.call_method1("send", (py.None(),))
|
||||
.unwrap_err();
|
||||
assert!(stop.is_instance_of::<PyStopIteration>(py));
|
||||
}
|
||||
} else {
|
||||
stream.call_method0("__next__").unwrap();
|
||||
stream.call_method0("close").unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
log.entries(),
|
||||
[
|
||||
"started",
|
||||
"begin",
|
||||
"opened",
|
||||
"delivered",
|
||||
"delivered",
|
||||
"delivered",
|
||||
"succeeded:None",
|
||||
"adapter.close"
|
||||
]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::draining(
|
||||
true,
|
||||
"cancelled",
|
||||
&["started", "begin", "opened", "delivered", "delivered", "delivered", "succeeded:None", "adapter.close"]
|
||||
)]
|
||||
#[case::not_draining(false, "cancelled", &["started", "begin", "opened", "delivered", "adapter.close"])]
|
||||
fn a_caller_cancelled_mid_read_leaves_only_a_draining_stream_reading_on(
|
||||
#[from(interpreter)] _python: Interpreter,
|
||||
#[case] drain: bool,
|
||||
#[case] outcome: &str,
|
||||
#[case] logged: &[&str],
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let log = Log::default();
|
||||
let opened = handed(
|
||||
py,
|
||||
run_call(
|
||||
py,
|
||||
move |_, _, request| {
|
||||
Ok(draining_streaming_machine(
|
||||
drain,
|
||||
std::time::Duration::from_millis(200),
|
||||
)(request))
|
||||
},
|
||||
support::StreamingBinding,
|
||||
hooks(&log, HookScript::Plain),
|
||||
PyDict::new(py).unbind(),
|
||||
call_options(true),
|
||||
),
|
||||
true,
|
||||
)
|
||||
.unwrap();
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("opened", opened).unwrap();
|
||||
py.run(
|
||||
c"
|
||||
import asyncio
|
||||
|
||||
async def read_then_cancel():
|
||||
stream = await opened
|
||||
await stream.__anext__()
|
||||
reading = asyncio.ensure_future(stream.__anext__())
|
||||
await asyncio.sleep(0.05)
|
||||
reading.cancel()
|
||||
try:
|
||||
await reading
|
||||
except asyncio.CancelledError:
|
||||
outcome = 'cancelled'
|
||||
else:
|
||||
outcome = 'read'
|
||||
await stream.aclose()
|
||||
return outcome
|
||||
|
||||
outcome = asyncio.run(read_then_cancel())
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let actual: String = locals
|
||||
.get_item("outcome")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract()
|
||||
.unwrap();
|
||||
assert_eq!(actual, outcome);
|
||||
assert_eq!(log.entries(), logged);
|
||||
});
|
||||
}
|
||||
|
||||
struct Classification {
|
||||
provider_fails: bool,
|
||||
op: OpScript,
|
||||
|
|
|
|||
|
|
@ -2,13 +2,15 @@
|
|||
|
||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||
|
||||
use futures_util::StreamExt;
|
||||
|
||||
use litellm_host::{
|
||||
call::{CallOutput, HostedMachine, hosted_call},
|
||||
hooks::CallHooks,
|
||||
interceptors::{Interceptors, RawResponse, RequestContext, WireRequest},
|
||||
lifecycle::{ExecutionEvent, Timing},
|
||||
machine::{CallMachine, MachineFault},
|
||||
protocol::{Protocol, Reply},
|
||||
protocol::{DetachedDrain, Protocol, Reply, UnobservedTail},
|
||||
};
|
||||
use litellm_host_python::{
|
||||
CallOptions, HookStep, InvokeError, PythonBinding, PythonCallEvent, PythonCallHooks,
|
||||
|
|
@ -559,7 +561,21 @@ impl Protocol for Streaming {
|
|||
type Request = ();
|
||||
type HostCall = std::convert::Infallible;
|
||||
type Chunk = &'static str;
|
||||
type StreamHead = Vec<(&'static str, &'static str)>;
|
||||
type StreamHead = StreamingHead;
|
||||
|
||||
fn detached_drain(head: &StreamingHead) -> Option<DetachedDrain> {
|
||||
head.drain
|
||||
}
|
||||
|
||||
fn unobserved_tail(head: &StreamingHead) -> Option<UnobservedTail> {
|
||||
head.tail.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct StreamingHead {
|
||||
headers: Vec<(&'static str, &'static str)>,
|
||||
tail: Option<UnobservedTail>,
|
||||
drain: Option<DetachedDrain>,
|
||||
}
|
||||
|
||||
pub struct StreamingBinding;
|
||||
|
|
@ -576,13 +592,9 @@ impl PythonBinding for StreamingBinding {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn encode_stream_head(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
head: Vec<(&'static str, &'static str)>,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
fn encode_stream_head(&mut self, py: Python<'_>, head: StreamingHead) -> PyResult<Py<PyAny>> {
|
||||
let headers = PyDict::new(py);
|
||||
for (name, value) in head {
|
||||
for (name, value) in head.headers {
|
||||
headers.set_item(name, value)?;
|
||||
}
|
||||
let hidden = PyDict::new(py);
|
||||
|
|
@ -629,25 +641,94 @@ pub fn streaming_machine() -> impl FnOnce(()) -> HostedMachine<Streaming> + Send
|
|||
move |()| {
|
||||
hosted_call((), None, |(), _, _, _observations| async {
|
||||
Ok(CallOutput::Stream {
|
||||
head: vec![("request-id", "req_1")],
|
||||
head: StreamingHead {
|
||||
headers: vec![("request-id", "req_1")],
|
||||
tail: None,
|
||||
drain: None,
|
||||
},
|
||||
chunks: Box::pin(futures_util::stream::iter([Ok("first"), Ok("second")])),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Streams `first` and `second`, then `trailer` on its unobserved tail.
|
||||
pub fn tailed_streaming_machine() -> impl FnOnce(()) -> HostedMachine<Streaming> + Send + Sync {
|
||||
move |()| {
|
||||
hosted_call((), None, |(), _, _, _observations| async {
|
||||
let tail = UnobservedTail::default();
|
||||
let begun = tail.clone();
|
||||
let trailer = futures_util::stream::once(async move {
|
||||
begun.begin();
|
||||
Ok("trailer")
|
||||
});
|
||||
Ok(CallOutput::Stream {
|
||||
head: StreamingHead {
|
||||
headers: Vec::new(),
|
||||
tail: Some(tail),
|
||||
drain: None,
|
||||
},
|
||||
chunks: Box::pin(futures_util::StreamExt::chain(
|
||||
futures_util::stream::iter([Ok("first"), Ok("second")]),
|
||||
trailer,
|
||||
)),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub fn draining_streaming_machine(
|
||||
drain: bool,
|
||||
delay: std::time::Duration,
|
||||
) -> impl FnOnce(()) -> HostedMachine<Streaming> + Send + Sync {
|
||||
move |()| {
|
||||
hosted_call((), None, move |(), _, _, _observations| async move {
|
||||
let second = futures_util::stream::once(async move {
|
||||
if !delay.is_zero() {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
Ok("second")
|
||||
});
|
||||
Ok(CallOutput::Stream {
|
||||
head: StreamingHead {
|
||||
headers: Vec::new(),
|
||||
tail: None,
|
||||
drain: drain.then_some(DetachedDrain {
|
||||
timeout: std::time::Duration::from_secs(5),
|
||||
limit: 4,
|
||||
}),
|
||||
},
|
||||
chunks: Box::pin(
|
||||
futures_util::stream::iter([Ok("first")])
|
||||
.chain(second)
|
||||
.chain(futures_util::stream::iter([Ok("third")])),
|
||||
),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts a streaming call and returns the caller's stream object.
|
||||
pub fn open_stream(
|
||||
py: Python<'_>,
|
||||
hooks: SyntheticHooks,
|
||||
options: CallOptions,
|
||||
) -> Bound<'_, PyAny> {
|
||||
open_stream_from(py, streaming_machine(), hooks, options)
|
||||
}
|
||||
|
||||
pub fn open_stream_from(
|
||||
py: Python<'_>,
|
||||
machine: impl FnOnce(()) -> HostedMachine<Streaming> + Send + Sync + 'static,
|
||||
hooks: SyntheticHooks,
|
||||
options: CallOptions,
|
||||
) -> Bound<'_, PyAny> {
|
||||
let asynchronous = options.asynchronous;
|
||||
let handed = handed(
|
||||
py,
|
||||
run_call(
|
||||
py,
|
||||
|_, _, request| Ok(streaming_machine()(request)),
|
||||
move |_, _, request| Ok(machine(request)),
|
||||
StreamingBinding,
|
||||
hooks,
|
||||
PyDict::new(py).unbind(),
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ litellm-coroutine.workspace = true
|
|||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio = { workspace = true, features = ["sync", "time"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,13 +1,58 @@
|
|||
use crate::observation::ObservationSender;
|
||||
use std::future::Future;
|
||||
use std::{
|
||||
future::Future,
|
||||
sync::atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
use futures_util::{TryStreamExt, stream::BoxStream};
|
||||
|
||||
use crate::{
|
||||
machine::{CallMachine, ChannelInterceptors, HostServices, MachineFault},
|
||||
protocol::Protocol,
|
||||
machine::{CallMachine, ChannelInterceptors, HostServices, MachineFault, StreamSender},
|
||||
protocol::{DetachedDrain, Protocol},
|
||||
};
|
||||
|
||||
static DETACHED_DRAINS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
struct DrainSlot;
|
||||
|
||||
impl DrainSlot {
|
||||
fn claim(limit: usize) -> Option<Self> {
|
||||
DETACHED_DRAINS
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |running| {
|
||||
(running < limit).then_some(running + 1)
|
||||
})
|
||||
.ok()
|
||||
.map(|_| Self)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DrainSlot {
|
||||
fn drop(&mut self) {
|
||||
DETACHED_DRAINS.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
async fn drain_detached<P>(
|
||||
stream: &StreamSender<P>,
|
||||
mut chunks: BoxStream<'static, Result<P::Chunk, P::Error>>,
|
||||
drain: DetachedDrain,
|
||||
) where
|
||||
P: Protocol,
|
||||
P::Error: From<MachineFault>,
|
||||
{
|
||||
let Some(_slot) = DrainSlot::claim(drain.limit) else {
|
||||
return;
|
||||
};
|
||||
let delivered = async {
|
||||
while let Ok(Some(chunk)) = chunks.try_next().await {
|
||||
if stream.send_chunk(chunk).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
let _ = tokio::time::timeout(drain.timeout, delivered).await;
|
||||
}
|
||||
|
||||
pub enum CallOutput<Response, Head, Chunk, Error> {
|
||||
Complete(Response),
|
||||
Stream {
|
||||
|
|
@ -61,11 +106,15 @@ where
|
|||
match execute(request, host.services, host.interceptors, host.observers).await? {
|
||||
CallOutput::Complete(response) => Ok(HostedCompletion::Complete(response)),
|
||||
CallOutput::Stream { head, mut chunks } => {
|
||||
let drain = P::detached_drain(&head);
|
||||
if host.stream.open_stream(head).await?.is_break() {
|
||||
return Ok(HostedCompletion::Detached);
|
||||
}
|
||||
while let Some(chunk) = chunks.try_next().await? {
|
||||
if host.stream.send_chunk(chunk).await?.is_break() {
|
||||
if let Some(drain) = drain {
|
||||
drain_detached(&host.stream, chunks, drain).await;
|
||||
}
|
||||
return Ok(HostedCompletion::Detached);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
use std::ops::ControlFlow;
|
||||
use std::{
|
||||
ops::ControlFlow,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
pub use litellm_coroutine::{Abandoned, Answer, Reply, reply};
|
||||
|
||||
|
|
@ -11,6 +17,37 @@ pub trait Protocol: Send + Sync + 'static {
|
|||
type HostCall: Send + 'static;
|
||||
type Chunk: Send + 'static;
|
||||
type StreamHead: Send + 'static;
|
||||
|
||||
fn detached_drain(_head: &Self::StreamHead) -> Option<DetachedDrain> {
|
||||
None
|
||||
}
|
||||
|
||||
fn unobserved_tail(_head: &Self::StreamHead) -> Option<UnobservedTail> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a stream's chunks stop reaching the stream hooks: once the route begins the tail, the
|
||||
/// chunks it delivers go to the caller alone, as a frame Python's relay adds after billing does.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct UnobservedTail(Arc<AtomicBool>);
|
||||
|
||||
impl UnobservedTail {
|
||||
pub fn begin(&self) {
|
||||
self.0.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub fn begun(&self) -> bool {
|
||||
self.0.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
/// A stream whose upstream is read to its end after the caller detaches, so the provider's
|
||||
/// terminal usage still reaches the stream hooks.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct DetachedDrain {
|
||||
pub timeout: std::time::Duration,
|
||||
pub limit: usize,
|
||||
}
|
||||
|
||||
pub enum HostRequest<P: Protocol> {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ use std::{
|
|||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use futures_util::{StreamExt, stream};
|
||||
|
|
@ -13,7 +14,7 @@ use litellm_host::{
|
|||
call::{CallOutput, HostedCompletion, hosted_call},
|
||||
lifecycle::{CallEvent, CallObserver, observe_call, observe_unary},
|
||||
machine::{Machine, MachineFault, MachineStep},
|
||||
protocol::{HostRequest, Protocol},
|
||||
protocol::{DetachedDrain, HostRequest, Protocol},
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
|
|
@ -94,6 +95,86 @@ async fn delivery_obeys_demand_and_distinguishes_detachment(
|
|||
assert_eq!(delivered, (0..expected_polls).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
struct DrainingProtocol;
|
||||
|
||||
impl Protocol for DrainingProtocol {
|
||||
type Response = &'static str;
|
||||
type Error = TestError;
|
||||
type Request = usize;
|
||||
type HostCall = Infallible;
|
||||
type Chunk = usize;
|
||||
type StreamHead = Option<DetachedDrain>;
|
||||
|
||||
fn detached_drain(head: &Option<DetachedDrain>) -> Option<DetachedDrain> {
|
||||
*head
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::draining(
|
||||
Some(DetachedDrain { timeout: Duration::from_secs(5), limit: 4 }),
|
||||
vec![0, 1, 2]
|
||||
)]
|
||||
#[case::no_drain(None, vec![0])]
|
||||
#[case::no_free_slot(Some(DetachedDrain { timeout: Duration::from_secs(5), limit: 0 }), vec![0])]
|
||||
#[tokio::test]
|
||||
async fn a_draining_stream_delivers_the_rest_after_the_caller_detaches(
|
||||
#[case] drain: Option<DetachedDrain>,
|
||||
#[case] expected: Vec<usize>,
|
||||
) {
|
||||
let mut machine =
|
||||
hosted_call::<DrainingProtocol, _, _>(3, None, move |count, _, _, _| async move {
|
||||
Ok(CallOutput::Stream {
|
||||
head: drain,
|
||||
chunks: stream::iter((0..count).map(Ok)).boxed(),
|
||||
})
|
||||
});
|
||||
let mut delivered = Vec::new();
|
||||
let completed = loop {
|
||||
match machine.resume().await.unwrap() {
|
||||
MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Open(_, reply))) => {
|
||||
reply.send(ControlFlow::Continue(()));
|
||||
}
|
||||
MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Chunk(chunk, reply))) => {
|
||||
delivered.push(chunk);
|
||||
reply.send(ControlFlow::Break(()));
|
||||
}
|
||||
MachineStep::Complete(result) => break result,
|
||||
_ => panic!("unexpected operation"),
|
||||
}
|
||||
};
|
||||
assert_eq!(completed, HostedCompletion::Detached);
|
||||
assert_eq!(delivered, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_detached_drain_stops_at_its_timeout() {
|
||||
let mut machine =
|
||||
hosted_call::<DrainingProtocol, _, _>(0, None, move |_, _, _, _| async move {
|
||||
let pending = stream::iter([Ok(0)]).chain(stream::pending());
|
||||
Ok(CallOutput::Stream {
|
||||
head: Some(DetachedDrain {
|
||||
timeout: Duration::from_millis(50),
|
||||
limit: 4,
|
||||
}),
|
||||
chunks: pending.boxed(),
|
||||
})
|
||||
});
|
||||
let completed = loop {
|
||||
match machine.resume().await.unwrap() {
|
||||
MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Open(_, reply))) => {
|
||||
reply.send(ControlFlow::Continue(()));
|
||||
}
|
||||
MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Chunk(_, reply))) => {
|
||||
reply.send(ControlFlow::Break(()));
|
||||
}
|
||||
MachineStep::Complete(result) => break result,
|
||||
_ => panic!("unexpected operation"),
|
||||
}
|
||||
};
|
||||
assert_eq!(completed, HostedCompletion::Detached);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Observer(Observations);
|
||||
struct Observations {
|
||||
|
|
|
|||
|
|
@ -52,8 +52,6 @@ pub(crate) enum Repr {
|
|||
RequestUrl(#[source] reqwest::Error),
|
||||
#[error("invalid request URL")]
|
||||
ParseUrl(#[source] url::ParseError),
|
||||
#[error("request forwards the header {0:?}, which the signer computes")]
|
||||
ComputedHeader(String),
|
||||
#[error("request signing failed: a streaming body cannot be signed")]
|
||||
UnsignableBody,
|
||||
#[error("request forwards a header that cannot be signed")]
|
||||
|
|
@ -247,10 +245,6 @@ impl Error {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn computed_header(name: impl Into<String>) -> Self {
|
||||
Repr::ComputedHeader(name.into()).into()
|
||||
}
|
||||
|
||||
pub fn request_body(error: serde_json::Error) -> Self {
|
||||
Repr::RequestBody(error).into()
|
||||
}
|
||||
|
|
@ -292,7 +286,6 @@ impl Error {
|
|||
| Repr::Request(_)
|
||||
| Repr::RequestUrl(_)
|
||||
| Repr::ParseUrl(_)
|
||||
| Repr::ComputedHeader(_)
|
||||
| Repr::UnsignableBody
|
||||
| Repr::UnsignableHeader(_)
|
||||
| Repr::Signature(_)
|
||||
|
|
@ -318,7 +311,6 @@ impl Error {
|
|||
Repr::Request(_)
|
||||
| Repr::RequestUrl(_)
|
||||
| Repr::ParseUrl(_)
|
||||
| Repr::ComputedHeader(_)
|
||||
| Repr::UnsignableHeader(_)
|
||||
| Repr::WebSocketUrl(_)
|
||||
| Repr::WebSocketScheme(_)
|
||||
|
|
@ -364,7 +356,6 @@ impl Error {
|
|||
| Repr::Request(_)
|
||||
| Repr::RequestUrl(_)
|
||||
| Repr::ParseUrl(_)
|
||||
| Repr::ComputedHeader(_)
|
||||
| Repr::UnsignableBody
|
||||
| Repr::UnsignableHeader(_)
|
||||
| Repr::Signature(_)
|
||||
|
|
@ -392,51 +383,10 @@ impl Error {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn is_computed_header(&self) -> bool {
|
||||
match &*self.0 {
|
||||
Repr::ComputedHeader(_) => true,
|
||||
Repr::Read { .. }
|
||||
| Repr::InvalidPem { .. }
|
||||
| Repr::Client(_)
|
||||
| Repr::TlsConfig(_)
|
||||
| Repr::RequestBody(_)
|
||||
| Repr::Request(_)
|
||||
| Repr::RequestUrl(_)
|
||||
| Repr::ParseUrl(_)
|
||||
| Repr::UnsignableBody
|
||||
| Repr::UnsignableHeader(_)
|
||||
| Repr::Signature(_)
|
||||
| Repr::Connect(_)
|
||||
| Repr::Network(_)
|
||||
| Repr::BodyCut { .. }
|
||||
| Repr::Timeout(_)
|
||||
| Repr::ReadTimeout { .. }
|
||||
| Repr::Status { .. }
|
||||
| Repr::WebSocketUrl(_)
|
||||
| Repr::WebSocketScheme(_)
|
||||
| Repr::WebSocketSchemeRewrite(_)
|
||||
| Repr::Handshake(_)
|
||||
| Repr::HandshakeTimeout(_)
|
||||
| Repr::WebSocket(_)
|
||||
| Repr::WebSocketClosed
|
||||
| Repr::BlockedUrl
|
||||
| Repr::Resolve(_)
|
||||
| Repr::DownloadDisabled
|
||||
| Repr::DownloadTooLarge
|
||||
| Repr::TooManyRedirects
|
||||
| Repr::MissingRedirectLocation
|
||||
| Repr::InvalidRedirect(_)
|
||||
| Repr::DownloadStatus(_)
|
||||
| Repr::Download(_)
|
||||
| Repr::DownloadTimeout(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_connect(&self) -> bool {
|
||||
match &*self.0 {
|
||||
Repr::Connect(_) => true,
|
||||
Repr::ComputedHeader(_)
|
||||
| Repr::Read { .. }
|
||||
Repr::Read { .. }
|
||||
| Repr::InvalidPem { .. }
|
||||
| Repr::Client(_)
|
||||
| Repr::TlsConfig(_)
|
||||
|
|
@ -486,7 +436,6 @@ impl Error {
|
|||
| Repr::Request(_)
|
||||
| Repr::RequestUrl(_)
|
||||
| Repr::ParseUrl(_)
|
||||
| Repr::ComputedHeader(_)
|
||||
| Repr::UnsignableBody
|
||||
| Repr::UnsignableHeader(_)
|
||||
| Repr::Signature(_)
|
||||
|
|
@ -523,7 +472,6 @@ impl Error {
|
|||
| Repr::Request(_)
|
||||
| Repr::RequestUrl(_)
|
||||
| Repr::ParseUrl(_)
|
||||
| Repr::ComputedHeader(_)
|
||||
| Repr::UnsignableBody
|
||||
| Repr::UnsignableHeader(_)
|
||||
| Repr::Signature(_)
|
||||
|
|
@ -564,7 +512,6 @@ impl Error {
|
|||
| Repr::Request(_)
|
||||
| Repr::RequestUrl(_)
|
||||
| Repr::ParseUrl(_)
|
||||
| Repr::ComputedHeader(_)
|
||||
| Repr::UnsignableBody
|
||||
| Repr::UnsignableHeader(_)
|
||||
| Repr::Signature(_)
|
||||
|
|
@ -600,7 +547,6 @@ impl Error {
|
|||
| Repr::Request(_)
|
||||
| Repr::RequestUrl(_)
|
||||
| Repr::ParseUrl(_)
|
||||
| Repr::ComputedHeader(_)
|
||||
| Repr::UnsignableBody
|
||||
| Repr::UnsignableHeader(_)
|
||||
| Repr::Signature(_)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,10 @@ use crate::{Error, error::Repr};
|
|||
/// Returns the headers to add to the request; it never sees a mutable request.
|
||||
pub trait RequestSigner: Send + Sync {
|
||||
fn sign(&self, request: &Request) -> Result<HeaderMap, Error>;
|
||||
|
||||
fn owned_headers(&self) -> &'static [&'static str] {
|
||||
&[]
|
||||
}
|
||||
}
|
||||
|
||||
pub fn json(
|
||||
|
|
@ -65,7 +69,11 @@ pub fn from_http<B: Into<reqwest::Body>>(request: http::Request<B>) -> Result<Re
|
|||
|
||||
pub fn signed(mut request: Request, signer: &dyn RequestSigner) -> Result<Request, Error> {
|
||||
let signature = signer.sign(&request)?;
|
||||
request.headers_mut().extend(signature);
|
||||
let headers = request.headers_mut();
|
||||
for name in signer.owned_headers() {
|
||||
headers.remove(*name);
|
||||
}
|
||||
headers.extend(signature);
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
|
|
@ -159,7 +167,7 @@ mod tests {
|
|||
struct Refuses;
|
||||
impl RequestSigner for Refuses {
|
||||
fn sign(&self, _request: &Request) -> Result<HeaderMap, Error> {
|
||||
Err(Error::computed_header("authorization"))
|
||||
Err(Error::unsignable_body())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -168,7 +176,44 @@ mod tests {
|
|||
&Refuses,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(error.is_computed_header());
|
||||
assert!(!error.dispatched());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_the_signer_owns_is_dropped_even_when_the_signer_omits_it() {
|
||||
struct OwnsDate;
|
||||
impl RequestSigner for OwnsDate {
|
||||
fn sign(&self, _request: &Request) -> Result<HeaderMap, Error> {
|
||||
Ok(HeaderMap::from_iter([(
|
||||
reqwest::header::AUTHORIZATION,
|
||||
HeaderValue::from_static("signed"),
|
||||
)]))
|
||||
}
|
||||
|
||||
fn owned_headers(&self) -> &'static [&'static str] {
|
||||
&["date", "authorization"]
|
||||
}
|
||||
}
|
||||
|
||||
let request = signed(
|
||||
json(
|
||||
"https://u.test/",
|
||||
&[
|
||||
("Date".into(), "caller".into()),
|
||||
("Authorization".into(), "caller".into()),
|
||||
("x-kept".into(), "1".into()),
|
||||
],
|
||||
&json!({}),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
&OwnsDate,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(!request.headers().contains_key("date"));
|
||||
assert_eq!(request.headers()["authorization"], "signed");
|
||||
assert_eq!(request.headers()["x-kept"], "1");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -328,13 +328,23 @@ async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::authorization("Authorization")]
|
||||
#[case::amz_date("x-amz-date")]
|
||||
#[case::security_token("x-amz-security-token")]
|
||||
#[case::date("Date")]
|
||||
#[case::authorization("Authorization", Some("forged"), false, None)]
|
||||
#[case::amz_date("x-amz-date", None, false, Some("x-amz-date"))]
|
||||
#[case::security_token(
|
||||
"x-amz-security-token",
|
||||
None,
|
||||
true,
|
||||
Some("x-amz-date;x-amz-security-token")
|
||||
)]
|
||||
#[case::date("Date", None, false, Some("content-type;date;host"))]
|
||||
#[tokio::test]
|
||||
async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
|
||||
let upstream = wiremock::MockServer::start().await;
|
||||
async fn a_forwarded_header_the_signer_computes_is_sent_as_python_sends_it(
|
||||
#[case] forwarded: &str,
|
||||
#[case] authorization: Option<&str>,
|
||||
#[case] kept: bool,
|
||||
#[case] signed_headers: Option<&str>,
|
||||
) {
|
||||
let upstream = upstream([bedrock_response()]).await;
|
||||
let base = upstream.uri();
|
||||
let call = ChatCompletionsRequest {
|
||||
api_key: None,
|
||||
|
|
@ -351,11 +361,27 @@ async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str)
|
|||
}),
|
||||
)
|
||||
};
|
||||
let error = complete(call)
|
||||
.await
|
||||
.expect_err("conflicting signing headers fail");
|
||||
assert!(error.declined().is_some(), "{error:?}");
|
||||
assert!(received(&upstream).await.is_empty());
|
||||
complete(call).await.expect("call succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
let signature = sent.header("authorization").expect("an authorization");
|
||||
match authorization {
|
||||
Some(expected) => assert_eq!(signature, expected),
|
||||
None => {
|
||||
assert!(signature.starts_with("AWS4-HMAC-SHA256"));
|
||||
assert_eq!(sent.header(forwarded) == Some("forged"), kept);
|
||||
assert!(signed_headers.is_some_and(|names| signature.contains(names)));
|
||||
}
|
||||
}
|
||||
let dated_by_date = forwarded == "Date";
|
||||
assert_eq!(
|
||||
(
|
||||
sent.header("x-amz-date")
|
||||
.is_some_and(|date| date != "forged"),
|
||||
sent.header("date").is_some_and(|date| date != "forged"),
|
||||
),
|
||||
(!dated_by_date, dated_by_date)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ litellm-http.workspace = true
|
|||
litellm-inference.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
litellm-llms-types.workspace = true
|
||||
litellm-python-compat.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
litellm-secrets.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
|
|
|
|||
|
|
@ -52,11 +52,15 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option<M
|
|||
LlmProviders::Tencent => &TENCENT_MESSAGES_CONFIG,
|
||||
LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG,
|
||||
LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG,
|
||||
LlmProviders::Bedrock if model.starts_with("claude_platform/") => {
|
||||
LlmProviders::Bedrock if has_route_prefix(model, "claude_platform/") => {
|
||||
&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
|
||||
}
|
||||
LlmProviders::Bedrock if model.starts_with("mantle/") => &BEDROCK_MANTLE_MESSAGES_CONFIG,
|
||||
LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
|
||||
LlmProviders::Bedrock if has_route_prefix(model, "converse/") => return None,
|
||||
LlmProviders::Bedrock if has_route_prefix(model, "mantle/") => {
|
||||
&BEDROCK_MANTLE_MESSAGES_CONFIG
|
||||
}
|
||||
LlmProviders::Bedrock if model.contains("claude") => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
|
||||
LlmProviders::Bedrock => return None,
|
||||
LlmProviders::BedrockMantle => &BEDROCK_MANTLE_MESSAGES_CONFIG,
|
||||
LlmProviders::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
|
||||
LlmProviders::Edenai => &EDENAI_MESSAGES_CONFIG,
|
||||
|
|
@ -70,6 +74,10 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option<M
|
|||
Some(MessagesProvider { provider, config })
|
||||
}
|
||||
|
||||
fn has_route_prefix(model: &str, prefix: &str) -> bool {
|
||||
model.starts_with(prefix) || model.contains(&format!("/{prefix}"))
|
||||
}
|
||||
|
||||
pub(super) fn string_headers(
|
||||
extra_headers: Option<Map<String, Value>>,
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
|
|
@ -155,6 +163,72 @@ mod tests {
|
|||
assert!(messages_provider(provider, model).is_none());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::invoke_claude("anthropic.claude-sonnet-4-5-20250929-v1:0", Some("bedrock"))]
|
||||
#[case::invoke_prefix("invoke/anthropic.claude-3-5-sonnet", Some("bedrock"))]
|
||||
#[case::cross_region_claude("us.anthropic.claude-opus-4-5", Some("bedrock"))]
|
||||
#[case::claude_arn(
|
||||
"arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5",
|
||||
Some("bedrock")
|
||||
)]
|
||||
#[case::converse_claude("converse/anthropic.claude-sonnet-4-5", None)]
|
||||
#[case::nested_converse("us-west-2/converse/anthropic.claude-sonnet-4-5", None)]
|
||||
#[case::non_claude("amazon.nova-pro-v1:0", None)]
|
||||
#[case::non_claude_invoke("invoke/meta.llama3-70b-instruct-v1:0", None)]
|
||||
#[case::capitalized_claude_is_not_claude("anthropic.Claude-x", None)]
|
||||
#[case::mantle("mantle/anthropic.claude-sonnet-5", Some("bedrock"))]
|
||||
#[case::claude_platform("claude_platform/claude-sonnet-4-5", Some("bedrock"))]
|
||||
#[case::bedrock_mantle_provider_prefix_is_not_the_mantle_route(
|
||||
"bedrock_mantle/openai.gpt-5",
|
||||
None
|
||||
)]
|
||||
fn bedrock_models_route_as_python_selects_their_messages_config(
|
||||
#[case] model: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(
|
||||
messages_provider(LlmProviders::Bedrock, model).map(|provider| provider.as_str()),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::invoke("anthropic.claude-3", false, false)]
|
||||
#[case::mantle("mantle/anthropic.claude-sonnet-5", true, false)]
|
||||
#[case::claude_platform("claude_platform/claude-sonnet-4-5", false, true)]
|
||||
fn bedrock_route_prefixes_pick_their_config(
|
||||
#[case] model: &str,
|
||||
#[case] mantle: bool,
|
||||
#[case] claude_platform: bool,
|
||||
) {
|
||||
use litellm_llms::{
|
||||
base_llm::messages::transformation::BaseMessagesConfig,
|
||||
bedrock::claude_platform::messages::transformation::BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG,
|
||||
bedrock_mantle::messages::transformation::BEDROCK_MANTLE_MESSAGES_CONFIG,
|
||||
};
|
||||
let params = litellm_router_types::LitellmParams::default();
|
||||
let region = |key: &str| {
|
||||
key.starts_with("AWS_REGION")
|
||||
.then(|| "us-east-1".to_owned())
|
||||
};
|
||||
let url = |config: &dyn BaseMessagesConfig| {
|
||||
format!(
|
||||
"{:?}",
|
||||
config.get_complete_url(None, model, ¶ms, false, ®ion)
|
||||
)
|
||||
};
|
||||
let routed = url(messages_provider(LlmProviders::Bedrock, model)
|
||||
.unwrap()
|
||||
.config());
|
||||
assert_eq!(
|
||||
(
|
||||
routed == url(&BEDROCK_MANTLE_MESSAGES_CONFIG),
|
||||
routed == url(&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG),
|
||||
),
|
||||
(mantle, claude_platform)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::int(json!(12345), "int")]
|
||||
#[case::float(json!(1.5), "float")]
|
||||
|
|
|
|||
|
|
@ -9,15 +9,18 @@ use litellm_core_utils::{
|
|||
dot_notation_indexing::delete_nested_value,
|
||||
get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
|
||||
};
|
||||
use litellm_host::interceptors::{
|
||||
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
|
||||
ResponseHead, ResultSource, WireRequest,
|
||||
use litellm_host::{
|
||||
interceptors::{
|
||||
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
|
||||
ResponseHead, ResultSource, WireRequest,
|
||||
},
|
||||
protocol::UnobservedTail,
|
||||
};
|
||||
use litellm_llms::base_llm::{
|
||||
auth::{Authenticated, AzureAdTokenHosts, ValidatedEnvironment, resolve_auth},
|
||||
messages::{
|
||||
context::MessagesTransformContext,
|
||||
streaming::{self, ByteStream, StreamDecoder},
|
||||
streaming::{ByteStream, StreamDecoder},
|
||||
transformation::{BaseMessagesConfig, FederatedEnvironment},
|
||||
},
|
||||
};
|
||||
|
|
@ -53,12 +56,38 @@ struct ProviderCall {
|
|||
stream: bool,
|
||||
whole_sse_frames: bool,
|
||||
joined_reads: bool,
|
||||
detached_drain: Option<litellm_host::protocol::DetachedDrain>,
|
||||
transform_context: MessagesTransformContext,
|
||||
native_output_fallback: Option<NativeOutputFallback>,
|
||||
ascii_response_headers: bool,
|
||||
unencodable_request_header: Option<Vec<u8>>,
|
||||
ascii_request_headers: bool,
|
||||
}
|
||||
|
||||
struct NativeOutputFallback {
|
||||
input: MessagesRequest,
|
||||
context: MessagesTransformContext,
|
||||
headers: Vec<(String, String)>,
|
||||
tool_custom_field: litellm_llms::base_llm::messages::tools::ToolCustomField,
|
||||
drops_unsupported_params: bool,
|
||||
drop_params: bool,
|
||||
}
|
||||
|
||||
impl NativeOutputFallback {
|
||||
fn body(&self, config: &dyn BaseMessagesConfig) -> Result<Value, Error> {
|
||||
let rebuilt = config.request_body(
|
||||
self.tool_custom_field.apply(
|
||||
config.transform_anthropic_messages_request(self.input.clone(), &self.context)?,
|
||||
self.drops_unsupported_params,
|
||||
),
|
||||
&self.headers,
|
||||
self.drop_params,
|
||||
)?;
|
||||
let (rebuilt, _) = config.finalize_request(rebuilt, self.headers.clone(), &self.context)?;
|
||||
Ok(config.wire_body(serde_json::to_value(&rebuilt).map_err(serialize_failure)?))
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCall {
|
||||
fn cacheable(&self) -> bool {
|
||||
self.signer.is_none()
|
||||
|
|
@ -290,7 +319,7 @@ async fn provider_call(
|
|||
)?;
|
||||
let transform_context = MessagesTransformContext {
|
||||
drops_unsupported_params: shaping.settings.drops_unsupported_params,
|
||||
logs_dropped_params: config.logs_dropped_params(),
|
||||
global_drop_params: shaping.settings.global_drop_params,
|
||||
..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params)
|
||||
};
|
||||
let dropped_keys = top_level_paths(&shaping.settings.additional_drop_params);
|
||||
|
|
@ -358,6 +387,9 @@ async fn provider_call(
|
|||
|
||||
let tool_custom_field = config.tool_custom_field(&trimmed.model);
|
||||
let requested_output_format = trimmed.params.output_format.clone();
|
||||
let fallback = config
|
||||
.native_output_fallback(&trimmed, &transform_context)
|
||||
.map(|fallback_context| (trimmed.clone(), fallback_context));
|
||||
let body = config.request_body(
|
||||
tool_custom_field.apply(
|
||||
config.transform_anthropic_messages_request(trimmed, &transform_context)?,
|
||||
|
|
@ -373,17 +405,26 @@ async fn provider_call(
|
|||
body.params.streams(),
|
||||
&env_lookup,
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: config.headers_for_url(
|
||||
config.request_headers(
|
||||
with_python_default_headers(validated.headers, config.default_headers()),
|
||||
&with_requested_output_format(&body, requested_output_format),
|
||||
),
|
||||
&url,
|
||||
let headers = config.headers_for_url(
|
||||
config.request_headers(
|
||||
with_python_default_headers(validated.headers, config.default_headers()),
|
||||
&with_requested_output_format(&body, requested_output_format),
|
||||
),
|
||||
&url,
|
||||
);
|
||||
let native_output_fallback = fallback.map(|(input, context)| NativeOutputFallback {
|
||||
input,
|
||||
context,
|
||||
headers: headers.clone(),
|
||||
tool_custom_field,
|
||||
drops_unsupported_params: shaping.settings.drops_unsupported_params,
|
||||
drop_params: shaping.settings.drop_params,
|
||||
});
|
||||
let (body, headers) = config.finalize_request(body, headers, &transform_context)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers,
|
||||
auth: validated.auth,
|
||||
};
|
||||
let body = config.finalize_request(body, &environment.headers, &transform_context);
|
||||
|
||||
let request_context = RequestContext {
|
||||
model: body.model.clone(),
|
||||
|
|
@ -392,6 +433,11 @@ async fn provider_call(
|
|||
secret_fields: Vec::new(),
|
||||
api_key: api_key.map(SecretValue::new),
|
||||
};
|
||||
let serialized = serde_json::to_value(&body).map_err(serialize_failure)?;
|
||||
let serialized_stream = serialized.get("stream").is_some();
|
||||
let wire_body = config.wire_body(serialized);
|
||||
let stream_moved_to_url =
|
||||
body.params.streams() && serialized_stream && wire_body.get("stream").is_none();
|
||||
let authenticated = resolve_auth(credentials.auth, environment, &|key| {
|
||||
std::env::var(key).ok()
|
||||
})
|
||||
|
|
@ -406,15 +452,15 @@ async fn provider_call(
|
|||
WireRequest {
|
||||
url,
|
||||
headers: joined_header_values(authenticated.headers),
|
||||
body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
|
||||
body: wire_body,
|
||||
},
|
||||
request_context,
|
||||
)
|
||||
.await?;
|
||||
let stream = wire
|
||||
.body
|
||||
.get("stream")
|
||||
.is_some_and(litellm_llms_types::recognized::python_truthy);
|
||||
let stream = wire.body.get("stream").map_or(
|
||||
stream_moved_to_url,
|
||||
litellm_llms_types::recognized::python_truthy,
|
||||
);
|
||||
Ok(ProviderCall {
|
||||
identity,
|
||||
wire,
|
||||
|
|
@ -424,7 +470,9 @@ async fn provider_call(
|
|||
stream,
|
||||
whole_sse_frames: shaping.settings.whole_sse_frames,
|
||||
joined_reads: shaping.settings.joined_reads,
|
||||
detached_drain: shaping.settings.detached_drain(),
|
||||
transform_context,
|
||||
native_output_fallback,
|
||||
ascii_response_headers: shaping.settings.ascii_response_headers,
|
||||
unencodable_request_header: shaping.settings.unencodable_request_header,
|
||||
ascii_request_headers: shaping.settings.ascii_request_headers,
|
||||
|
|
@ -498,6 +546,13 @@ fn requested_optional_params(
|
|||
if provider.accepts_service_tier() {
|
||||
return Ok(request);
|
||||
}
|
||||
if request.params.service_tier.is_some() {
|
||||
litellm_tracing::warn!(
|
||||
"Dropping `service_tier` for model={} on provider={}. Only the Anthropic API routes accept it.",
|
||||
request.model,
|
||||
provider.as_str()
|
||||
);
|
||||
}
|
||||
Ok(MessagesRequest {
|
||||
params: MessagesOptionalParams {
|
||||
service_tier: None,
|
||||
|
|
@ -560,6 +615,7 @@ async fn call_provider(
|
|||
stream,
|
||||
whole_sse_frames,
|
||||
joined_reads,
|
||||
detached_drain,
|
||||
..
|
||||
} = request;
|
||||
let provider_name = provider.as_str();
|
||||
|
|
@ -581,6 +637,7 @@ async fn call_provider(
|
|||
timeout,
|
||||
joined_reads,
|
||||
);
|
||||
let streamed = draining_after_detach(streamed, config.stream_decoder(), detached_drain);
|
||||
return Ok(if whole_sse_frames {
|
||||
framed(streamed)
|
||||
} else {
|
||||
|
|
@ -616,6 +673,7 @@ async fn hosted_response(
|
|||
identity,
|
||||
provider,
|
||||
whole_sse_frames,
|
||||
detached_drain,
|
||||
..
|
||||
} = request;
|
||||
let provider_name = provider.as_str();
|
||||
|
|
@ -633,6 +691,7 @@ async fn hosted_response(
|
|||
.await?;
|
||||
let streamed =
|
||||
hosted_streaming_response(headers, chunks, config.stream_decoder(), provider_name);
|
||||
let streamed = draining_after_detach(streamed, config.stream_decoder(), detached_drain);
|
||||
return Ok(if whole_sse_frames {
|
||||
framed(streamed)
|
||||
} else {
|
||||
|
|
@ -706,6 +765,7 @@ async fn send_recovering(
|
|||
let config = request.provider.config();
|
||||
let mut body = std::borrow::Cow::Borrowed(&request.wire.body);
|
||||
let mut attempt = 1;
|
||||
let mut recovered: Vec<(u16, String)> = Vec::new();
|
||||
loop {
|
||||
log_request_body(request.provider.as_str(), request.stream, &body);
|
||||
if let Some(value) = &request.unencodable_request_header {
|
||||
|
|
@ -779,6 +839,20 @@ async fn send_recovering(
|
|||
}
|
||||
hosted @ Upstream::Hosted(_) => return Ok((hosted, sent_body)),
|
||||
};
|
||||
if let Some(fallback) = request.native_output_fallback.as_ref().filter(|_| {
|
||||
config.retries_without_native_output(
|
||||
failure.status,
|
||||
&failure.text,
|
||||
&body,
|
||||
&request.identity.model,
|
||||
)
|
||||
}) {
|
||||
let rebuilt = fallback.body(config)?;
|
||||
let replayed = replayed_recoveries(config, rebuilt, &recovered, request);
|
||||
body = std::borrow::Cow::Owned(replayed.body.clone());
|
||||
context.provider_request_retried(replayed).await?;
|
||||
continue;
|
||||
}
|
||||
let retried = if attempt == MESSAGES_HTTP_ERROR_ATTEMPTS {
|
||||
None
|
||||
} else {
|
||||
|
|
@ -793,6 +867,7 @@ async fn send_recovering(
|
|||
return Err(failure.into());
|
||||
};
|
||||
attempt += 1;
|
||||
recovered.push((failure.status, failure.text.clone()));
|
||||
tracing::debug!(
|
||||
"Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt {attempt}/{MESSAGES_HTTP_ERROR_ATTEMPTS})."
|
||||
);
|
||||
|
|
@ -801,6 +876,36 @@ async fn send_recovering(
|
|||
}
|
||||
}
|
||||
|
||||
fn replayed_recoveries(
|
||||
config: &dyn BaseMessagesConfig,
|
||||
rebuilt: Value,
|
||||
recovered: &[(u16, String)],
|
||||
request: &ProviderCall,
|
||||
) -> litellm_host::interceptors::RetriedRequest {
|
||||
recovered.iter().fold(
|
||||
litellm_host::interceptors::RetriedRequest {
|
||||
reasons: Vec::new(),
|
||||
body: rebuilt,
|
||||
},
|
||||
|replayed, (status, text)| match config.retry_after_http_error(
|
||||
*status,
|
||||
text,
|
||||
&replayed.body,
|
||||
&request.transform_context,
|
||||
) {
|
||||
Some(retried) => litellm_host::interceptors::RetriedRequest {
|
||||
reasons: replayed
|
||||
.reasons
|
||||
.into_iter()
|
||||
.chain(retried.reasons)
|
||||
.collect(),
|
||||
body: retried.body,
|
||||
},
|
||||
None => replayed,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn serialize_failure(err: serde_json::Error) -> Error {
|
||||
Error::invalid_request(litellm_llms::ErrorDetail::failed(
|
||||
"Anthropic messages request serialization",
|
||||
|
|
@ -972,16 +1077,23 @@ fn streaming_response(
|
|||
_ => chunks.boxed(),
|
||||
}
|
||||
.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk));
|
||||
let chunks = match decoder {
|
||||
let unobserved_tail = decoder.map(|_| UnobservedTail::default());
|
||||
let chunks = match decoder.zip(unobserved_tail.clone()) {
|
||||
None => chunks.map_err(Error::from).boxed(),
|
||||
Some(decode) => decoded_chunks(
|
||||
Some((decode, tail)) => decoded_chunks(
|
||||
chunks.map_err(std::io::Error::other).boxed(),
|
||||
decode,
|
||||
headers.clone(),
|
||||
provider,
|
||||
tail,
|
||||
),
|
||||
};
|
||||
MessagesCallResponse::Stream {
|
||||
head: super::route::MessagesStreamHead { headers },
|
||||
head: super::route::MessagesStreamHead {
|
||||
headers,
|
||||
detached_drain: None,
|
||||
unobserved_tail,
|
||||
},
|
||||
chunks,
|
||||
}
|
||||
}
|
||||
|
|
@ -993,20 +1105,44 @@ fn hosted_streaming_response(
|
|||
provider: &'static str,
|
||||
) -> MessagesCallResponse {
|
||||
let chunks = chunks.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk));
|
||||
let chunks = match decoder {
|
||||
let unobserved_tail = decoder.map(|_| UnobservedTail::default());
|
||||
let chunks = match decoder.zip(unobserved_tail.clone()) {
|
||||
None => chunks.boxed(),
|
||||
Some(decode) => decoded_chunks(
|
||||
Some((decode, tail)) => decoded_chunks(
|
||||
chunks.map_err(std::io::Error::other).boxed(),
|
||||
decode,
|
||||
headers.clone(),
|
||||
provider,
|
||||
tail,
|
||||
),
|
||||
};
|
||||
MessagesCallResponse::Stream {
|
||||
head: super::route::MessagesStreamHead { headers },
|
||||
head: super::route::MessagesStreamHead {
|
||||
headers,
|
||||
detached_drain: None,
|
||||
unobserved_tail,
|
||||
},
|
||||
chunks,
|
||||
}
|
||||
}
|
||||
|
||||
fn draining_after_detach(
|
||||
response: MessagesCallResponse,
|
||||
decoder: Option<StreamDecoder>,
|
||||
detached_drain: Option<litellm_host::protocol::DetachedDrain>,
|
||||
) -> MessagesCallResponse {
|
||||
match response {
|
||||
MessagesCallResponse::Stream { head, chunks } => MessagesCallResponse::Stream {
|
||||
head: super::route::MessagesStreamHead {
|
||||
detached_drain: decoder.and(detached_drain),
|
||||
..head
|
||||
},
|
||||
chunks,
|
||||
},
|
||||
complete => complete,
|
||||
}
|
||||
}
|
||||
|
||||
fn framed(response: MessagesCallResponse) -> MessagesCallResponse {
|
||||
let MessagesCallResponse::Stream { head, chunks } = response else {
|
||||
return response;
|
||||
|
|
@ -1056,17 +1192,14 @@ fn complete_frames_end(pending: &[u8]) -> usize {
|
|||
fn decoded_chunks(
|
||||
bytes: ByteStream,
|
||||
decode: StreamDecoder,
|
||||
headers: Vec<(String, String)>,
|
||||
provider: &'static str,
|
||||
tail: UnobservedTail,
|
||||
) -> BoxStream<'static, Result<Bytes, Error>> {
|
||||
futures_util::stream::try_unfold(decode(bytes), move |mut events| async move {
|
||||
let Some(event) = events.try_next().await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let chunk = streaming::encode(&event)?;
|
||||
log_chunk(provider, "client_response", &chunk);
|
||||
Ok(Some((chunk, events)))
|
||||
})
|
||||
.boxed()
|
||||
let frames = decode(bytes, headers).map_err(Error::from).boxed();
|
||||
super::relay::relayed(frames, tail)
|
||||
.inspect_ok(move |chunk| log_chunk(provider, "client_response", chunk))
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn log_request_body(provider: &str, stream: bool, body: &serde_json::Value) {
|
||||
|
|
@ -1112,6 +1245,7 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_llms::base_llm::messages::streaming;
|
||||
use litellm_llms_types::headers::ProviderSpecificHeaders;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
|
@ -1849,8 +1983,8 @@ mod tests {
|
|||
|
||||
#[rstest]
|
||||
#[case::event(
|
||||
"data: {\"type\":\"ping\"}\n\n",
|
||||
Some("event: ping\ndata: {\"type\":\"ping\"}\n\n")
|
||||
"data: {\"type\":\"message_stop\"}\n\n",
|
||||
Some("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n")
|
||||
)]
|
||||
#[rstest::rstest]
|
||||
#[case::invalid_event("data: invalid\n\ndata: {\"type\":\"ping\"}\n\n", None)]
|
||||
|
|
@ -1871,7 +2005,11 @@ mod tests {
|
|||
.unwrap();
|
||||
let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response(
|
||||
response,
|
||||
Some(streaming::event_stream),
|
||||
Some(|bytes, _| {
|
||||
streaming::event_stream(bytes)
|
||||
.and_then(|event| futures_util::future::ready(streaming::encode(&event)))
|
||||
.boxed()
|
||||
}),
|
||||
"test",
|
||||
Duration::from_secs(5),
|
||||
false,
|
||||
|
|
@ -1887,6 +2025,78 @@ mod tests {
|
|||
assert!(chunks.next().await.is_none());
|
||||
}
|
||||
|
||||
async fn upstream_stream(body: &'static str) -> reqwest::Response {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(any())
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(body, "text/event-stream"))
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
litellm_http::Client::for_test()
|
||||
.get(upstream.uri())
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn reencoding(bytes: ByteStream, _: Vec<(String, String)>) -> streaming::FrameStream {
|
||||
streaming::event_stream(bytes)
|
||||
.and_then(|event| futures_util::future::ready(streaming::encode(&event)))
|
||||
.boxed()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::decoded_and_cut_short(
|
||||
Some(reencoding as StreamDecoder),
|
||||
"data: {\"type\":\"ping\"}\n\n",
|
||||
true
|
||||
)]
|
||||
#[case::decoded_and_stopped(
|
||||
Some(reencoding as StreamDecoder),
|
||||
"data: {\"type\":\"ping\"}\n\ndata: {\"type\":\"message_stop\"}\n\n",
|
||||
false
|
||||
)]
|
||||
#[case::passed_through_and_cut_short(None, "data: {\"type\":\"ping\"}\n\n", false)]
|
||||
#[tokio::test]
|
||||
async fn a_decoded_stream_cut_short_ends_with_the_incomplete_stream_error_on_its_tail(
|
||||
#[case] decoder: Option<StreamDecoder>,
|
||||
#[case] body: &'static str,
|
||||
#[case] incomplete: bool,
|
||||
) {
|
||||
let relays = decoder.is_some();
|
||||
let MessagesCallResponse::Stream { head, mut chunks } = streaming_response(
|
||||
upstream_stream(body).await,
|
||||
decoder,
|
||||
"test",
|
||||
Duration::from_secs(5),
|
||||
false,
|
||||
) else {
|
||||
panic!("a streaming response returns chunks");
|
||||
};
|
||||
let tail = head.unobserved_tail;
|
||||
assert_eq!(tail.is_some(), relays);
|
||||
let mut delivered = Vec::new();
|
||||
let mut on_tail = Vec::new();
|
||||
while let Some(chunk) = chunks.try_next().await.unwrap() {
|
||||
match tail.as_ref().is_some_and(UnobservedTail::begun) {
|
||||
true => on_tail.push(chunk),
|
||||
false => delivered.push(chunk),
|
||||
}
|
||||
}
|
||||
let error_event = Bytes::from(format!(
|
||||
"event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{}\"}}}}\n\n",
|
||||
super::super::relay::INCOMPLETE_STREAM_ERROR_MESSAGE
|
||||
));
|
||||
assert!(!delivered.contains(&error_event));
|
||||
assert_eq!(
|
||||
on_tail,
|
||||
if incomplete {
|
||||
vec![error_event]
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_delimiter(b"data: a".as_slice(), 0)]
|
||||
#[case::lf(b"data: a\n\ndata: b", 9)]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ mod handler;
|
|||
mod host_answer;
|
||||
mod pre_request;
|
||||
mod reads;
|
||||
mod relay;
|
||||
pub mod route;
|
||||
mod safeguards;
|
||||
mod transport;
|
||||
|
|
|
|||
202
litellm/rust/crates/inference-messages/src/relay.rs
Normal file
202
litellm/rust/crates/inference-messages/src/relay.rs
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
use bytes::Bytes;
|
||||
use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
|
||||
use litellm_host::protocol::UnobservedTail;
|
||||
use litellm_python_compat::{Value, json};
|
||||
|
||||
use super::Error;
|
||||
|
||||
pub(crate) const INCOMPLETE_STREAM_ERROR_MESSAGE: &str = "Provider stream ended before emitting a message_stop event; the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated.";
|
||||
|
||||
type Frames = BoxStream<'static, Result<Bytes, Error>>;
|
||||
|
||||
/// `_incomplete_stream_error_sse_event`.
|
||||
fn incomplete_stream_error_event() -> Bytes {
|
||||
Bytes::from(format!(
|
||||
"event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{INCOMPLETE_STREAM_ERROR_MESSAGE}\"}}}}\n\n"
|
||||
))
|
||||
}
|
||||
|
||||
/// `bytes.splitlines()`.
|
||||
fn lines(chunk: &[u8]) -> impl Iterator<Item = &[u8]> {
|
||||
let mut rest = Some(chunk);
|
||||
std::iter::from_fn(move || {
|
||||
let current = rest.filter(|current| !current.is_empty())?;
|
||||
match current
|
||||
.iter()
|
||||
.position(|byte| matches!(byte, b'\n' | b'\r'))
|
||||
{
|
||||
Some(at) => {
|
||||
let skip = if current[at..].starts_with(b"\r\n") {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
rest = Some(¤t[at + skip..]);
|
||||
Some(¤t[..at])
|
||||
}
|
||||
None => {
|
||||
rest = None;
|
||||
Some(current)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn field<'a>(fields: &'a [(Value, Value)], name: &str) -> Option<&'a Value> {
|
||||
fields
|
||||
.iter()
|
||||
.find(|(key, _)| matches!(key, Value::Str(key) if key == name))
|
||||
.map(|(_, value)| value)
|
||||
}
|
||||
|
||||
/// `_decoded_sse_data_line`.
|
||||
fn decoded_data_line(line: &[u8]) -> Option<Value> {
|
||||
let data = line.strip_prefix(b"data:")?.trim_ascii();
|
||||
json::loads(std::str::from_utf8(data).ok()?).ok()
|
||||
}
|
||||
|
||||
/// `is_message_stop_chunk` for a chunk of SSE bytes.
|
||||
fn is_message_stop_chunk(chunk: &[u8]) -> bool {
|
||||
lines(chunk).any(|line| line == b"event: message_stop")
|
||||
}
|
||||
|
||||
/// `is_provider_error_chunk` for a chunk of SSE bytes.
|
||||
fn is_provider_error_chunk(chunk: &[u8]) -> bool {
|
||||
lines(chunk)
|
||||
.filter_map(decoded_data_line)
|
||||
.find_map(|candidate| match candidate {
|
||||
Value::Dict(fields)
|
||||
if matches!(field(&fields, "type"), Some(Value::Str(kind)) if kind == "error") =>
|
||||
{
|
||||
Some(fields)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.is_some_and(|fields| matches!(field(&fields, "error"), Some(Value::Dict(_))))
|
||||
}
|
||||
|
||||
/// `_is_terminal_stream_chunk`.
|
||||
fn is_terminal_stream_chunk(chunk: &[u8]) -> bool {
|
||||
is_message_stop_chunk(chunk) || is_provider_error_chunk(chunk)
|
||||
}
|
||||
|
||||
struct Relay {
|
||||
frames: Option<Frames>,
|
||||
terminal: bool,
|
||||
tail: UnobservedTail,
|
||||
}
|
||||
|
||||
/// A decoded provider stream as `BaseAnthropicMessagesStreamingIterator.async_sse_wrapper`
|
||||
/// relays it: one that ends without a `message_stop` or provider error event closes with an
|
||||
/// `api_error` event. Python adds that event after billing, so it goes out on `tail`, which the
|
||||
/// stream hooks never observe.
|
||||
pub(crate) fn relayed(frames: Frames, tail: UnobservedTail) -> Frames {
|
||||
let relay = Relay {
|
||||
frames: Some(frames),
|
||||
terminal: false,
|
||||
tail,
|
||||
};
|
||||
futures_util::stream::try_unfold(relay, |mut relay| async move {
|
||||
let Some(frames) = relay.frames.as_mut() else {
|
||||
return Ok(None);
|
||||
};
|
||||
if let Some(frame) = frames.try_next().await? {
|
||||
relay.terminal |= is_terminal_stream_chunk(&frame);
|
||||
return Ok(Some((frame, relay)));
|
||||
}
|
||||
relay.frames = None;
|
||||
if relay.terminal {
|
||||
return Ok(None);
|
||||
}
|
||||
relay.tail.begin();
|
||||
Ok(Some((incomplete_stream_error_event(), relay)))
|
||||
})
|
||||
.boxed()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
const START: &str = "event: message_start\ndata: {\"type\": \"message_start\"}\n\n";
|
||||
const STOP: &str = "event: message_stop\ndata: {\"type\": \"message_stop\"}\n\n";
|
||||
const PROVIDER_ERROR: &str = "event: error\ndata: {\"type\": \"error\", \"error\": {\"type\": \"overloaded_error\"}}\n\n";
|
||||
const UNSHAPED_ERROR: &str =
|
||||
"event: error\ndata: {\"type\": \"error\", \"error\": \"overloaded\"}\n\n";
|
||||
|
||||
fn frames(items: Vec<Result<&'static str, Error>>) -> Frames {
|
||||
futures_util::stream::iter(items.into_iter().map(|item| item.map(Bytes::from))).boxed()
|
||||
}
|
||||
|
||||
async fn relayed_frames(
|
||||
items: Vec<Result<&'static str, Error>>,
|
||||
) -> (Vec<Result<Bytes, Error>>, UnobservedTail) {
|
||||
let tail = UnobservedTail::default();
|
||||
let relayed = relayed(frames(items), tail.clone()).collect().await;
|
||||
(relayed, tail)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::message_stop(vec![START, STOP])]
|
||||
#[case::provider_error(vec![START, PROVIDER_ERROR])]
|
||||
#[case::stop_then_more(vec![START, STOP, START])]
|
||||
#[tokio::test]
|
||||
async fn a_terminated_stream_is_relayed_as_it_is(#[case] sent: Vec<&'static str>) {
|
||||
let (relayed, tail) = relayed_frames(sent.iter().copied().map(Ok).collect()).await;
|
||||
let relayed: Vec<Bytes> = relayed.into_iter().map(Result::unwrap).collect();
|
||||
assert_eq!(
|
||||
relayed,
|
||||
sent.into_iter().map(Bytes::from).collect::<Vec<_>>()
|
||||
);
|
||||
assert!(!tail.begun());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty(vec![])]
|
||||
#[case::cut_short(vec![START])]
|
||||
#[case::error_without_an_error_object(vec![START, UNSHAPED_ERROR])]
|
||||
#[tokio::test]
|
||||
async fn an_unterminated_stream_ends_with_the_incomplete_stream_error_on_the_tail(
|
||||
#[case] sent: Vec<&'static str>,
|
||||
) {
|
||||
let (relayed, tail) = relayed_frames(sent.iter().copied().map(Ok).collect()).await;
|
||||
let mut relayed: Vec<Bytes> = relayed.into_iter().map(Result::unwrap).collect();
|
||||
assert_eq!(
|
||||
relayed.pop().unwrap(),
|
||||
Bytes::from(format!(
|
||||
"event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{INCOMPLETE_STREAM_ERROR_MESSAGE}\"}}}}\n\n"
|
||||
))
|
||||
);
|
||||
assert_eq!(
|
||||
relayed,
|
||||
sent.into_iter().map(Bytes::from).collect::<Vec<_>>()
|
||||
);
|
||||
assert!(tail.begun());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_stream_ends_with_its_failure() {
|
||||
let failure = Error::from(litellm_http::Error::upstream(500, "cut".to_owned()));
|
||||
let (relayed, tail) = relayed_frames(vec![Ok(START), Err(failure)]).await;
|
||||
assert_eq!(relayed.len(), 2);
|
||||
assert_eq!(relayed[0].as_ref().unwrap(), START.as_bytes());
|
||||
assert!(relayed[1].is_err());
|
||||
assert!(!tail.begun());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::crlf("event: message_stop\r\ndata: {}\r\n\r\n", true)]
|
||||
#[case::cr("event: ping\revent: message_stop\r", true)]
|
||||
#[case::unspaced_event("event:message_stop\n\n", false)]
|
||||
#[case::error_data_padded("data: {\"type\": \"error\", \"error\": {}} \n\n", true)]
|
||||
#[case::first_error_payload_decides(
|
||||
"data: {\"type\": \"error\", \"error\": 1}\ndata: {\"type\": \"error\", \"error\": {}}\n\n",
|
||||
false
|
||||
)]
|
||||
#[case::undecodable_data("data: {\n\n", false)]
|
||||
fn terminal_chunks_are_python_s(#[case] chunk: &str, #[case] terminal: bool) {
|
||||
assert_eq!(is_terminal_stream_chunk(chunk.as_bytes()), terminal);
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,8 @@ pub type MessagesOutput = HostedCompletion<MessagesBody>;
|
|||
/// The upstream response as the caller sees it at stream hand-off, before any chunk.
|
||||
pub struct MessagesStreamHead {
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub detached_drain: Option<litellm_host::protocol::DetachedDrain>,
|
||||
pub unobserved_tail: Option<litellm_host::protocol::UnobservedTail>,
|
||||
}
|
||||
|
||||
pub struct Messages;
|
||||
|
|
@ -24,6 +26,16 @@ impl Protocol for Messages {
|
|||
type HostCall = Infallible;
|
||||
type Chunk = Bytes;
|
||||
type StreamHead = MessagesStreamHead;
|
||||
|
||||
fn detached_drain(head: &MessagesStreamHead) -> Option<litellm_host::protocol::DetachedDrain> {
|
||||
head.detached_drain
|
||||
}
|
||||
|
||||
fn unobserved_tail(
|
||||
head: &MessagesStreamHead,
|
||||
) -> Option<litellm_host::protocol::UnobservedTail> {
|
||||
head.unobserved_tail.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub type MessagesMachine = HostedMachine<Messages>;
|
||||
|
|
@ -66,6 +78,8 @@ impl litellm_inference::caching::StreamCachable for Messages {
|
|||
Some(litellm_host::call::CallOutput::Stream {
|
||||
head: MessagesStreamHead {
|
||||
headers: Vec::new(),
|
||||
detached_drain: None,
|
||||
unobserved_tail: None,
|
||||
},
|
||||
chunks: Box::pin(futures_util::stream::iter([Ok(data)])),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -168,6 +168,8 @@ mod tests {
|
|||
MessagesCallResponse::Stream {
|
||||
head: MessagesStreamHead {
|
||||
headers: Vec::new(),
|
||||
detached_drain: None,
|
||||
unobserved_tail: None,
|
||||
},
|
||||
chunks: futures_util::stream::iter(chunks).boxed(),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,15 +1,13 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
|
||||
use litellm_host::call::CallOutput;
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
use litellm_llms::{
|
||||
ErrorDetail,
|
||||
base_llm::messages::context::{MessagesModelCapabilities, MessagesShapeOptions},
|
||||
};
|
||||
use litellm_llms_types::{
|
||||
formats::messages::{MessagesRequest, MessagesResponse, SystemPrompt},
|
||||
formats::messages::{MessagesRequest, MessagesResponse},
|
||||
headers::ProviderSpecificHeaders,
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
|
|
@ -31,32 +29,10 @@ pub struct MessagesCall {
|
|||
pub shaping: MessagesShaping,
|
||||
}
|
||||
|
||||
/// The request body. `custom_llm_provider` is the call's, as the provider is resolved from it
|
||||
/// and the model.
|
||||
pub fn messages_body(
|
||||
body: Map<String, Value>,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
if !reads_unparsed_system(&body, custom_llm_provider)
|
||||
&& let Some(system) = body.get(SYSTEM).filter(|system| !system.is_null())
|
||||
&& let Err(err) = SystemPrompt::deserialize(system)
|
||||
{
|
||||
return Err(invalid_request(err));
|
||||
}
|
||||
pub fn messages_body(body: Map<String, Value>) -> Result<MessagesRequest, Error> {
|
||||
serde_json::from_value(Value::Object(body)).map_err(invalid_request)
|
||||
}
|
||||
|
||||
const SYSTEM: &str = "system";
|
||||
|
||||
/// Only `azure_ai` keeps a system prompt Rust could not type as sent, as Python does. By the
|
||||
/// owner's ruling every other provider, `anthropic` included, keeps main's request error for it.
|
||||
fn reads_unparsed_system(body: &Map<String, Value>, custom_llm_provider: Option<&str>) -> bool {
|
||||
body.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|model| resolve_llm_provider(model, custom_llm_provider, "messages").ok())
|
||||
.is_some_and(|resolved| resolved.provider == LlmProviders::AzureAi)
|
||||
}
|
||||
|
||||
/// The caller's litellm params, projected by a host from the keys [`LitellmParams::fields`]
|
||||
/// names. A key present with a value of the wrong type is a request error, as it is for
|
||||
/// Python's `GenericLiteLLMParams(**kwargs)`.
|
||||
|
|
@ -110,6 +86,8 @@ pub struct MessagesSettings {
|
|||
#[serde(default)]
|
||||
pub drops_unsupported_params: bool,
|
||||
#[serde(default)]
|
||||
pub global_drop_params: bool,
|
||||
#[serde(default)]
|
||||
pub intercepted: bool,
|
||||
#[serde(default)]
|
||||
pub mcp_gateway: bool,
|
||||
|
|
@ -126,6 +104,10 @@ pub struct MessagesSettings {
|
|||
/// settings dict.
|
||||
#[serde(skip)]
|
||||
pub module_api_key: Option<SecretValue>,
|
||||
#[serde(default)]
|
||||
pub detached_drain_timeout: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub detached_drain_limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl MessagesSettings {
|
||||
|
|
@ -133,10 +115,18 @@ impl MessagesSettings {
|
|||
MessagesShapeOptions {
|
||||
reasoning_auto_summary: self.reasoning_auto_summary,
|
||||
presanitized: self.presanitized,
|
||||
..MessagesShapeOptions::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Python's Bedrock Messages relay keeps reading the provider stream after its client leaves,
|
||||
/// for at most this long and in at most this many calls at once.
|
||||
pub fn detached_drain(&self) -> Option<litellm_host::protocol::DetachedDrain> {
|
||||
Some(litellm_host::protocol::DetachedDrain {
|
||||
timeout: Duration::try_from_secs_f64(self.detached_drain_timeout?).ok()?,
|
||||
limit: self.detached_drain_limit?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn request_timeout(&self) -> Option<Duration> {
|
||||
self.request_timeout
|
||||
.and_then(|secs| Duration::try_from_secs_f64(secs).ok())
|
||||
|
|
@ -225,6 +215,7 @@ mod tests {
|
|||
supports_sampling_params: false,
|
||||
supports_speed: true,
|
||||
supports_mid_conversation_system: false,
|
||||
bedrock: Default::default(),
|
||||
effort_tiers: SupportedEffortTiers {
|
||||
minimal: false,
|
||||
low: true,
|
||||
|
|
|
|||
|
|
@ -227,6 +227,57 @@ async fn managed_aws_credentials_sign_the_exact_native_request(
|
|||
assert_eq!(request.json()["model"], "claude-test");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn bedrock_invoke_keeps_the_region_path_and_signs_caller_headers_as_python_does(
|
||||
call: MessagesCall,
|
||||
) {
|
||||
let upstream = upstream([message_response()]).await;
|
||||
let secrets = Arc::new(RecordingSecrets::new([("AWS_REGION_NAME", "us-east-1")]));
|
||||
run_with(
|
||||
secrets,
|
||||
MessagesCall {
|
||||
api_base: Some(upstream.uri()),
|
||||
litellm_params: litellm_router_types::LitellmParams {
|
||||
aws: litellm_auth::AwsParams {
|
||||
aws_access_key_id: Some("test-access".into()),
|
||||
aws_secret_access_key: Some("test-secret".into()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
extra_headers: Some(serde_json::Map::from_iter([
|
||||
("x-amz-date".to_string(), json!("20200101T000000Z")),
|
||||
("x-amz-security-token".to_string(), json!("caller-token")),
|
||||
("date".to_string(), json!("Wed, 01 Jan 2020 00:00:00 GMT")),
|
||||
])),
|
||||
..with_model(
|
||||
call,
|
||||
"bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let request = only_request(&upstream).await;
|
||||
let auth = request.header("authorization").unwrap();
|
||||
assert_eq!(
|
||||
request.url.path(),
|
||||
"/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke"
|
||||
);
|
||||
assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=test-access/"));
|
||||
assert!(auth.contains(
|
||||
"/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;date;host;x-amz-security-token, "
|
||||
));
|
||||
assert_eq!(request.header("x-amz-security-token"), Some("caller-token"));
|
||||
assert_eq!(request.header("x-amz-date"), None);
|
||||
assert!(
|
||||
request
|
||||
.header("date")
|
||||
.is_some_and(|date| date.ends_with(" -0000") && !date.contains("2020"))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn private_mantle_endpoint_is_used_and_signed(call: MessagesCall) {
|
||||
|
|
|
|||
|
|
@ -920,13 +920,12 @@ impl litellm_tracing::Sink for Warnings {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::anthropic_logs_as_main("anthropic", false)]
|
||||
#[case::azure_ai_logs_python_s_warning("azure_ai", true)]
|
||||
#[case::anthropic("anthropic")]
|
||||
#[case::azure_ai("azure_ai")]
|
||||
#[tokio::test]
|
||||
async fn a_speed_dropped_under_drop_params_is_logged_only_on_azure_ai(
|
||||
async fn a_speed_dropped_under_drop_params_logs_python_s_warning(
|
||||
call: MessagesCall,
|
||||
#[case] provider: &str,
|
||||
#[case] logged: bool,
|
||||
) {
|
||||
let upstream = upstream([message_response()]).await;
|
||||
let warnings = Warnings::default();
|
||||
|
|
@ -956,11 +955,7 @@ async fn a_speed_dropped_under_drop_params_is_logged_only_on_azure_ai(
|
|||
.filter(|message| message.starts_with("Dropping unsupported `speed`"))
|
||||
.cloned()
|
||||
.collect();
|
||||
assert_eq!(
|
||||
speed_warnings.len(),
|
||||
usize::from(logged),
|
||||
"{speed_warnings:?}"
|
||||
);
|
||||
assert_eq!(speed_warnings.len(), 1, "{speed_warnings:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -483,7 +483,7 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes
|
|||
#[case::mistyped_param(json!({"model": MODEL, "messages": "hello", "max_tokens": 16}))]
|
||||
#[case::missing_messages(json!({"model": MODEL, "max_tokens": 16}))]
|
||||
fn a_body_that_does_not_parse_is_an_invalid_request(#[case] raw: Value) {
|
||||
let error = messages_body(object(raw), None).expect_err("the body is rejected");
|
||||
let error = messages_body(object(raw)).expect_err("the body is rejected");
|
||||
|
||||
assert!(error.is_request(), "{error:?}");
|
||||
assert!(
|
||||
|
|
@ -502,38 +502,27 @@ const UNPARSED_SYSTEMS: [&str; 4] = [
|
|||
];
|
||||
|
||||
#[rstest]
|
||||
fn a_system_prompt_that_does_not_parse_is_main_s_request_error_except_on_azure_ai(
|
||||
fn a_system_prompt_that_does_not_parse_is_kept_as_python_keeps_it(
|
||||
#[values(0, 1, 2, 3)] system: usize,
|
||||
#[values(
|
||||
("anthropic/claude-sonnet-4-5", None),
|
||||
("claude-sonnet-4-5", Some("anthropic")),
|
||||
("vertex_ai/claude-sonnet-4-5", None),
|
||||
("azure_ai/claude-sonnet-4-5", None),
|
||||
("claude-sonnet-4-5", Some("azure_ai"))
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
"vertex_ai/claude-sonnet-4-5",
|
||||
"azure_ai/claude-sonnet-4-5"
|
||||
)]
|
||||
call: (&str, Option<&str>),
|
||||
model: &str,
|
||||
) {
|
||||
let (model, custom_llm_provider) = call;
|
||||
let system: Value = serde_json::from_str(UNPARSED_SYSTEMS[system]).unwrap();
|
||||
let body = object(json!({"model": model, "messages": [], "max_tokens": 16, "system": system}));
|
||||
let parsed = messages_body(body, custom_llm_provider);
|
||||
let azure_ai = model.starts_with("azure_ai/") || custom_llm_provider == Some("azure_ai");
|
||||
match parsed {
|
||||
match messages_body(body) {
|
||||
Err(error) => {
|
||||
assert!(
|
||||
!azure_ai || system.is_number() || system.is_boolean(),
|
||||
"{error:?}"
|
||||
);
|
||||
assert!(system.is_number() || system.is_boolean(), "{error:?}");
|
||||
let text = litellm_error::Report::new(&error).to_string();
|
||||
let expected = if azure_ai {
|
||||
"system must be a string or a list of content blocks"
|
||||
} else {
|
||||
"invalid Anthropic messages request: data did not match any variant of untagged enum SystemPrompt"
|
||||
};
|
||||
assert!(text.contains(expected), "{text}");
|
||||
assert!(
|
||||
text.contains("system must be a string or a list of content blocks"),
|
||||
"{text}"
|
||||
);
|
||||
}
|
||||
Ok(request) => {
|
||||
assert!(azure_ai, "{request:?}");
|
||||
assert!(
|
||||
request
|
||||
.params
|
||||
|
|
|
|||
|
|
@ -215,11 +215,6 @@ impl Error {
|
|||
|
||||
impl From<litellm_http::Error> for Error {
|
||||
fn from(error: litellm_http::Error) -> Self {
|
||||
// Python drops the caller's copy and prefers a forwarded Authorization
|
||||
// over the signature, so leave the request to it.
|
||||
if error.is_computed_header() {
|
||||
return Self::Declined(Decline::new("request forwards a header AWS SigV4 computes"));
|
||||
}
|
||||
Self::Http(error)
|
||||
}
|
||||
}
|
||||
|
|
@ -330,16 +325,6 @@ mod tests {
|
|||
assert!(error.is_missing_credential());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::direct(litellm_http::Error::computed_header("authorization").into())]
|
||||
#[case::through_a_transformation(
|
||||
LlmError::Http(litellm_http::Error::computed_header("authorization")).into()
|
||||
)]
|
||||
fn a_header_the_signer_computes_is_left_to_the_host_on_every_route(#[case] error: Error) {
|
||||
assert!(error.declined().is_some(), "{error:?}");
|
||||
assert!(!error.dispatched());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_transformation_decline_survives_into_the_route_error() {
|
||||
let decline = Decline::new("unrecognized request parameter");
|
||||
|
|
|
|||
|
|
@ -137,6 +137,8 @@ pub enum MessagesStreamEvent {
|
|||
MessageStop {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
usage: Option<MessagesStreamUsage>,
|
||||
#[serde(flatten)]
|
||||
extra: Map<String, Value>,
|
||||
},
|
||||
Ping,
|
||||
Error {
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ litellm-model-catalog.workspace = true
|
|||
litellm-tracing.workspace = true
|
||||
base64.workspace = true
|
||||
bytes.workspace = true
|
||||
crc32fast = "1.5"
|
||||
data-url = "0.3.2"
|
||||
fancy-regex.workspace = true
|
||||
futures-util.workspace = true
|
||||
|
|
@ -39,6 +40,7 @@ strum.workspace = true
|
|||
thiserror.workspace = true
|
||||
time.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio-util = { version = "0.7", features = ["codec"] }
|
||||
url.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -1,8 +1,13 @@
|
|||
use std::collections::BTreeSet;
|
||||
|
||||
use std::fmt::Display;
|
||||
|
||||
use litellm_llms_types::{
|
||||
formats::messages::{ContentBlock, ContentBlockType, Message, MessageContent},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet},
|
||||
formats::messages::{
|
||||
ContentBlock, ContentBlockType, Message, MessageContent, MessagesOptionalParams,
|
||||
MessagesRequest, ThinkingConfig,
|
||||
},
|
||||
providers::anthropic::{AnthropicBeta, BetaSet},
|
||||
recognized::Recognized,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
|
@ -16,15 +21,20 @@ const CONTENT: &str = "content";
|
|||
const TYPE: &str = "type";
|
||||
const OUTPUT_CONFIG: &str = "output_config";
|
||||
const TOOL_CHANGE_BLOCK_TYPES: [&str; 2] = ["tool_addition", "tool_removal"];
|
||||
const BLOCK_BINDING: &str = "block_binding";
|
||||
const DISPLAY: &str = "display";
|
||||
const DISPLAY_UPDATES: &str = "updates";
|
||||
|
||||
/// The mid-conversation system fields of Python's `BETA_GATED_FIELDS`, named as Python's warning
|
||||
/// names them and declared in the order it sorts those names.
|
||||
/// The fields of Python's `BETA_GATED_FIELDS`, named as Python's warning names them and declared
|
||||
/// in the order it sorts those names.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
enum GatedField {
|
||||
Emptied,
|
||||
ClearAt,
|
||||
ToolChanges,
|
||||
OutputConfig,
|
||||
BlockBinding,
|
||||
DisplayUpdates,
|
||||
}
|
||||
|
||||
impl GatedField {
|
||||
|
|
@ -34,6 +44,8 @@ impl GatedField {
|
|||
Self::ClearAt => "messages[].system.clear_at",
|
||||
Self::ToolChanges => "messages[].system.content[].tool_addition|tool_removal",
|
||||
Self::OutputConfig => "messages[].system.output_config",
|
||||
Self::BlockBinding => "thinking.block_binding",
|
||||
Self::DisplayUpdates => "thinking.display=updates",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -49,6 +61,8 @@ impl GatedField {
|
|||
AnthropicBeta::MidConversationOutputConfig20260701,
|
||||
AnthropicBeta::PerTurnControl20260701,
|
||||
],
|
||||
Self::BlockBinding => &[AnthropicBeta::ThinkingBindingControls20260801],
|
||||
Self::DisplayUpdates => &[AnthropicBeta::ThinkingDisplayUpdates20260818],
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -82,11 +96,98 @@ pub fn strip_beta_gated_system_messages(
|
|||
active: &BetaSet,
|
||||
drops_unsupported_params: bool,
|
||||
model: &str,
|
||||
provider: BetaProvider,
|
||||
provider: impl Display,
|
||||
) -> Vec<Recognized<Message>> {
|
||||
if !drops_unsupported_params {
|
||||
return messages;
|
||||
}
|
||||
let (messages, gated) = stripped_system_messages(messages, active);
|
||||
log_beta_gated_strips(&gated, model, provider);
|
||||
messages
|
||||
}
|
||||
|
||||
/// Python's `strip_beta_gated_native_request` and `log_beta_gated_strips`, run once the final
|
||||
/// `anthropic-beta` values are known: under drop_params the `thinking` fields and the system
|
||||
/// message fields no beta in `active` unlocks are dropped, and one warning names them all.
|
||||
pub fn strip_beta_gated_request(
|
||||
request: MessagesRequest,
|
||||
active: &BetaSet,
|
||||
drops_unsupported_params: bool,
|
||||
provider: impl Display,
|
||||
) -> MessagesRequest {
|
||||
if !drops_unsupported_params {
|
||||
return request;
|
||||
}
|
||||
let (thinking, thinking_gated) = stripped_thinking(request.params.thinking, active);
|
||||
let (messages, messages_gated) = stripped_system_messages(request.messages, active);
|
||||
log_beta_gated_strips(
|
||||
&thinking_gated.union(&messages_gated).copied().collect(),
|
||||
&request.model,
|
||||
provider,
|
||||
);
|
||||
MessagesRequest {
|
||||
messages,
|
||||
params: MessagesOptionalParams {
|
||||
thinking,
|
||||
..request.params
|
||||
},
|
||||
..request
|
||||
}
|
||||
}
|
||||
|
||||
fn log_beta_gated_strips(gated: &BTreeSet<GatedField>, model: &str, provider: impl Display) {
|
||||
if gated.is_empty() {
|
||||
return;
|
||||
}
|
||||
litellm_tracing::warn!(
|
||||
"Dropped {} for model={model} provider={provider} because drop_params is on and the provider would reject them on this request, either since this provider leg does not send that anthropic-beta (see anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user messages.",
|
||||
gated
|
||||
.iter()
|
||||
.copied()
|
||||
.map(GatedField::described)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
/// Python's `strip_beta_gated_thinking`, which reads `thinking` as any mapping.
|
||||
fn stripped_thinking(
|
||||
thinking: Option<Recognized<ThinkingConfig>>,
|
||||
active: &BetaSet,
|
||||
) -> (Option<Recognized<ThinkingConfig>>, BTreeSet<GatedField>) {
|
||||
let Some(Value::Object(fields)) = thinking.as_ref().and_then(|t| serde_json::to_value(t).ok())
|
||||
else {
|
||||
return (thinking, BTreeSet::new());
|
||||
};
|
||||
let binding_gated =
|
||||
fields.contains_key(BLOCK_BINDING) && GatedField::BlockBinding.gated(active);
|
||||
let display_gated = fields.get(DISPLAY).and_then(Value::as_str) == Some(DISPLAY_UPDATES)
|
||||
&& GatedField::DisplayUpdates.gated(active);
|
||||
if !binding_gated && !display_gated {
|
||||
return (thinking, BTreeSet::new());
|
||||
}
|
||||
let kept: Map<String, Value> = fields
|
||||
.into_iter()
|
||||
.filter(|(key, _)| {
|
||||
!((binding_gated && key == BLOCK_BINDING) || (display_gated && key == DISPLAY))
|
||||
})
|
||||
.collect();
|
||||
let gated = [
|
||||
(binding_gated, GatedField::BlockBinding),
|
||||
(display_gated, GatedField::DisplayUpdates),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(gated, field)| gated.then_some(field))
|
||||
.collect();
|
||||
let thinking = serde_json::from_value(Value::Object(kept.clone()))
|
||||
.unwrap_or(Recognized::Unrecognized(Value::Object(kept)));
|
||||
(Some(thinking), gated)
|
||||
}
|
||||
|
||||
fn stripped_system_messages(
|
||||
messages: Vec<Recognized<Message>>,
|
||||
active: &BetaSet,
|
||||
) -> (Vec<Recognized<Message>>, BTreeSet<GatedField>) {
|
||||
let stripped: Vec<(Option<Recognized<Message>>, Vec<GatedField>)> = messages
|
||||
.iter()
|
||||
.cloned()
|
||||
|
|
@ -112,24 +213,16 @@ pub fn strip_beta_gated_system_messages(
|
|||
.flat_map(|(_, gated)| gated.iter().copied())
|
||||
.collect();
|
||||
if gated.is_empty() {
|
||||
return messages;
|
||||
return (messages, gated);
|
||||
}
|
||||
let kept: Vec<Recognized<Message>> = stripped
|
||||
.into_iter()
|
||||
.filter_map(|(message, _)| message)
|
||||
.collect();
|
||||
if ends_on_assistant(&kept) && !ends_on_assistant(&messages) {
|
||||
return messages;
|
||||
return (messages, BTreeSet::new());
|
||||
}
|
||||
litellm_tracing::warn!(
|
||||
"Dropped {} for model={model} provider={provider} because drop_params is on and the provider would reject them on this request, either since this provider leg does not send that anthropic-beta (see anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user messages.",
|
||||
gated
|
||||
.into_iter()
|
||||
.map(GatedField::described)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
);
|
||||
kept
|
||||
(kept, gated)
|
||||
}
|
||||
|
||||
fn strip_system_message(message: Message, active: &BetaSet) -> (Option<Message>, Vec<GatedField>) {
|
||||
|
|
@ -263,14 +356,14 @@ fn ends_on_assistant(messages: &[Recognized<Message>]) -> bool {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_llms_types::{
|
||||
formats::messages::Message,
|
||||
formats::messages::{Message, MessagesRequest},
|
||||
providers::anthropic::{BetaProvider, BetaSet},
|
||||
recognized::Recognized,
|
||||
};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::strip_beta_gated_system_messages;
|
||||
use super::{strip_beta_gated_request, strip_beta_gated_system_messages};
|
||||
|
||||
fn strip(
|
||||
messages: Vec<Recognized<Message>>,
|
||||
|
|
@ -445,6 +538,99 @@ mod tests {
|
|||
assert_eq!(strip(sent.clone(), &BetaSet::default(), true), sent);
|
||||
}
|
||||
|
||||
fn thinking_request(thinking: Value, system: Value) -> MessagesRequest {
|
||||
serde_json::from_value(json!({
|
||||
"model": "test-model",
|
||||
"max_tokens": 16,
|
||||
"messages": conversation(system),
|
||||
"thinking": thinking,
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::block_binding_without_its_beta(
|
||||
json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}),
|
||||
"",
|
||||
json!({"type": "adaptive"})
|
||||
)]
|
||||
#[case::block_binding_with_its_beta(
|
||||
json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}),
|
||||
"thinking-binding-controls-2026-08-01",
|
||||
json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}})
|
||||
)]
|
||||
#[case::display_updates_without_its_beta(
|
||||
json!({"type": "adaptive", "display": "updates"}),
|
||||
"",
|
||||
json!({"type": "adaptive"})
|
||||
)]
|
||||
#[case::display_updates_with_its_beta(
|
||||
json!({"type": "adaptive", "display": "updates"}),
|
||||
"thinking-display-updates-2026-08-18",
|
||||
json!({"type": "adaptive", "display": "updates"})
|
||||
)]
|
||||
#[case::another_display(
|
||||
json!({"type": "adaptive", "display": "summarized"}),
|
||||
"",
|
||||
json!({"type": "adaptive", "display": "summarized"})
|
||||
)]
|
||||
#[case::both_on_enabled_thinking(
|
||||
json!({"type": "enabled", "budget_tokens": 1024, "display": "updates", "block_binding": {}}),
|
||||
"",
|
||||
json!({"type": "enabled", "budget_tokens": 1024})
|
||||
)]
|
||||
fn gated_thinking_fields_are_dropped(
|
||||
#[case] thinking: Value,
|
||||
#[case] active: &str,
|
||||
#[case] expected: Value,
|
||||
) {
|
||||
let stripped = strip_beta_gated_request(
|
||||
thinking_request(thinking, json!({"role": "user", "content": "more"})),
|
||||
&betas(active),
|
||||
true,
|
||||
"test",
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(stripped.params.thinking).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_request_strip_drops_gated_thinking_and_system_fields_together() {
|
||||
let stripped = strip_beta_gated_request(
|
||||
thinking_request(
|
||||
json!({"type": "adaptive", "display": "updates"}),
|
||||
json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}),
|
||||
),
|
||||
&BetaSet::default(),
|
||||
true,
|
||||
"test",
|
||||
);
|
||||
assert_eq!(
|
||||
(
|
||||
serde_json::to_value(stripped.params.thinking).unwrap(),
|
||||
stripped.messages
|
||||
),
|
||||
(
|
||||
json!({"type": "adaptive"}),
|
||||
messages(conversation(json!({"role": "system", "content": "note"})))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn without_drop_params_the_request_is_sent_as_is() {
|
||||
let sent = thinking_request(
|
||||
json!({"type": "adaptive", "display": "updates", "block_binding": {}}),
|
||||
json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}),
|
||||
);
|
||||
assert_eq!(
|
||||
strip_beta_gated_request(sent.clone(), &BetaSet::default(), false, "test"),
|
||||
sent
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn non_system_messages_keep_their_fields() {
|
||||
let sent = messages(json!([
|
||||
|
|
|
|||
|
|
@ -16,20 +16,38 @@ pub enum BetaPolicy {
|
|||
|
||||
impl BetaPolicy {
|
||||
pub fn apply(self, headers: Headers) -> Headers {
|
||||
self.apply_keeping(headers, &BetaSet::default())
|
||||
}
|
||||
|
||||
/// Python's `update_headers_with_filtered_beta` with `keep`: a beta the routed model
|
||||
/// accepts is sent as the caller sent it, after the betas the filter kept.
|
||||
pub fn apply_keeping(self, headers: Headers, keep: &BetaSet) -> Headers {
|
||||
let provider = match self {
|
||||
Self::Forward => return headers,
|
||||
Self::Drop => return without_headers(headers, &[BETA_HEADER]),
|
||||
Self::Filter(provider) => provider,
|
||||
};
|
||||
let accepted: BetaSet = existing_betas(&headers)
|
||||
let sent = existing_betas(&headers);
|
||||
let accepted: BetaSet = sent
|
||||
.iter()
|
||||
.filter(|beta| !keep.contains(beta))
|
||||
.filter_map(|beta| beta.on(provider))
|
||||
.collect();
|
||||
let kept: Vec<&AnthropicBeta> = sent
|
||||
.iter()
|
||||
.filter(|beta| keep.contains(beta) && !accepted.contains(beta))
|
||||
.collect();
|
||||
let headers = without_headers(headers, &[BETA_HEADER]);
|
||||
if accepted.is_empty() {
|
||||
if accepted.is_empty() && kept.is_empty() {
|
||||
return headers;
|
||||
}
|
||||
with_header(headers, BETA_HEADER, accepted.to_string())
|
||||
let value = accepted
|
||||
.iter()
|
||||
.chain(kept)
|
||||
.map(AnthropicBeta::as_str)
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
with_header(headers, BETA_HEADER, value)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -46,7 +64,7 @@ pub fn accepted_or_unlisted(betas: BetaSet, provider: BetaProvider) -> BetaSet {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::anthropic::beta_headers::BetaPolicy;
|
||||
use litellm_llms_types::providers::anthropic::BetaProvider;
|
||||
use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet};
|
||||
use rstest::rstest;
|
||||
|
||||
fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
|
||||
|
|
@ -125,6 +143,37 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::kept_after_the_filtered_betas(
|
||||
"per-turn-control-2026-07-01,advanced-tool-use-2025-11-20,example-beta-2099-01-01",
|
||||
Some("tool-search-tool-2025-10-19,per-turn-control-2026-07-01")
|
||||
)]
|
||||
#[case::kept_alone(
|
||||
"per-turn-control-2026-07-01,example-beta-2099-01-01",
|
||||
Some("per-turn-control-2026-07-01")
|
||||
)]
|
||||
#[case::not_sent("advanced-tool-use-2025-11-20", Some("tool-search-tool-2025-10-19"))]
|
||||
#[case::nothing_survives("example-beta-2099-01-01", None)]
|
||||
fn filter_sends_a_kept_beta_as_sent_after_the_filtered_ones(
|
||||
#[case] sent: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let keep: BetaSet = "per-turn-control-2026-07-01"
|
||||
.parse()
|
||||
.unwrap_or_else(|never| match never {});
|
||||
assert_eq!(
|
||||
BetaPolicy::Filter(BetaProvider::Bedrock).apply_keeping(
|
||||
headers(&[("x-api-key", "k"), ("anthropic-beta", sent)]),
|
||||
&keep
|
||||
),
|
||||
[("x-api-key", "k")]
|
||||
.into_iter()
|
||||
.chain(expected.map(|beta| ("anthropic-beta", beta)))
|
||||
.map(|(name, value)| (name.to_string(), value.to_string()))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::anthropic(
|
||||
BetaProvider::Anthropic,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use crate::base_llm::messages::context::{MessagesModelCapabilities, UnparsedMessages};
|
||||
use crate::base_llm::messages::context::MessagesModelCapabilities;
|
||||
use fancy_regex::Regex;
|
||||
use litellm_auth::CredentialPlacement;
|
||||
use litellm_secret_value::SecretValue;
|
||||
|
|
@ -322,26 +322,19 @@ pub fn has_advisor_tool(tools: Option<&[Recognized<MessagesTool>]>) -> bool {
|
|||
pub fn requires_native_compaction_beta(
|
||||
compaction: Option<&Value>,
|
||||
messages: &[Recognized<Message>],
|
||||
unparsed: UnparsedMessages,
|
||||
) -> bool {
|
||||
compaction.is_some()
|
||||
|| messages
|
||||
.iter()
|
||||
.any(|message| carries_signed_compaction_block(message, unparsed))
|
||||
compaction.is_some() || messages.iter().any(carries_signed_compaction_block)
|
||||
}
|
||||
|
||||
/// Python's `carries_signed_compaction_block` reads any mapping, so where unparsed messages are
|
||||
/// read a message that did not parse is scanned through its raw content too.
|
||||
fn carries_signed_compaction_block(
|
||||
message: &Recognized<Message>,
|
||||
unparsed: UnparsedMessages,
|
||||
) -> bool {
|
||||
/// Python's `carries_signed_compaction_block` reads any mapping, so a message that did not parse
|
||||
/// is scanned through its raw content too.
|
||||
fn carries_signed_compaction_block(message: &Recognized<Message>) -> bool {
|
||||
match message {
|
||||
Recognized::Known(message) => message.blocks().iter().any(|block| {
|
||||
block.is_type(ContentBlockType::Compaction)
|
||||
&& block.signature.as_deref().is_some_and(|s| !s.is_empty())
|
||||
}),
|
||||
Recognized::Unrecognized(Value::Object(fields)) if unparsed.read() => matches!(
|
||||
Recognized::Unrecognized(Value::Object(fields)) => matches!(
|
||||
fields.get("content"),
|
||||
Some(Value::Array(blocks)) if blocks.iter().any(|block| {
|
||||
block.get("type").and_then(Value::as_str) == Some("compaction")
|
||||
|
|
@ -2773,33 +2766,19 @@ mod tests {
|
|||
#[case] messages: Value,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
for unparsed in [UnparsedMessages::Read, UnparsedMessages::Skipped] {
|
||||
assert_eq!(
|
||||
requires_native_compaction_beta(
|
||||
compaction.as_ref(),
|
||||
&history(messages.clone()),
|
||||
unparsed
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
requires_native_compaction_beta(compaction.as_ref(), &history(messages)),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::read(UnparsedMessages::Read, true)]
|
||||
#[case::skipped(UnparsedMessages::Skipped, false)]
|
||||
fn a_signed_compaction_block_in_an_unparsed_message_counts_only_where_unparsed_messages_are_read(
|
||||
#[case] unparsed: UnparsedMessages,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
#[test]
|
||||
fn a_signed_compaction_block_in_an_unparsed_message_counts() {
|
||||
let messages = history(json!([{"role": "assistant", "content": [
|
||||
{"type": "text", "text": 5},
|
||||
{"type": "compaction", "content": "c", "signature": "s"}
|
||||
]}]));
|
||||
assert_eq!(
|
||||
requires_native_compaction_beta(None, &messages, unparsed),
|
||||
expected
|
||||
);
|
||||
assert!(requires_native_compaction_beta(None, &messages));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -2975,6 +2954,7 @@ mod tests {
|
|||
supports_speed: false,
|
||||
supports_mid_conversation_system: false,
|
||||
effort_tiers: tiers(false, false, false, false, false, false),
|
||||
bedrock: Default::default(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use litellm_llms_types::{
|
|||
AdaptiveThinking, EnabledThinking, Message, MessagesOptionalParams, MessagesRequest,
|
||||
ThinkingConfig, ThinkingDisplay,
|
||||
},
|
||||
recognized::{Recognized, map_known_runs},
|
||||
recognized::Recognized,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
|
|
@ -15,30 +15,25 @@ use crate::{
|
|||
strip_empty_raw_content_blocks, strip_provider_specific_fields,
|
||||
strip_raw_provider_specific_fields,
|
||||
},
|
||||
base_llm::messages::context::{MessagesShapeOptions, UnparsedMessages},
|
||||
base_llm::messages::context::MessagesShapeOptions,
|
||||
};
|
||||
|
||||
pub fn shape_anthropic_messages_request(
|
||||
request: MessagesRequest,
|
||||
options: MessagesShapeOptions,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
let messages = match (options.presanitized, options.unparsed) {
|
||||
(true, UnparsedMessages::Read) => map_messages(
|
||||
let messages = if options.presanitized {
|
||||
map_messages(
|
||||
request.messages,
|
||||
strip_provider_specific_fields,
|
||||
strip_raw_provider_specific_fields,
|
||||
),
|
||||
(false, UnparsedMessages::Read) => map_messages(
|
||||
)
|
||||
} else {
|
||||
map_messages(
|
||||
request.messages,
|
||||
sanitize_anthropic_messages,
|
||||
sanitize_raw_anthropic_message,
|
||||
),
|
||||
(true, UnparsedMessages::Skipped) => {
|
||||
map_known_runs(request.messages, strip_provider_specific_fields)
|
||||
}
|
||||
(false, UnparsedMessages::Skipped) => {
|
||||
map_known_runs(request.messages, sanitize_anthropic_messages)
|
||||
}
|
||||
)
|
||||
};
|
||||
Ok(MessagesRequest {
|
||||
messages,
|
||||
|
|
@ -334,7 +329,6 @@ mod tests {
|
|||
MessagesShapeOptions {
|
||||
reasoning_auto_summary: true,
|
||||
presanitized: false,
|
||||
unparsed: UnparsedMessages::Skipped,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
|
@ -355,7 +349,6 @@ mod tests {
|
|||
#[rstest]
|
||||
#[case::sanitized(
|
||||
false,
|
||||
UnparsedMessages::Read,
|
||||
json!([
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": [
|
||||
|
|
@ -365,7 +358,6 @@ mod tests {
|
|||
)]
|
||||
#[case::presanitized(
|
||||
true,
|
||||
UnparsedMessages::Read,
|
||||
json!([
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "user", "content": [{"type": "text", "text": 5}]},
|
||||
|
|
@ -375,33 +367,8 @@ mod tests {
|
|||
]}
|
||||
])
|
||||
)]
|
||||
#[case::skipped(
|
||||
false,
|
||||
UnparsedMessages::Skipped,
|
||||
json!([
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "user", "content": [{"type": "text", "text": 5}]},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "text", "text": 5},
|
||||
{"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}, "provider_specific_fields": {"x": 1}}
|
||||
]}
|
||||
])
|
||||
)]
|
||||
#[case::skipped_presanitized(
|
||||
true,
|
||||
UnparsedMessages::Skipped,
|
||||
json!([
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "user", "content": [{"type": "text", "text": 5}]},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "text", "text": 5},
|
||||
{"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}, "provider_specific_fields": {"x": 1}}
|
||||
]}
|
||||
])
|
||||
)]
|
||||
fn shaping_rewrites_a_message_that_did_not_parse_only_where_unparsed_messages_are_read(
|
||||
fn shaping_rewrites_a_message_that_did_not_parse(
|
||||
#[case] presanitized: bool,
|
||||
#[case] unparsed: UnparsedMessages,
|
||||
#[case] expected: Value,
|
||||
) {
|
||||
let shaped = shape_anthropic_messages_request(
|
||||
|
|
@ -416,7 +383,6 @@ mod tests {
|
|||
MessagesShapeOptions {
|
||||
reasoning_auto_summary: false,
|
||||
presanitized,
|
||||
unparsed,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
|
@ -434,7 +400,6 @@ mod tests {
|
|||
MessagesShapeOptions {
|
||||
reasoning_auto_summary: false,
|
||||
presanitized: true,
|
||||
unparsed: UnparsedMessages::Skipped,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use litellm_llms_types::{
|
|||
MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay,
|
||||
},
|
||||
providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet, DEFAULT_HEADERS},
|
||||
recognized::{Recognized, map_known_runs},
|
||||
recognized::Recognized,
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
|
@ -17,9 +17,11 @@ use super::{
|
|||
scalar::{is_one, python_str, python_value},
|
||||
thinking::{drop_unsupported_effort, translate_thinking},
|
||||
};
|
||||
use crate::anthropic::beta_headers::accepted_or_unlisted;
|
||||
use crate::anthropic::{
|
||||
beta_gated_fields::strip_beta_gated_request, beta_headers::accepted_or_unlisted,
|
||||
};
|
||||
use crate::base_llm::messages::{
|
||||
context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages},
|
||||
context::{MessagesShapeOptions, MessagesTransformContext},
|
||||
tools::ToolCustomField,
|
||||
};
|
||||
use crate::{
|
||||
|
|
@ -32,7 +34,7 @@ use crate::{
|
|||
is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error,
|
||||
is_claude_strict_tools_rejection, is_tool_search_used, map_messages, merge_beta_headers,
|
||||
minted_oauth_environment, optionally_handle_anthropic_oauth,
|
||||
requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks,
|
||||
requires_native_compaction_beta, strip_advisor_blocks,
|
||||
strip_encrypted_reasoning_blocks_restoring_redacted_data, strip_raw_advisor_blocks,
|
||||
strip_raw_encrypted_reasoning_blocks,
|
||||
strip_thinking_blocks_from_anthropic_messages_request, tools_without_strict,
|
||||
|
|
@ -132,6 +134,23 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
|
|||
)
|
||||
}
|
||||
|
||||
/// Python's transform strips the thinking and system fields the final header does not
|
||||
/// unlock.
|
||||
fn finalize_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
headers: Headers,
|
||||
context: &MessagesTransformContext,
|
||||
) -> Result<(MessagesRequest, Headers), Error> {
|
||||
let stripped = strip_beta_gated_request(
|
||||
request,
|
||||
&existing_betas(&headers),
|
||||
context.drops_unsupported_params,
|
||||
BetaProvider::Anthropic,
|
||||
);
|
||||
Ok((stripped, headers))
|
||||
}
|
||||
|
||||
fn retry_after_http_error(
|
||||
&self,
|
||||
status: u16,
|
||||
|
|
@ -301,19 +320,16 @@ pub(crate) fn transform_messages_request(
|
|||
.context_management
|
||||
.clone()
|
||||
.map(map_openai_context_management_to_anthropic);
|
||||
let reads_unparsed = context.unparsed.read();
|
||||
let messages = if has_advisor_tool(request.params.tools.as_deref())
|
||||
|| continues_message_thread(&request.params)
|
||||
{
|
||||
request.messages
|
||||
} else if reads_unparsed {
|
||||
} else {
|
||||
map_messages(
|
||||
request.messages,
|
||||
strip_advisor_blocks,
|
||||
strip_raw_advisor_blocks,
|
||||
)
|
||||
} else {
|
||||
map_known_runs(request.messages, strip_advisor_blocks)
|
||||
};
|
||||
let messages = if context.drops_unsupported_params {
|
||||
drop_non_string_tool_result_texts(messages)
|
||||
|
|
@ -321,15 +337,11 @@ pub(crate) fn transform_messages_request(
|
|||
messages
|
||||
};
|
||||
Ok(MessagesRequest {
|
||||
messages: if reads_unparsed {
|
||||
map_messages(
|
||||
messages,
|
||||
strip_encrypted_reasoning_blocks_restoring_redacted_data,
|
||||
strip_raw_encrypted_reasoning_blocks,
|
||||
)
|
||||
} else {
|
||||
map_known_runs(messages, strip_encrypted_reasoning_blocks)
|
||||
},
|
||||
messages: map_messages(
|
||||
messages,
|
||||
strip_encrypted_reasoning_blocks_restoring_redacted_data,
|
||||
strip_raw_encrypted_reasoning_blocks,
|
||||
),
|
||||
params: MessagesOptionalParams {
|
||||
context_management,
|
||||
extra: without_null_safeguards(request.params.extra),
|
||||
|
|
@ -428,21 +440,17 @@ pub(crate) fn update_headers_with_anthropic_beta(
|
|||
headers: Headers,
|
||||
request: &MessagesRequest,
|
||||
) -> Headers {
|
||||
merge_beta_headers(headers, feature_betas(request, UnparsedMessages::Skipped))
|
||||
merge_beta_headers(headers, feature_betas(request))
|
||||
}
|
||||
|
||||
/// `unparsed` says whether a beta trigger in a message that did not parse counts, as it does
|
||||
/// for Python, which reads any mapping.
|
||||
pub(crate) fn update_headers_with_safeguards_beta(
|
||||
headers: Headers,
|
||||
request: &MessagesRequest,
|
||||
provider: BetaProvider,
|
||||
unparsed: UnparsedMessages,
|
||||
) -> Headers {
|
||||
merge_beta_headers(
|
||||
headers,
|
||||
feature_betas(request, unparsed)
|
||||
.union(on_provider(mapped_host_betas(&request.params), provider)),
|
||||
feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -450,7 +458,7 @@ pub(crate) fn update_headers_with_safeguards_beta(
|
|||
/// through that column.
|
||||
pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet {
|
||||
on_provider(
|
||||
feature_betas(request, UnparsedMessages::Skipped).union(mapped_host_betas(&request.params)),
|
||||
feature_betas(request).union(mapped_host_betas(&request.params)),
|
||||
provider,
|
||||
)
|
||||
}
|
||||
|
|
@ -483,7 +491,7 @@ fn is_threaded(params: &MessagesOptionalParams) -> bool {
|
|||
.is_some_and(|thread| !thread.is_null())
|
||||
}
|
||||
|
||||
fn feature_betas(request: &MessagesRequest, unparsed: UnparsedMessages) -> BetaSet {
|
||||
fn feature_betas(request: &MessagesRequest) -> BetaSet {
|
||||
let params = &request.params;
|
||||
let tools = params.tools.as_deref();
|
||||
let scanned_messages: &[Recognized<Message>] = if continues_message_thread(params) {
|
||||
|
|
@ -492,19 +500,19 @@ fn feature_betas(request: &MessagesRequest, unparsed: UnparsedMessages) -> BetaS
|
|||
&request.messages
|
||||
};
|
||||
[
|
||||
requires_native_compaction_beta(params.compaction.as_ref(), scanned_messages, unparsed)
|
||||
requires_native_compaction_beta(params.compaction.as_ref(), scanned_messages)
|
||||
.then_some(AnthropicBeta::Compact20260904),
|
||||
uses_structured_output(params).then_some(AnthropicBeta::StructuredOutputs20251113),
|
||||
(params.speed == Some(Recognized::Known(Speed::Fast)))
|
||||
.then_some(AnthropicBeta::FastMode20260201),
|
||||
messages_carry_output_config(scanned_messages, unparsed)
|
||||
messages_carry_output_config(scanned_messages)
|
||||
.then_some(AnthropicBeta::PerTurnControl20260701),
|
||||
is_threaded(params).then_some(AnthropicBeta::MessageThreads20260812),
|
||||
uses_thinking_display_updates(params)
|
||||
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
|
||||
uses_thinking_block_binding(params)
|
||||
.then_some(AnthropicBeta::ThinkingBindingControls20260801),
|
||||
messages_carry_tool_changes(scanned_messages, unparsed)
|
||||
messages_carry_tool_changes(scanned_messages)
|
||||
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
|
||||
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
|
||||
is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120),
|
||||
|
|
@ -550,17 +558,12 @@ fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
|
|||
.is_some_and(|value| !value.is_null())
|
||||
}
|
||||
|
||||
/// Python's `messages_carry_output_config` reads any mapping, so where unparsed messages are read
|
||||
/// a message that did not parse, such as a system turn sent without content, counts too.
|
||||
fn messages_carry_output_config(
|
||||
messages: &[Recognized<Message>],
|
||||
unparsed: UnparsedMessages,
|
||||
) -> bool {
|
||||
/// Python's `messages_carry_output_config` reads any mapping, so a message that did not parse,
|
||||
/// such as a system turn sent without content, counts too.
|
||||
fn messages_carry_output_config(messages: &[Recognized<Message>]) -> bool {
|
||||
messages.iter().any(|message| match message {
|
||||
Recognized::Known(message) => message.extra.contains_key(OUTPUT_CONFIG),
|
||||
Recognized::Unrecognized(Value::Object(fields)) => {
|
||||
unparsed.read() && fields.contains_key(OUTPUT_CONFIG)
|
||||
}
|
||||
Recognized::Unrecognized(Value::Object(fields)) => fields.contains_key(OUTPUT_CONFIG),
|
||||
Recognized::Unrecognized(_) => false,
|
||||
})
|
||||
}
|
||||
|
|
@ -618,19 +621,15 @@ fn is_raw_tool_reference_change(block: &Value) -> bool {
|
|||
== Some("tool_reference")
|
||||
}
|
||||
|
||||
/// `AnthropicModelInfo.is_mid_conversation_tool_change_used` reads any mapping, so where unparsed
|
||||
/// messages are read a system message that did not parse is scanned through its raw content too.
|
||||
fn messages_carry_tool_changes(
|
||||
messages: &[Recognized<Message>],
|
||||
unparsed: UnparsedMessages,
|
||||
) -> bool {
|
||||
/// `AnthropicModelInfo.is_mid_conversation_tool_change_used` reads any mapping, so a system
|
||||
/// message that did not parse is scanned through its raw content too.
|
||||
fn messages_carry_tool_changes(messages: &[Recognized<Message>]) -> bool {
|
||||
messages.iter().any(|message| match message {
|
||||
Recognized::Known(message) => {
|
||||
message.role == "system" && message.blocks().iter().any(is_tool_reference_change)
|
||||
}
|
||||
Recognized::Unrecognized(Value::Object(fields)) => {
|
||||
unparsed.read()
|
||||
&& fields.get("role").and_then(Value::as_str) == Some("system")
|
||||
fields.get("role").and_then(Value::as_str) == Some("system")
|
||||
&& matches!(
|
||||
fields.get("content"),
|
||||
Some(Value::Array(blocks)) if blocks.iter().any(is_raw_tool_reference_change)
|
||||
|
|
@ -671,11 +670,9 @@ pub fn drop_unsupported_params(
|
|||
let speed = match ¶ms.speed {
|
||||
Some(speed) if !capabilities.supports_speed => {
|
||||
reject("speed", speed_repr(speed), "")?;
|
||||
if context.logs_dropped_params {
|
||||
litellm_tracing::warn!(
|
||||
"Dropping unsupported `speed` for model={model} (drop_params=True). Fast mode is only supported on select Opus models."
|
||||
);
|
||||
}
|
||||
litellm_tracing::warn!(
|
||||
"Dropping unsupported `speed` for model={model} (drop_params=True). Fast mode is only supported on select Opus models."
|
||||
);
|
||||
None
|
||||
}
|
||||
_ => params.speed.clone(),
|
||||
|
|
@ -1346,8 +1343,7 @@ mod tests {
|
|||
},
|
||||
drop_params: true,
|
||||
drops_unsupported_params: false,
|
||||
unparsed: UnparsedMessages::Skipped,
|
||||
logs_dropped_params: false,
|
||||
global_drop_params: false,
|
||||
}
|
||||
);
|
||||
return;
|
||||
|
|
@ -1736,10 +1732,7 @@ mod tests {
|
|||
&["message-threads-2026-08-12", "per-turn-control-2026-07-01"]
|
||||
)]
|
||||
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
|
||||
assert_eq!(
|
||||
feature_betas(&request(fields), UnparsedMessages::Read),
|
||||
betas(expected)
|
||||
);
|
||||
assert_eq!(feature_betas(&request(fields)), betas(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -1758,21 +1751,16 @@ mod tests {
|
|||
{"role": "user", "content": "Use the ping tool"},
|
||||
{"role": "system", "content": [{"type": "text", "text": 5}, {"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}))]
|
||||
fn a_beta_trigger_in_an_unparsed_message_counts_only_where_unparsed_messages_are_read(
|
||||
#[case] fields: Value,
|
||||
) {
|
||||
fn a_beta_trigger_in_an_unparsed_message_counts_on_every_provider(#[case] fields: Value) {
|
||||
let request = request(fields);
|
||||
assert_ne!(feature_betas(&request), BetaSet::default());
|
||||
assert_eq!(
|
||||
feature_betas(&request, UnparsedMessages::Skipped),
|
||||
BetaSet::default()
|
||||
existing_betas(&update_headers_with_anthropic_beta(Vec::new(), &request)),
|
||||
feature_betas(&request)
|
||||
);
|
||||
assert_eq!(
|
||||
provider_feature_betas(&request, BetaProvider::Anthropic),
|
||||
BetaSet::default()
|
||||
);
|
||||
assert_ne!(
|
||||
feature_betas(&request, UnparsedMessages::Read),
|
||||
BetaSet::default()
|
||||
feature_betas(&request)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,32 @@
|
|||
use std::sync::LazyLock;
|
||||
|
||||
use crate::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers};
|
||||
use crate::base_llm::messages::context::{
|
||||
BedrockCapabilities, MessagesModelCapabilities, NativeOutputSupport, SupportedEffortTiers,
|
||||
};
|
||||
use crate::bedrock::common_utils::{
|
||||
bedrock_base_model, is_bedrock_application_inference_profile_arn, without_routing_prefixes,
|
||||
bedrock_base_model, is_bedrock_application_inference_profile_arn, strip_bedrock_routing_prefix,
|
||||
};
|
||||
use fancy_regex::Regex;
|
||||
use litellm_auth_aws::is_bedrock_region;
|
||||
use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
|
||||
use litellm_llms_types::formats::messages::EffortLevel;
|
||||
use litellm_model_catalog::{Catalog, wire::Fields};
|
||||
use litellm_python_compat::{json::from_json, truthy::truthy};
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
const SAMPLING_RESTRICTED_NAMES: [&str; 9] = [
|
||||
"fable", "opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7", "opus-4-8", "opus_4_8", "opus-4.8",
|
||||
"opus_4.8",
|
||||
];
|
||||
const SAMPLING_TOLERANT_PROVIDERS: [&str; 1] = ["openrouter"];
|
||||
const BEDROCK: &str = "bedrock";
|
||||
const NATIVE_OUTPUT: &str = "supports_native_structured_output";
|
||||
const BEDROCK_NATIVE_OUTPUT: &str = "supports_bedrock_native_structured_output";
|
||||
const NATIVE_OUTPUT_BY_DEFAULT: &str = "supports_bedrock_native_structured_output_by_default";
|
||||
const OUTPUT_CONFIG: &str = "supports_output_config";
|
||||
const PER_TURN_CONTROL: &str = "supports_per_turn_control";
|
||||
const PROMPT_CACHE_TTL_PRICE: &str = "cache_creation_input_token_cost_above_1hr";
|
||||
const BACKSTOP_PREFIXES: [&str; 4] = [
|
||||
"bedrock/converse/",
|
||||
"bedrock/invoke/",
|
||||
|
|
@ -45,7 +58,8 @@ pub fn anthropic_model_capabilities(
|
|||
.map_or((model, "anthropic"), |resolved| {
|
||||
(resolved.model, resolved.custom_llm_provider)
|
||||
});
|
||||
let (model, name) = match base_model.map(without_routing_prefixes) {
|
||||
let requested = model;
|
||||
let (model, name) = match base_model.map(strip_bedrock_routing_prefix) {
|
||||
Some(base)
|
||||
if is_bedrock(provider) && is_bedrock_application_inference_profile_arn(resolved) =>
|
||||
{
|
||||
|
|
@ -53,18 +67,7 @@ pub fn anthropic_model_capabilities(
|
|||
}
|
||||
_ => (model, resolved),
|
||||
};
|
||||
let probe = Probe {
|
||||
catalog,
|
||||
declared,
|
||||
fallbacks: backstop_candidates(model),
|
||||
info: catalog.model_info(
|
||||
&candidate_names(name, provider)
|
||||
.each_ref()
|
||||
.map(String::as_str),
|
||||
provider,
|
||||
),
|
||||
bare: catalog.lookup(name).map(|found| found.entry.fields()),
|
||||
};
|
||||
let probe = Probe::new(catalog, model, name, provider, declared);
|
||||
let thinking_always_on = probe.supports("thinking_always_on");
|
||||
MessagesModelCapabilities {
|
||||
supports_reasoning: probe.supports("supports_reasoning"),
|
||||
|
|
@ -98,9 +101,73 @@ pub fn anthropic_model_capabilities(
|
|||
xhigh: probe.supports("supports_xhigh_reasoning_effort"),
|
||||
max: probe.supports("supports_max_reasoning_effort"),
|
||||
},
|
||||
bedrock: if provider == BEDROCK {
|
||||
bedrock_model_capabilities(catalog, requested, declared)
|
||||
} else {
|
||||
BedrockCapabilities::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bedrock_model_capabilities(
|
||||
catalog: &Catalog,
|
||||
model: &str,
|
||||
declared: &dyn Fn(&str) -> Option<bool>,
|
||||
) -> BedrockCapabilities {
|
||||
let model = bedrock_model(model);
|
||||
let probe = Probe::new(catalog, model, bedrock_model(model), BEDROCK, &|_| None);
|
||||
let mapped = probe.factory(NATIVE_OUTPUT);
|
||||
let opted_out = declared(BEDROCK_NATIVE_OUTPUT).map_or_else(
|
||||
|| mapped_native_output_flag(catalog, model) == Some(false),
|
||||
|opted_in| !opted_in,
|
||||
);
|
||||
BedrockCapabilities {
|
||||
output_config_effort_ceiling: probe
|
||||
.info
|
||||
.as_ref()
|
||||
.and_then(|info| info.get("bedrock_output_config_effort_ceiling"))
|
||||
.and_then(|ceiling| EffortLevel::deserialize(ceiling).ok()),
|
||||
supports_output_config: probe.factory(OUTPUT_CONFIG),
|
||||
supports_prompt_cache_ttl: [model, bedrock_base_model(model)].iter().any(|name| {
|
||||
catalog
|
||||
.get(name)
|
||||
.and_then(|fields| fields.get(PROMPT_CACHE_TTL_PRICE))
|
||||
.is_some_and(|price| !price.is_null())
|
||||
}),
|
||||
supports_tool_search: probe.supports("supports_tool_search"),
|
||||
supports_per_turn_control: declared(PER_TURN_CONTROL)
|
||||
.unwrap_or_else(|| probe.supports(PER_TURN_CONTROL)),
|
||||
native_output: NativeOutputSupport {
|
||||
enabled: declared(NATIVE_OUTPUT)
|
||||
.unwrap_or_else(|| declared(BEDROCK_NATIVE_OUTPUT) != Some(false) && mapped),
|
||||
opt_in: declared(BEDROCK_NATIVE_OUTPUT)
|
||||
.unwrap_or_else(|| probe.factory(BEDROCK_NATIVE_OUTPUT)),
|
||||
mapped,
|
||||
replaces_unforceable_tool: backstop(
|
||||
catalog,
|
||||
&probe.fallbacks,
|
||||
"supports_forced_tool_use",
|
||||
) == Some(false)
|
||||
&& !opted_out,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn bedrock_model(model: &str) -> &str {
|
||||
get_custom_llm_provider(model, Some(BEDROCK)).map_or(model, |found| found.model)
|
||||
}
|
||||
|
||||
fn mapped_native_output_flag(catalog: &Catalog, model: &str) -> Option<bool> {
|
||||
let routed = strip_bedrock_routing_prefix(model);
|
||||
[
|
||||
format!("{BEDROCK}/{routed}"),
|
||||
routed.to_owned(),
|
||||
without_region(routed).to_owned(),
|
||||
]
|
||||
.iter()
|
||||
.find_map(|name| flag(catalog.get(name), BEDROCK_NATIVE_OUTPUT))
|
||||
}
|
||||
|
||||
struct Probe<'a> {
|
||||
catalog: &'a Catalog,
|
||||
declared: &'a dyn Fn(&str) -> Option<bool>,
|
||||
|
|
@ -109,7 +176,28 @@ struct Probe<'a> {
|
|||
bare: Option<&'a Fields>,
|
||||
}
|
||||
|
||||
impl Probe<'_> {
|
||||
impl<'a> Probe<'a> {
|
||||
fn new(
|
||||
catalog: &'a Catalog,
|
||||
model: &str,
|
||||
name: &str,
|
||||
provider: &str,
|
||||
declared: &'a dyn Fn(&str) -> Option<bool>,
|
||||
) -> Self {
|
||||
Self {
|
||||
catalog,
|
||||
declared,
|
||||
fallbacks: backstop_candidates(model),
|
||||
info: catalog.model_info(
|
||||
&candidate_names(name, provider)
|
||||
.each_ref()
|
||||
.map(String::as_str),
|
||||
provider,
|
||||
),
|
||||
bare: catalog.lookup(name).map(|found| found.entry.fields()),
|
||||
}
|
||||
}
|
||||
|
||||
fn supports(&self, capability: &str) -> bool {
|
||||
if let Some(supported) = (self.declared)(capability) {
|
||||
return supported;
|
||||
|
|
@ -117,10 +205,35 @@ impl Probe<'_> {
|
|||
if let Some(supported) = flag(self.info.as_ref(), capability) {
|
||||
return supported;
|
||||
}
|
||||
if self.info.is_some() && flag(self.bare, capability) == Some(true) {
|
||||
return true;
|
||||
self.bare_supports(capability)
|
||||
|| backstop(self.catalog, &self.fallbacks, capability) == Some(true)
|
||||
}
|
||||
|
||||
fn factory(&self, capability: &str) -> bool {
|
||||
match self.info.as_ref() {
|
||||
Some(info) => {
|
||||
model_info_flag(info, capability).unwrap_or_else(|| self.bare_supports(capability))
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
backstop(self.catalog, &self.fallbacks, capability) == Some(true)
|
||||
}
|
||||
|
||||
fn bare_supports(&self, capability: &str) -> bool {
|
||||
self.info.is_some() && flag(self.bare, capability) == Some(true)
|
||||
}
|
||||
}
|
||||
|
||||
fn model_info_flag(info: &Fields, capability: &str) -> Option<bool> {
|
||||
match capability {
|
||||
NATIVE_OUTPUT
|
||||
if info
|
||||
.get(NATIVE_OUTPUT_BY_DEFAULT)
|
||||
.is_some_and(|value| truthy(&from_json(value.clone()))) =>
|
||||
{
|
||||
Some(info.get(BEDROCK_NATIVE_OUTPUT) != Some(&Value::Bool(false)))
|
||||
}
|
||||
OUTPUT_CONFIG => None,
|
||||
_ => flag(Some(info), capability),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -170,7 +283,7 @@ fn split_model<'a>(model: &'a str, provider: &str) -> &'a str {
|
|||
if !is_bedrock(provider) {
|
||||
return model;
|
||||
}
|
||||
let routed = without_routing_prefixes(model);
|
||||
let routed = strip_bedrock_routing_prefix(model);
|
||||
let region_free = without_region(routed);
|
||||
if !region_free.starts_with("arn:") {
|
||||
return routed;
|
||||
|
|
@ -239,7 +352,9 @@ mod tests {
|
|||
use rstest::{fixture, rstest};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::anthropic_model_capabilities;
|
||||
use super::{anthropic_model_capabilities, bedrock_model_capabilities};
|
||||
use crate::base_llm::messages::context::{BedrockCapabilities, NativeOutputSupport};
|
||||
use litellm_llms_types::formats::messages::EffortLevel;
|
||||
|
||||
const CATALOG: &str = r#"{
|
||||
"claude-test-5": {
|
||||
|
|
@ -460,6 +575,137 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn bundled() -> Catalog {
|
||||
Catalog::parse(BUNDLED_REGISTRY).unwrap().value
|
||||
}
|
||||
|
||||
type Expected = (Option<EffortLevel>, [bool; 8]);
|
||||
|
||||
fn projected(capabilities: BedrockCapabilities) -> Expected {
|
||||
let BedrockCapabilities {
|
||||
output_config_effort_ceiling,
|
||||
supports_output_config,
|
||||
supports_prompt_cache_ttl,
|
||||
supports_tool_search,
|
||||
supports_per_turn_control,
|
||||
native_output:
|
||||
NativeOutputSupport {
|
||||
enabled,
|
||||
opt_in,
|
||||
mapped,
|
||||
replaces_unforceable_tool,
|
||||
},
|
||||
} = capabilities;
|
||||
(
|
||||
output_config_effort_ceiling,
|
||||
[
|
||||
supports_output_config,
|
||||
supports_prompt_cache_ttl,
|
||||
supports_tool_search,
|
||||
supports_per_turn_control,
|
||||
enabled,
|
||||
opt_in,
|
||||
mapped,
|
||||
replaces_unforceable_tool,
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
const XHIGH: Option<EffortLevel> = Some(EffortLevel::Xhigh);
|
||||
const PROFILE: &str = "bedrock/arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-opus-4-8";
|
||||
const FOUNDATION: &str = "bedrock/invoke/arn:aws:bedrock:us-east-1:123456789012:foundation-model/anthropic.claude-opus-4-8";
|
||||
const APPLICATION: &str =
|
||||
"bedrock/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123";
|
||||
|
||||
#[rstest]
|
||||
#[case::us("bedrock/us.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))]
|
||||
#[case::eu("bedrock/eu.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))]
|
||||
#[case::global("bedrock/global.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))]
|
||||
#[case::invoke_prefix("bedrock/invoke/us.anthropic.claude-opus-4-8", (XHIGH, [false, true, true, true, true, true, true, false]))]
|
||||
#[case::region_path("bedrock/us-east-1/anthropic.claude-opus-4-8", (XHIGH, [false, true, true, true, false, true, false, false]))]
|
||||
#[case::inference_profile_arn(PROFILE, (XHIGH, [false, true, true, true, true, true, true, false]))]
|
||||
#[case::invoke_foundation_arn(FOUNDATION, (XHIGH, [false, true, true, true, false, true, false, false]))]
|
||||
#[case::unprefixed("anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, false, true, false, false]))]
|
||||
#[case::other_casing("bedrock/ANTHROPIC.CLAUDE-OPUS-4-8", (XHIGH, [true, false, true, true, false, true, false, false]))]
|
||||
#[case::sonnet_4_5("bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", (None, [false, true, true, true, true, false, true, false]))]
|
||||
#[case::converse_prefix("bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0", (None, [false, true, true, true, true, false, true, false]))]
|
||||
#[case::throughput_suffix("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0:1000k", (None, [false, true, true, true, true, false, true, false]))]
|
||||
#[case::claude_3_5("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", (None, [false; 8]))]
|
||||
#[case::opus_4_7_without_version("bedrock/global.anthropic.claude-opus-4-7", (XHIGH, [true, true, true, true, false, false, false, false]))]
|
||||
#[case::opus_4_7_invoke("bedrock/invoke/global.anthropic.claude-opus-4-7-v1:0", (None, [false, false, true, true, false, false, false, false]))]
|
||||
#[case::unforceable_tool("bedrock/anthropic.claude-fable-5-1", (XHIGH, [true, true, true, true, false, false, false, true]))]
|
||||
#[case::native_by_default("bedrock/us.anthropic.claude-mythos-5-2", (XHIGH, [true, true, true, true, true, false, true, false]))]
|
||||
#[case::opt_in_only("bedrock/anthropic.claude-sonnet-5", (XHIGH, [true, true, true, true, false, true, false, false]))]
|
||||
#[case::unmapped_claude_version("bedrock/anthropic.claude-opus-9-9-v1:0", (None, [false, false, true, true, false, false, false, false]))]
|
||||
#[case::unmapped_claude("bedrock/anthropic.claude-newthing", (None, [false; 8]))]
|
||||
#[case::application_profile(APPLICATION, (None, [false; 8]))]
|
||||
#[case::other_family("bedrock/amazon.nova-pro-v1:0", (None, [false; 8]))]
|
||||
#[case::unknown("bedrock/unknown-model", (None, [false; 8]))]
|
||||
fn bedrock_capabilities_match_python(
|
||||
bundled: Catalog,
|
||||
#[case] model: &str,
|
||||
#[case] expected: Expected,
|
||||
) {
|
||||
assert_eq!(
|
||||
projected(bedrock_model_capabilities(&bundled, model, &|_| None)),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::native_on("bedrock/us.anthropic.claude-opus-4-8", &[("supports_native_structured_output", true)], (XHIGH, [true, true, true, true, true, true, true, false]))]
|
||||
#[case::native_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_native_structured_output", false)], (XHIGH, [true, true, true, true, false, true, true, false]))]
|
||||
#[case::bedrock_native_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_bedrock_native_structured_output", false)], (XHIGH, [true, true, true, true, false, false, true, false]))]
|
||||
#[case::per_turn_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_per_turn_control", false)], (XHIGH, [true, true, true, false, true, true, true, false]))]
|
||||
#[case::native_off_opt_in(
|
||||
"bedrock/us.anthropic.claude-opus-4-8",
|
||||
&[("supports_native_structured_output", false), ("supports_bedrock_native_structured_output", true)],
|
||||
(XHIGH, [true, true, true, true, false, true, true, false])
|
||||
)]
|
||||
#[case::unforceable_native_on("bedrock/anthropic.claude-fable-5-1", &[("supports_native_structured_output", true)], (XHIGH, [true, true, true, true, true, false, false, true]))]
|
||||
#[case::unforceable_opt_in("bedrock/anthropic.claude-fable-5-1", &[("supports_bedrock_native_structured_output", true)], (XHIGH, [true, true, true, true, false, true, false, true]))]
|
||||
#[case::unforceable_opted_out("bedrock/anthropic.claude-fable-5-1", &[("supports_bedrock_native_structured_output", false)], (XHIGH, [true, true, true, true, false, false, false, false]))]
|
||||
#[case::bedrock_native_on("bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", &[("supports_bedrock_native_structured_output", true)], (None, [false, true, true, true, true, true, true, false]))]
|
||||
#[case::per_turn_off_haiku("bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", &[("supports_per_turn_control", false)], (None, [false, true, true, false, true, false, true, false]))]
|
||||
fn deployment_model_info_overrides_the_catalog_as_python_does(
|
||||
bundled: Catalog,
|
||||
#[case] model: &str,
|
||||
#[case] declared: &[(&str, bool)],
|
||||
#[case] expected: Expected,
|
||||
) {
|
||||
let lookup = |capability: &str| {
|
||||
declared
|
||||
.iter()
|
||||
.find(|(name, _)| *name == capability)
|
||||
.map(|(_, supported)| *supported)
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
projected(bedrock_model_capabilities(&bundled, model, &lookup)),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bedrock("bedrock/us.anthropic.claude-opus-4-8", None, true)]
|
||||
#[case::configured_bedrock("us.anthropic.claude-opus-4-8", Some("bedrock"), true)]
|
||||
#[case::anthropic("anthropic/claude-opus-4-8", None, false)]
|
||||
fn only_bedrock_deployments_carry_bedrock_capabilities(
|
||||
bundled: Catalog,
|
||||
#[case] model: &str,
|
||||
#[case] custom_llm_provider: Option<&str>,
|
||||
#[case] carried: bool,
|
||||
) {
|
||||
let capabilities =
|
||||
anthropic_model_capabilities(&bundled, model, custom_llm_provider, None, &|_| None);
|
||||
|
||||
assert_eq!(
|
||||
capabilities.bedrock != BedrockCapabilities::default(),
|
||||
carried
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn every_bundled_anthropic_row_shapes_with_its_own_flags() {
|
||||
let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value;
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ use serde_json::Value;
|
|||
use crate::{
|
||||
Error,
|
||||
anthropic::{
|
||||
beta_gated_fields::strip_beta_gated_system_messages,
|
||||
beta_gated_fields::strip_beta_gated_request,
|
||||
beta_headers::BetaPolicy,
|
||||
common_utils::{API_KEY_PLACEMENT, existing_betas, strip_trailing_slashes},
|
||||
messages::{
|
||||
|
|
@ -34,7 +34,7 @@ use crate::{
|
|||
base_llm::{
|
||||
auth::{AuthScheme, Headers, ValidatedEnvironment},
|
||||
messages::{
|
||||
context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages},
|
||||
context::{MessagesShapeOptions, MessagesTransformContext},
|
||||
normalization::{normalize_system_role_messages, strip_cache_control_scope},
|
||||
transformation::BaseMessagesConfig,
|
||||
},
|
||||
|
|
@ -59,10 +59,6 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
|
|||
}
|
||||
}
|
||||
|
||||
fn logs_dropped_params(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn retry_after_http_error(
|
||||
&self,
|
||||
status: u16,
|
||||
|
|
@ -78,7 +74,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
|
|||
request: MessagesRequest,
|
||||
options: MessagesShapeOptions,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
shape_anthropic_messages_request(request, options.reading_unparsed_messages())
|
||||
shape_anthropic_messages_request(request, options)
|
||||
}
|
||||
|
||||
fn get_complete_url(
|
||||
|
|
@ -97,12 +93,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
|
|||
request: MessagesRequest,
|
||||
context: &MessagesTransformContext,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
transform_messages_request(
|
||||
request,
|
||||
&context
|
||||
.keeping_display_updates()
|
||||
.reading_unparsed_messages(),
|
||||
)
|
||||
transform_messages_request(request, &context.keeping_display_updates())
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> &'static [&'static str] {
|
||||
|
|
@ -211,30 +202,25 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
|
|||
headers,
|
||||
request,
|
||||
BetaProvider::AzureAi,
|
||||
UnparsedMessages::Read,
|
||||
))
|
||||
}
|
||||
|
||||
/// Python's transform strips the system fields the final header does not unlock, then
|
||||
/// normalizes system roles and drops `cache_control.scope`, all after the betas were chosen
|
||||
/// from the request as sent.
|
||||
/// Python's transform strips the thinking and system fields the final header does not
|
||||
/// unlock, then normalizes system roles and drops `cache_control.scope`, all after the betas
|
||||
/// were chosen from the request as sent.
|
||||
fn finalize_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
headers: &Headers,
|
||||
headers: Headers,
|
||||
context: &MessagesTransformContext,
|
||||
) -> MessagesRequest {
|
||||
let stripped = MessagesRequest {
|
||||
messages: strip_beta_gated_system_messages(
|
||||
request.messages,
|
||||
&existing_betas(headers),
|
||||
context.drops_unsupported_params,
|
||||
&request.model,
|
||||
BetaProvider::AzureAi,
|
||||
),
|
||||
..request
|
||||
};
|
||||
strip_cache_control_scope(normalize_system_role_messages(
|
||||
) -> Result<(MessagesRequest, Headers), Error> {
|
||||
let stripped = strip_beta_gated_request(
|
||||
request,
|
||||
&existing_betas(&headers),
|
||||
context.drops_unsupported_params,
|
||||
BetaProvider::AzureAi,
|
||||
);
|
||||
let normalized = strip_cache_control_scope(normalize_system_role_messages(
|
||||
stripped,
|
||||
context
|
||||
.thinking
|
||||
|
|
@ -242,7 +228,8 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
|
|||
.supports_mid_conversation_system,
|
||||
context.drops_unsupported_params,
|
||||
BetaProvider::AzureAi,
|
||||
))
|
||||
));
|
||||
Ok((normalized, headers))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -430,7 +417,12 @@ mod tests {
|
|||
.iter()
|
||||
.map(|(name, value)| (name.to_string(), value.to_string()))
|
||||
.collect();
|
||||
to_value(AZURE_ANTHROPIC_MESSAGES_CONFIG.finalize_request(request, &headers, context))
|
||||
to_value(
|
||||
AZURE_ANTHROPIC_MESSAGES_CONFIG
|
||||
.finalize_request(request, headers, context)
|
||||
.expect("finalized request")
|
||||
.0,
|
||||
)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use litellm_auth::{AuthServices, CredentialPlacement, TokenProviderHandle};
|
|||
use litellm_auth_azure::AzureAuthInputs;
|
||||
use litellm_secret_value::SecretValue;
|
||||
|
||||
use litellm_auth_aws::{AwsCredentialSource, SigV4Signer};
|
||||
use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders, SigV4Signer};
|
||||
use litellm_auth_gcp::VertexConfig;
|
||||
use litellm_http::request::with_header;
|
||||
|
||||
|
|
@ -57,6 +57,7 @@ pub enum AuthScheme {
|
|||
region: String,
|
||||
service: &'static str,
|
||||
credentials: Box<AwsCredentialSource>,
|
||||
caller_headers: CallerSigningHeaders,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -191,11 +192,13 @@ pub async fn resolve_auth(
|
|||
region,
|
||||
service,
|
||||
credentials,
|
||||
caller_headers,
|
||||
} => Ok(Authenticated {
|
||||
headers,
|
||||
signer: Some(
|
||||
SigV4Signer::resolve(&services.aws, region, service, *credentials, env_lookup)
|
||||
.await?,
|
||||
.await?
|
||||
.with_caller_headers(caller_headers),
|
||||
),
|
||||
}),
|
||||
}
|
||||
|
|
@ -445,6 +448,7 @@ mod tests {
|
|||
None,
|
||||
"test",
|
||||
))),
|
||||
caller_headers: CallerSigningHeaders::Dropped,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
|
||||
use litellm_llms_types::formats::messages::EffortLevel;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
|
@ -23,39 +24,10 @@ impl SupportedEffortTiers {
|
|||
}
|
||||
}
|
||||
|
||||
/// What the shared Anthropic shaping does with a message, or a beta trigger in one, that Rust
|
||||
/// could not type. `Read` reads it as Python does, off any mapping. `Skipped` leaves it as sent
|
||||
/// and finds no beta in it, as main's native route does. Only `azure_ai` reads them: by the
|
||||
/// owner's ruling `anthropic` keeps main's behaviour.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum UnparsedMessages {
|
||||
#[default]
|
||||
Skipped,
|
||||
Read,
|
||||
}
|
||||
|
||||
impl UnparsedMessages {
|
||||
pub fn read(self) -> bool {
|
||||
self == Self::Read
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct MessagesShapeOptions {
|
||||
pub reasoning_auto_summary: bool,
|
||||
pub presanitized: bool,
|
||||
pub unparsed: UnparsedMessages,
|
||||
}
|
||||
|
||||
impl MessagesShapeOptions {
|
||||
/// The same options for a provider whose shaping reads unparsed messages.
|
||||
#[must_use]
|
||||
pub fn reading_unparsed_messages(self) -> Self {
|
||||
Self {
|
||||
unparsed: UnparsedMessages::Read,
|
||||
..self
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
|
@ -80,6 +52,46 @@ pub struct MessagesModelCapabilities {
|
|||
pub supports_mid_conversation_system: bool,
|
||||
#[serde(default)]
|
||||
pub effort_tiers: SupportedEffortTiers,
|
||||
#[serde(default)]
|
||||
pub bedrock: BedrockCapabilities,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct BedrockCapabilities {
|
||||
#[serde(default)]
|
||||
pub output_config_effort_ceiling: Option<EffortLevel>,
|
||||
#[serde(default)]
|
||||
pub supports_output_config: bool,
|
||||
#[serde(default)]
|
||||
pub supports_prompt_cache_ttl: bool,
|
||||
#[serde(default)]
|
||||
pub supports_tool_search: bool,
|
||||
#[serde(default)]
|
||||
pub supports_per_turn_control: bool,
|
||||
#[serde(default)]
|
||||
pub native_output: NativeOutputSupport,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct NativeOutputSupport {
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
#[serde(default)]
|
||||
pub opt_in: bool,
|
||||
#[serde(default)]
|
||||
pub mapped: bool,
|
||||
#[serde(default)]
|
||||
pub replaces_unforceable_tool: bool,
|
||||
}
|
||||
|
||||
impl NativeOutputSupport {
|
||||
pub fn is_opt_in(self) -> bool {
|
||||
!self.enabled && (self.opt_in || self.replaces_unforceable_tool)
|
||||
}
|
||||
|
||||
pub fn retries(self) -> bool {
|
||||
self.enabled || self.is_opt_in()
|
||||
}
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
|
|
@ -99,6 +111,7 @@ impl Default for MessagesModelCapabilities {
|
|||
supports_speed: false,
|
||||
supports_mid_conversation_system: false,
|
||||
effort_tiers: SupportedEffortTiers::default(),
|
||||
bedrock: BedrockCapabilities::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -111,6 +124,7 @@ pub struct ThinkingBudgets {
|
|||
pub high: u64,
|
||||
pub xhigh: u64,
|
||||
pub max: u64,
|
||||
pub bedrock_min: u64,
|
||||
}
|
||||
|
||||
impl Default for ThinkingBudgets {
|
||||
|
|
@ -122,6 +136,7 @@ impl Default for ThinkingBudgets {
|
|||
high: 4096,
|
||||
xhigh: 8192,
|
||||
max: 16384,
|
||||
bedrock_min: 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -140,6 +155,9 @@ impl ThinkingBudgets {
|
|||
high: tier("HIGH", defaults.high),
|
||||
xhigh: tier("XHIGH", defaults.xhigh),
|
||||
max: tier("MAX", defaults.max),
|
||||
bedrock_min: env
|
||||
.parsed::<u64>("BEDROCK_MIN_THINKING_BUDGET_TOKENS")
|
||||
.unwrap_or(defaults.bedrock_min),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -159,9 +177,7 @@ pub struct MessagesTransformContext {
|
|||
pub thinking: ThinkingContext,
|
||||
pub drop_params: bool,
|
||||
pub drops_unsupported_params: bool,
|
||||
pub unparsed: UnparsedMessages,
|
||||
/// Logs Python's warning for a param `drop_params` drops because the model lacks it.
|
||||
pub logs_dropped_params: bool,
|
||||
pub global_drop_params: bool,
|
||||
}
|
||||
impl MessagesTransformContext {
|
||||
pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self {
|
||||
|
|
@ -182,8 +198,7 @@ impl MessagesTransformContext {
|
|||
},
|
||||
drop_params,
|
||||
drops_unsupported_params: false,
|
||||
unparsed: UnparsedMessages::Skipped,
|
||||
logs_dropped_params: false,
|
||||
global_drop_params: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -198,13 +213,4 @@ impl MessagesTransformContext {
|
|||
..*self
|
||||
}
|
||||
}
|
||||
|
||||
/// The same context for a provider whose transform reads unparsed messages.
|
||||
#[must_use]
|
||||
pub fn reading_unparsed_messages(&self) -> Self {
|
||||
Self {
|
||||
unparsed: UnparsedMessages::Read,
|
||||
..*self
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ pub use crate::base_llm::base_model_iterator::ByteStream;
|
|||
use crate::{Error, base_llm::sse};
|
||||
|
||||
pub type EventStream = sse::EventStream<MessagesStreamEvent>;
|
||||
pub type StreamDecoder = fn(ByteStream) -> EventStream;
|
||||
pub type FrameStream = futures_util::stream::BoxStream<'static, Result<Bytes, Error>>;
|
||||
pub type StreamDecoder = fn(ByteStream, Vec<(String, String)>) -> FrameStream;
|
||||
|
||||
fn no_control_frames(_: &litellm_framer::sse::SseEvent) -> bool {
|
||||
false
|
||||
|
|
@ -36,7 +37,7 @@ mod tests {
|
|||
MessagesContentBlockDelta, MessagesStreamUsage,
|
||||
};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
use serde_json::{Map, json};
|
||||
|
||||
use super::*;
|
||||
|
||||
|
|
@ -61,6 +62,7 @@ mod tests {
|
|||
output_tokens: Some(7),
|
||||
..MessagesStreamUsage::default()
|
||||
}),
|
||||
extra: Map::new(),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
|
@ -88,7 +90,10 @@ mod tests {
|
|||
text: "hello".into(),
|
||||
},
|
||||
},
|
||||
MessagesStreamEvent::MessageStop { usage: None },
|
||||
MessagesStreamEvent::MessageStop {
|
||||
usage: None,
|
||||
extra: Map::new(),
|
||||
},
|
||||
];
|
||||
let streamed = event_stream(in_pieces(wire.as_bytes(), size))
|
||||
.try_collect::<Vec<_>>()
|
||||
|
|
@ -138,7 +143,11 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn an_event_is_named_by_its_type() {
|
||||
let encoded = encode(&MessagesStreamEvent::MessageStop { usage: None }).unwrap();
|
||||
let encoded = encode(&MessagesStreamEvent::MessageStop {
|
||||
usage: None,
|
||||
extra: Map::new(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
encoded,
|
||||
Bytes::from(format!(
|
||||
|
|
|
|||
|
|
@ -49,15 +49,28 @@ pub trait BaseMessagesConfig: Sync {
|
|||
Ok(request)
|
||||
}
|
||||
|
||||
fn tool_custom_field(&self, _model: &str) -> ToolCustomField {
|
||||
ToolCustomField::Keep
|
||||
fn native_output_fallback(
|
||||
&self,
|
||||
_request: &MessagesRequest,
|
||||
_context: &MessagesTransformContext,
|
||||
) -> Option<MessagesTransformContext> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether a param `drop_params` drops because the model lacks it logs Python's warning.
|
||||
fn logs_dropped_params(&self) -> bool {
|
||||
fn retries_without_native_output(
|
||||
&self,
|
||||
_status: u16,
|
||||
_error_text: &str,
|
||||
_request_body: &Value,
|
||||
_model: &str,
|
||||
) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn tool_custom_field(&self, _model: &str) -> ToolCustomField {
|
||||
ToolCustomField::Keep
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_response(
|
||||
&self,
|
||||
_model: &str,
|
||||
|
|
@ -149,14 +162,15 @@ pub trait BaseMessagesConfig: Sync {
|
|||
}
|
||||
|
||||
/// The request steps Python runs once the final `anthropic-beta` header is known, which
|
||||
/// read that header or must not change which betas the request asks for.
|
||||
/// read that header, must not change which betas the request asks for, or rewrite the
|
||||
/// headers that go out with the body.
|
||||
fn finalize_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
_headers: &Headers,
|
||||
headers: Headers,
|
||||
_context: &MessagesTransformContext,
|
||||
) -> MessagesRequest {
|
||||
request
|
||||
) -> Result<(MessagesRequest, Headers), Error> {
|
||||
Ok((request, headers))
|
||||
}
|
||||
|
||||
/// The JSON that goes on the wire, for a host whose body differs from the typed request:
|
||||
|
|
@ -249,7 +263,6 @@ mod tests {
|
|||
MessagesShapeOptions {
|
||||
reasoning_auto_summary,
|
||||
presanitized,
|
||||
..MessagesShapeOptions::default()
|
||||
}
|
||||
),
|
||||
Ok(request)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
use litellm_auth::AwsParams;
|
||||
use litellm_auth_aws::{
|
||||
AwsCredentialSource, bedrock_model_id_and_region,
|
||||
constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE},
|
||||
resolve_bedrock_region,
|
||||
AwsCredentialSource, CallerSigningHeaders, bedrock_model_id_and_region,
|
||||
bedrock_runtime_endpoint, constants::BEDROCK_SERVICE, resolve_bedrock_region,
|
||||
};
|
||||
use litellm_core_utils::core_helpers::json_type_name;
|
||||
use litellm_llms_types::{
|
||||
|
|
@ -161,7 +160,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion));
|
||||
.unwrap_or_else(|| bedrock_runtime_endpoint(®ion));
|
||||
Ok(format!(
|
||||
"{}/model/{model_id}/converse",
|
||||
endpoint.trim_end_matches('/')
|
||||
|
|
@ -187,6 +186,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
region: resolve_bedrock_region(model_region.as_deref(), ¶ms, env_lookup),
|
||||
service: BEDROCK_SERVICE,
|
||||
credentials: Box::new(AwsCredentialSource::from_params(¶ms, env_lookup)),
|
||||
caller_headers: CallerSigningHeaders::Dropped,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,4 +2,5 @@
|
|||
|
||||
- Chat Completions over Converse. Converse wire docs live in `llms-types/src/providers/bedrock/AGENTS.md`
|
||||
- `topK` is not forwarded, because Python's placement depends on the model catalog this crate cannot see
|
||||
- `invoke_handler.rs` decodes InvokeModel event streams for Claude, reusing `anthropic/chat` response handling
|
||||
- `invoke_handler.rs` ports Python's `AWSEventStreamDecoder`: it decodes InvokeModel event streams the way botocore's `EventStreamBuffer` does
|
||||
- `invoke_transformations/base_invoke_transformation.rs` ports the InvokeModel helpers of Python's `base_invoke_transformation.py`
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ use litellm_auth::CredentialPlacement;
|
|||
use litellm_secret_value::SecretValue;
|
||||
|
||||
use litellm_auth_aws::{
|
||||
AwsCredentialSource, bedrock_model_id_and_region,
|
||||
constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE},
|
||||
AwsCredentialSource, CallerSigningHeaders, bedrock_model_id_and_region,
|
||||
bedrock_runtime_endpoint,
|
||||
constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_SERVICE},
|
||||
resolve_bedrock_region,
|
||||
};
|
||||
use litellm_core_utils::decline::Decline;
|
||||
|
|
@ -128,7 +129,7 @@ impl BaseConfig for AmazonConverseConfig {
|
|||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion));
|
||||
.unwrap_or_else(|| bedrock_runtime_endpoint(®ion));
|
||||
let endpoint = endpoint.trim_end_matches('/');
|
||||
// A host that already built the full Converse URL (LiteLLM's Python
|
||||
// path encodes the model id itself) passes it through untouched, the
|
||||
|
|
@ -253,6 +254,7 @@ impl BaseConfig for AmazonConverseConfig {
|
|||
region: resolve_bedrock_region(model_region.as_deref(), ¶ms, env_lookup),
|
||||
service: BEDROCK_SERVICE,
|
||||
credentials: Box::new(AwsCredentialSource::from_params(¶ms, env_lookup)),
|
||||
caller_headers: CallerSigningHeaders::Botocore,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -582,16 +584,25 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_the_converse_url_from_the_region_in_the_model_id() {
|
||||
#[rstest]
|
||||
#[case::commercial(
|
||||
"us-east-1/anthropic.claude-v2",
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
|
||||
)]
|
||||
#[case::china_partition(
|
||||
"cn-north-1/anthropic.claude-v2",
|
||||
"https://bedrock-runtime.cn-north-1.amazonaws.com.cn/model/anthropic.claude-v2/converse"
|
||||
)]
|
||||
fn builds_the_converse_url_from_the_region_in_the_model_id(
|
||||
#[case] model: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG;
|
||||
assert_eq!(
|
||||
config
|
||||
.get_complete_url(None, "us-east-1/anthropic.claude-v2", &Map::new(), &|_| {
|
||||
None
|
||||
})
|
||||
.get_complete_url(None, model, &Map::new(), &|_| None)
|
||||
.expect("url builds"),
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,132 +1,685 @@
|
|||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
use base64::Engine;
|
||||
use bytes::Buf;
|
||||
use futures_util::{Stream, StreamExt};
|
||||
use bytes::BytesMut;
|
||||
use futures_util::{StreamExt, stream};
|
||||
use litellm_framer::{
|
||||
EventStreamError,
|
||||
aws_event_stream::{AwsEventStreamCodec, Message},
|
||||
aws_event_stream::{AwsEventStreamCodec, HeaderValue, Message},
|
||||
frames,
|
||||
};
|
||||
use litellm_llms_types::formats::messages::streaming::MessagesStreamEvent;
|
||||
use serde::Deserialize;
|
||||
use litellm_python_compat::{Value as PythonValue, json as python_json, repr};
|
||||
use serde_json::Value;
|
||||
use tokio_util::codec::Decoder;
|
||||
|
||||
use crate::{
|
||||
Error,
|
||||
base_llm::{
|
||||
base_model_iterator::tunnelled_http,
|
||||
messages::streaming::{ByteStream, EventStream},
|
||||
},
|
||||
};
|
||||
use crate::{Error, ErrorDetail, base_llm::base_model_iterator::tunnelled_http};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct InvokeChunkPayload {
|
||||
bytes: String,
|
||||
const STREAM_HEAD_BYTES: usize = 200;
|
||||
const PRELUDE_BYTES: usize = 12;
|
||||
const MAX_HEADERS_BYTES: u32 = 128 * 1024;
|
||||
const MAX_PAYLOAD_BYTES: i64 = 24 * 1024 * 1024;
|
||||
|
||||
const PROVIDER_REQUEST_ID_HEADERS: [&str; 7] = [
|
||||
"x-amzn-requestid",
|
||||
"x-request-id",
|
||||
"request-id",
|
||||
"x-ms-request-id",
|
||||
"apim-request-id",
|
||||
"x-goog-request-id",
|
||||
"cf-ray",
|
||||
];
|
||||
|
||||
const STREAM_EVENT_STATUSES: [(&str, Option<u16>); 13] = [
|
||||
("chunk", None),
|
||||
("messageStart", None),
|
||||
("contentBlockStart", None),
|
||||
("contentBlockDelta", None),
|
||||
("contentBlockStop", None),
|
||||
("messageStop", None),
|
||||
("metadata", None),
|
||||
("internalServerException", Some(500)),
|
||||
("modelStreamErrorException", Some(424)),
|
||||
("validationException", Some(400)),
|
||||
("throttlingException", Some(429)),
|
||||
("modelTimeoutException", Some(408)),
|
||||
("serviceUnavailableException", Some(503)),
|
||||
];
|
||||
|
||||
const RESPONSE_STREAM_EXCEPTION_STATUSES: [(&str, u16); 6] = [
|
||||
("internalServerException", 500),
|
||||
("modelStreamErrorException", 424),
|
||||
("validationException", 400),
|
||||
("throttlingException", 429),
|
||||
("modelTimeoutException", 408),
|
||||
("serviceUnavailableException", 503),
|
||||
];
|
||||
|
||||
/// An exception Python's Bedrock stream decoder raises as it is, unmapped: botocore's
|
||||
/// `EventStreamBuffer` parser errors, `json.loads` failing on an event payload, and the
|
||||
/// chunk parser's `pop` on a payload that is not a JSON object.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum BedrockStreamRaise {
|
||||
#[error("Checksum mismatch: expected 0x{expected:08x}, calculated 0x{calculated:08x}")]
|
||||
ChecksumMismatch { expected: u32, calculated: u32 },
|
||||
#[error("Header length of {0} exceeded the maximum of {MAX_HEADERS_BYTES}")]
|
||||
InvalidHeadersLength(u32),
|
||||
#[error("Payload length of {0} exceeded the maximum of {MAX_PAYLOAD_BYTES}")]
|
||||
InvalidPayloadLength(i64),
|
||||
#[error("Duplicate header present: \"{0}\"")]
|
||||
DuplicateHeader(String),
|
||||
#[error("Bedrock event payload is not JSON: {0}")]
|
||||
InvalidJson(String),
|
||||
#[error("Bedrock chunk is not a JSON object: {0}")]
|
||||
NotAnObject(String),
|
||||
}
|
||||
|
||||
pub fn decode_invoke_chunk(message: Message) -> Result<Value, Error> {
|
||||
let payload: InvokeChunkPayload =
|
||||
serde_json::from_slice(message.payload()).map_err(|error| {
|
||||
Error::InvalidResponse(crate::ErrorDetail::invalid("Bedrock event payload", error))
|
||||
})?;
|
||||
let chunk = base64::engine::general_purpose::STANDARD
|
||||
.decode(payload.bytes)
|
||||
.map_err(|error| {
|
||||
Error::InvalidResponse(crate::ErrorDetail::invalid(
|
||||
"Bedrock event payload base64",
|
||||
error,
|
||||
))
|
||||
})?;
|
||||
serde_json::from_slice(&chunk).map_err(|error| {
|
||||
Error::InvalidResponse(crate::ErrorDetail::invalid("Anthropic stream event", error))
|
||||
pub(crate) fn raised(raise: BedrockStreamRaise) -> Error {
|
||||
Error::InvalidResponse(ErrorDetail::failed("Bedrock event stream", raise))
|
||||
}
|
||||
|
||||
/// Port of Python's `_EventStreamTally`.
|
||||
#[derive(Default)]
|
||||
struct EventStreamTally {
|
||||
bytes_received: usize,
|
||||
bytes_decoded: usize,
|
||||
events: usize,
|
||||
recognized_events: usize,
|
||||
unrecognized_event_types: std::collections::BTreeSet<String>,
|
||||
unrecognized_head: Vec<u8>,
|
||||
head: Vec<u8>,
|
||||
}
|
||||
|
||||
type SharedEventStreamTally = Arc<Mutex<EventStreamTally>>;
|
||||
|
||||
fn with_tally<T>(
|
||||
tally: &SharedEventStreamTally,
|
||||
update: impl FnOnce(&mut EventStreamTally) -> T,
|
||||
) -> T {
|
||||
update(&mut tally.lock().unwrap_or_else(PoisonError::into_inner))
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum FrameError {
|
||||
#[error(transparent)]
|
||||
Raise(#[from] BedrockStreamRaise),
|
||||
#[error(transparent)]
|
||||
Framing(#[from] EventStreamError),
|
||||
}
|
||||
|
||||
fn be_u32(bytes: &[u8], at: usize) -> u32 {
|
||||
u32::from_be_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]])
|
||||
}
|
||||
|
||||
fn checked(data: &[u8], expected: u32, seed: u32) -> Result<(), BedrockStreamRaise> {
|
||||
let mut hasher = crc32fast::Hasher::new_with_initial(seed);
|
||||
hasher.update(data);
|
||||
let calculated = hasher.finalize();
|
||||
if calculated == expected {
|
||||
return Ok(());
|
||||
}
|
||||
Err(BedrockStreamRaise::ChecksumMismatch {
|
||||
expected,
|
||||
calculated,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn invoke_chunk_stream<S, B, E>(input: S) -> impl Stream<Item = Result<Value, Error>> + Send
|
||||
where
|
||||
S: Stream<Item = Result<B, E>> + Send,
|
||||
B: Buf + Send,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
frames(input, AwsEventStreamCodec).map(|message| decode_invoke_chunk(message.map_err(framing)?))
|
||||
/// Validates each frame in the order botocore's `EventStreamBuffer` does, so a body that is not
|
||||
/// an event stream fails with the parser error Python raises.
|
||||
struct BotocoreCodec {
|
||||
tally: SharedEventStreamTally,
|
||||
}
|
||||
|
||||
fn framing(error: EventStreamError) -> Error {
|
||||
impl BotocoreCodec {
|
||||
fn prelude(src: &[u8]) -> Result<usize, BedrockStreamRaise> {
|
||||
let total = be_u32(src, 0);
|
||||
let headers = be_u32(src, 4);
|
||||
checked(&src[..PRELUDE_BYTES - 4], be_u32(src, 8), 0)?;
|
||||
if headers > MAX_HEADERS_BYTES {
|
||||
return Err(BedrockStreamRaise::InvalidHeadersLength(headers));
|
||||
}
|
||||
let payload = i64::from(total) - i64::from(headers) - 16;
|
||||
if payload > MAX_PAYLOAD_BYTES {
|
||||
return Err(BedrockStreamRaise::InvalidPayloadLength(payload));
|
||||
}
|
||||
Ok(total as usize)
|
||||
}
|
||||
|
||||
fn unique_headers(message: &Message) -> Result<(), BedrockStreamRaise> {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
message
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|header| header.name().as_str())
|
||||
.find(|name| !seen.insert(*name))
|
||||
.map_or(Ok(()), |name| {
|
||||
Err(BedrockStreamRaise::DuplicateHeader(name.to_string()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Decoder for BotocoreCodec {
|
||||
type Item = Message;
|
||||
type Error = FrameError;
|
||||
|
||||
fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Message>, FrameError> {
|
||||
if src.len() < PRELUDE_BYTES {
|
||||
return Ok(None);
|
||||
}
|
||||
let length = Self::prelude(src)?;
|
||||
if src.len() < length {
|
||||
return Ok(None);
|
||||
}
|
||||
if length >= 16 {
|
||||
checked(
|
||||
&src[PRELUDE_BYTES - 4..length - 4],
|
||||
be_u32(src, length - 4),
|
||||
be_u32(src, 8),
|
||||
)?;
|
||||
}
|
||||
let Some(message) = AwsEventStreamCodec.decode(src)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Self::unique_headers(&message)?;
|
||||
with_tally(&self.tally, |tally| tally.bytes_decoded += length);
|
||||
Ok(Some(message))
|
||||
}
|
||||
|
||||
fn decode_eof(&mut self, src: &mut BytesMut) -> Result<Option<Message>, FrameError> {
|
||||
self.decode(src)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for FrameError {
|
||||
fn from(error: std::io::Error) -> Self {
|
||||
Self::Framing(EventStreamError::Body(error))
|
||||
}
|
||||
}
|
||||
|
||||
fn header_text<'a>(message: &'a Message, name: &str) -> Option<&'a str> {
|
||||
message
|
||||
.headers()
|
||||
.iter()
|
||||
.find(|header| header.name().as_str() == name)
|
||||
.and_then(|header| match header.value() {
|
||||
HeaderValue::String(value) => Some(value.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn event_status(event_type: &str) -> Option<Option<u16>> {
|
||||
STREAM_EVENT_STATUSES
|
||||
.iter()
|
||||
.find(|(known, _)| *known == event_type)
|
||||
.map(|(_, status)| *status)
|
||||
}
|
||||
|
||||
fn python_bytes_repr(bytes: &[u8]) -> String {
|
||||
repr::repr(&PythonValue::Bytes(bytes.to_vec()))
|
||||
}
|
||||
|
||||
fn python_optional_str_repr(value: Option<&str>) -> String {
|
||||
repr::repr(&value.map_or(PythonValue::None, |text| PythonValue::Str(text.to_string())))
|
||||
}
|
||||
|
||||
fn response_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
}
|
||||
|
||||
fn request_id_headers(headers: &[(String, String)]) -> Vec<(String, String)> {
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, value)| {
|
||||
PROVIDER_REQUEST_ID_HEADERS.contains(&name.to_lowercase().as_str()) && value.is_ascii()
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bedrock_error(status: u16, message: String, headers: &[(String, String)]) -> Error {
|
||||
Error::Http(litellm_http::Error::upstream_with_headers(
|
||||
status,
|
||||
message,
|
||||
request_id_headers(headers),
|
||||
))
|
||||
}
|
||||
|
||||
/// Port of Python's `_EventStreamTally.add_event`.
|
||||
fn add_event(tally: &SharedEventStreamTally, message: &Message) {
|
||||
let message_type = header_text(message, ":message-type");
|
||||
let event_type = header_text(message, ":event-type");
|
||||
with_tally(tally, |tally| {
|
||||
tally.events += 1;
|
||||
if message_type != Some("event") || event_type.and_then(event_status).is_some() {
|
||||
tally.recognized_events += 1;
|
||||
return;
|
||||
}
|
||||
tally
|
||||
.unrecognized_event_types
|
||||
.insert(event_type.unwrap_or("<missing>").to_string());
|
||||
if tally.unrecognized_head.is_empty() {
|
||||
let payload = message.payload();
|
||||
tally.unrecognized_head = payload[..payload.len().min(STREAM_HEAD_BYTES)].to_vec();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Port of Python's `_EventStreamTally.undecoded_stream_error`.
|
||||
fn undecoded_stream_error(tally: &EventStreamTally, headers: &[(String, String)]) -> Option<Error> {
|
||||
let undecoded = tally.bytes_received.saturating_sub(tally.bytes_decoded);
|
||||
if tally.recognized_events > 0 && undecoded == 0 {
|
||||
return None;
|
||||
}
|
||||
let detail = format!(
|
||||
"content-type={}, x-amzn-requestid={}, {} bytes received",
|
||||
python_optional_str_repr(response_header(headers, "content-type")),
|
||||
python_optional_str_repr(response_header(headers, "x-amzn-requestid")),
|
||||
tally.bytes_received
|
||||
);
|
||||
let message = if tally.events == 0 {
|
||||
format!(
|
||||
"Bedrock answered the stream with HTTP 200 but its body decoded to no events ({detail}, first bytes={})",
|
||||
python_bytes_repr(&tally.head)
|
||||
)
|
||||
} else if tally.recognized_events == 0 {
|
||||
let event_types = PythonValue::List(
|
||||
tally
|
||||
.unrecognized_event_types
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(PythonValue::Str)
|
||||
.collect(),
|
||||
);
|
||||
format!(
|
||||
"Bedrock answered the stream with HTTP 200 but none of its {} events carried a known event type (event types={}, {detail}, first payload={})",
|
||||
tally.events,
|
||||
repr::repr(&event_types),
|
||||
python_bytes_repr(&tally.unrecognized_head)
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"Bedrock stream ended with {undecoded} undecoded bytes after {} events ({detail})",
|
||||
tally.events
|
||||
)
|
||||
};
|
||||
Some(bedrock_error(502, message, headers))
|
||||
}
|
||||
|
||||
fn framing(error: FrameError) -> Error {
|
||||
let error = match error {
|
||||
FrameError::Raise(raise) => return raised(raise),
|
||||
FrameError::Framing(error) => error,
|
||||
};
|
||||
match &error {
|
||||
EventStreamError::Body(body) if let Some(http) = tunnelled_http(body) => {
|
||||
Error::Http(http.clone())
|
||||
}
|
||||
EventStreamError::Body(_)
|
||||
| EventStreamError::InvalidLength(_)
|
||||
| EventStreamError::Truncated
|
||||
| EventStreamError::Malformed(_) => {
|
||||
Error::InvalidResponse(crate::ErrorDetail::failed("stream framing", error))
|
||||
}
|
||||
_ => Error::InvalidResponse(ErrorDetail::failed("stream framing", error)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode_invoke_anthropic_chunk(chunk: Value) -> Result<MessagesStreamEvent, Error> {
|
||||
serde_json::from_value(chunk).map_err(|error| {
|
||||
Error::InvalidResponse(crate::ErrorDetail::invalid("Anthropic stream event", error))
|
||||
})
|
||||
fn invalid_payload(error: impl std::error::Error + Send + Sync + 'static) -> Error {
|
||||
Error::InvalidResponse(ErrorDetail::invalid("Bedrock event payload", error))
|
||||
}
|
||||
|
||||
pub fn invoke_anthropic_event_stream(bytes: ByteStream) -> EventStream {
|
||||
Box::pin(invoke_chunk_stream(bytes).map(|chunk| decode_invoke_anthropic_chunk(chunk?)))
|
||||
/// Port of Python's `bedrock_event_chunk_text`.
|
||||
fn bedrock_event_chunk_text(payload: &[u8]) -> Result<Option<Vec<u8>>, Error> {
|
||||
let parsed: Value = serde_json::from_slice(payload).map_err(invalid_payload)?;
|
||||
let Some(chunk) = parsed.as_object().filter(|chunk| !chunk.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let encoded = chunk
|
||||
.get("bytes")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or(Error::InvalidResponse(ErrorDetail::MissingField {
|
||||
subject: "Bedrock chunk event",
|
||||
field: "bytes",
|
||||
}))?;
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.map(Some)
|
||||
.map_err(invalid_payload)
|
||||
}
|
||||
|
||||
/// Port of Python's `AWSEventStreamDecoder._parse_message_from_response`, parsed with `json.loads`.
|
||||
fn parse_message_from_response(
|
||||
message: &Message,
|
||||
headers: &[(String, String)],
|
||||
) -> Result<Option<PythonValue>, Error> {
|
||||
let body = String::from_utf8_lossy(message.payload());
|
||||
if header_text(message, ":message-type") != Some("event") {
|
||||
let error = match header_text(message, ":exception-type") {
|
||||
Some(exception_type) => {
|
||||
let status = RESPONSE_STREAM_EXCEPTION_STATUSES
|
||||
.iter()
|
||||
.find(|(known, _)| *known == exception_type)
|
||||
.map_or(400, |(_, status)| *status);
|
||||
bedrock_error(status, format!("{exception_type} {body}"), headers)
|
||||
}
|
||||
None => bedrock_error(400, body.into_owned(), headers),
|
||||
};
|
||||
return Err(error);
|
||||
}
|
||||
let event_type = header_text(message, ":event-type");
|
||||
if let Some((event_type, status)) =
|
||||
event_type.and_then(|event_type| Some((event_type, event_status(event_type).flatten()?)))
|
||||
{
|
||||
return Err(bedrock_error(
|
||||
status,
|
||||
format!("{event_type} {body}"),
|
||||
headers,
|
||||
));
|
||||
}
|
||||
let text = if event_type == Some("chunk") {
|
||||
bedrock_event_chunk_text(message.payload())?
|
||||
} else {
|
||||
Some(message.payload().to_vec()).filter(|payload| !payload.is_empty())
|
||||
};
|
||||
text.filter(|text| !text.is_empty())
|
||||
.map(|text| {
|
||||
let text = String::from_utf8_lossy(&text);
|
||||
python_json::loads(&text)
|
||||
.map_err(|_| raised(BedrockStreamRaise::InvalidJson(text.into_owned())))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Port of Python's `AWSEventStreamDecoder.aiter_bytes`, yielding the decoded JSON of each event.
|
||||
pub fn aiter_bytes(
|
||||
bytes: crate::base_llm::messages::streaming::ByteStream,
|
||||
headers: Vec<(String, String)>,
|
||||
) -> impl futures_util::Stream<Item = Result<PythonValue, Error>> + Send {
|
||||
let tally = SharedEventStreamTally::default();
|
||||
let counted = {
|
||||
let tally = tally.clone();
|
||||
bytes.inspect(move |chunk| {
|
||||
if let Ok(chunk) = chunk {
|
||||
with_tally(&tally, |tally| {
|
||||
tally.bytes_received += chunk.len();
|
||||
let room = STREAM_HEAD_BYTES.saturating_sub(tally.head.len());
|
||||
tally
|
||||
.head
|
||||
.extend_from_slice(&chunk[..chunk.len().min(room)]);
|
||||
});
|
||||
}
|
||||
})
|
||||
};
|
||||
let headers = Arc::new(headers);
|
||||
let events = {
|
||||
let tally = tally.clone();
|
||||
let headers = headers.clone();
|
||||
frames(
|
||||
counted,
|
||||
BotocoreCodec {
|
||||
tally: tally.clone(),
|
||||
},
|
||||
)
|
||||
.map(move |message| {
|
||||
let message = message.map_err(framing)?;
|
||||
add_event(&tally, &message);
|
||||
parse_message_from_response(&message, &headers)
|
||||
})
|
||||
};
|
||||
let end = stream::once(async move {
|
||||
with_tally(&tally, |tally| undecoded_stream_error(tally, &headers)).map_or(Ok(None), Err)
|
||||
});
|
||||
events
|
||||
.chain(end)
|
||||
.scan(false, |failed, item| {
|
||||
if *failed {
|
||||
return futures_util::future::ready(None);
|
||||
}
|
||||
*failed = item.is_err();
|
||||
futures_util::future::ready(Some(item))
|
||||
})
|
||||
.filter_map(|item| futures_util::future::ready(item.transpose()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use aws_smithy_eventstream::frame::write_message_to;
|
||||
use aws_smithy_types::event_stream::{Header, HeaderValue, Message};
|
||||
use aws_smithy_types::event_stream::{
|
||||
Header, HeaderValue as WireHeaderValue, Message as WireMessage,
|
||||
};
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use bytes::Bytes;
|
||||
use futures_util::TryStreamExt;
|
||||
use litellm_llms_types::formats::messages::streaming::MessagesContentBlockDelta;
|
||||
use futures_util::StreamExt;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
use crate::base_llm::messages::streaming;
|
||||
|
||||
fn in_pieces(wire: &[u8]) -> ByteStream {
|
||||
let pieces: Vec<Bytes> = wire.chunks(3).map(Bytes::copy_from_slice).collect();
|
||||
futures_util::stream::iter(pieces.into_iter().map(Ok)).boxed()
|
||||
}
|
||||
|
||||
const TEXT_DELTA: &str =
|
||||
r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}"#;
|
||||
|
||||
fn aws_wire(chunk: &str) -> Vec<u8> {
|
||||
let payload = serde_json::json!({"bytes": STANDARD.encode(chunk)});
|
||||
let message = Message::new(Bytes::from(serde_json::to_vec(&payload).unwrap())).add_header(
|
||||
Header::new(":event-type", HeaderValue::String("chunk".into())),
|
||||
fn frame(headers: &[(&str, &str)], payload: &[u8]) -> Vec<u8> {
|
||||
let message = headers.iter().fold(
|
||||
WireMessage::new(Bytes::copy_from_slice(payload)),
|
||||
|message, (name, value)| {
|
||||
message.add_header(Header::new(
|
||||
name.to_string(),
|
||||
WireHeaderValue::String(value.to_string().into()),
|
||||
))
|
||||
},
|
||||
);
|
||||
let mut wire = Vec::new();
|
||||
write_message_to(&message, &mut wire).unwrap();
|
||||
wire
|
||||
}
|
||||
|
||||
fn chunk(event: &Value) -> Vec<u8> {
|
||||
let payload = json!({"bytes": STANDARD.encode(event.to_string())});
|
||||
frame(
|
||||
&[(":event-type", "chunk"), (":message-type", "event")],
|
||||
payload.to_string().as_bytes(),
|
||||
)
|
||||
}
|
||||
|
||||
fn response_headers() -> Vec<(String, String)> {
|
||||
vec![
|
||||
(
|
||||
"content-type".into(),
|
||||
"application/vnd.amazon.eventstream".into(),
|
||||
),
|
||||
("x-amzn-requestid".into(), "rid-1".into()),
|
||||
("x-other".into(), "dropped".into()),
|
||||
]
|
||||
}
|
||||
|
||||
async fn decoded(wire: Vec<u8>) -> (Vec<PythonValue>, Option<Error>) {
|
||||
let bytes = futures_util::stream::iter(
|
||||
wire.chunks(5)
|
||||
.map(|piece| Ok(Bytes::copy_from_slice(piece)))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.boxed();
|
||||
let items: Vec<Result<PythonValue, Error>> =
|
||||
aiter_bytes(bytes, response_headers()).collect().await;
|
||||
let (events, errors): (Vec<_>, Vec<_>) = items.into_iter().partition(Result::is_ok);
|
||||
(
|
||||
events.into_iter().filter_map(Result::ok).collect(),
|
||||
errors.into_iter().find_map(Result::err),
|
||||
)
|
||||
}
|
||||
|
||||
fn upstream(error: Option<Error>) -> (u16, String, Vec<(String, String)>) {
|
||||
let Some(Error::Http(error)) = error else {
|
||||
panic!("expected an upstream error, got {error:?}");
|
||||
};
|
||||
let (status, body) = error.upstream_response().unwrap();
|
||||
(status, body.to_string(), error.upstream_headers().to_vec())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aws_and_sse_framing_decode_to_the_same_anthropic_events() {
|
||||
let aws = aws_wire(TEXT_DELTA);
|
||||
let sse = format!("event: content_block_delta\ndata: {TEXT_DELTA}\n\n");
|
||||
|
||||
let from_aws = invoke_anthropic_event_stream(in_pieces(&aws))
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
let from_sse = streaming::event_stream(in_pieces(sse.as_bytes()))
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
async fn chunk_events_decode_to_anthropic_events() {
|
||||
let event = json!({"type": "message_stop"});
|
||||
let (events, error) = decoded(chunk(&event)).await;
|
||||
assert_eq!(
|
||||
from_aws,
|
||||
vec![MessagesStreamEvent::ContentBlockDelta {
|
||||
index: 0,
|
||||
delta: MessagesContentBlockDelta::TextDelta {
|
||||
text: "hello".into(),
|
||||
},
|
||||
}]
|
||||
(events, error.is_none()),
|
||||
(vec![python_json::loads(&event.to_string()).unwrap()], true)
|
||||
);
|
||||
assert_eq!(from_aws, from_sse);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::throttling("throttlingException", 429)]
|
||||
#[case::model_stream_error("modelStreamErrorException", 424)]
|
||||
#[case::validation("validationException", 400)]
|
||||
#[case::unmodeled("somethingNewException", 400)]
|
||||
#[tokio::test]
|
||||
async fn exception_frames_raise_their_modeled_status(
|
||||
#[case] exception_type: &str,
|
||||
#[case] status: u16,
|
||||
) {
|
||||
let wire = [
|
||||
chunk(&json!({"type": "ping"})),
|
||||
frame(
|
||||
&[
|
||||
(":message-type", "exception"),
|
||||
(":exception-type", exception_type),
|
||||
],
|
||||
br#"{"message":"slow down"}"#,
|
||||
),
|
||||
]
|
||||
.concat();
|
||||
let (events, error) = decoded(wire).await;
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(
|
||||
upstream(error),
|
||||
(
|
||||
status,
|
||||
format!(r#"{exception_type} {{"message":"slow down"}}"#),
|
||||
vec![("x-amzn-requestid".to_string(), "rid-1".to_string())]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_error_event_type_raises_its_status() {
|
||||
let wire = frame(
|
||||
&[
|
||||
(":message-type", "event"),
|
||||
(":event-type", "internalServerException"),
|
||||
],
|
||||
b"boom",
|
||||
);
|
||||
assert_eq!(upstream(decoded(wire).await.1).0, 500);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_empty_body_is_a_502() {
|
||||
let (_, body, _) = upstream(decoded(Vec::new()).await.1);
|
||||
assert_eq!(
|
||||
body,
|
||||
"Bedrock answered the stream with HTTP 200 but its body decoded to no events (content-type='application/vnd.amazon.eventstream', x-amzn-requestid='rid-1', 0 bytes received, first bytes=b'')"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_event_types_are_a_502() {
|
||||
let wire = frame(
|
||||
&[(":message-type", "event"), (":event-type", "mystery")],
|
||||
b"{}",
|
||||
);
|
||||
let (status, body, _) = upstream(decoded(wire.clone()).await.1);
|
||||
assert_eq!(
|
||||
(status, body),
|
||||
(
|
||||
502,
|
||||
format!(
|
||||
"Bedrock answered the stream with HTTP 200 but none of its 1 events carried a known event type (event types=['mystery'], content-type='application/vnd.amazon.eventstream', x-amzn-requestid='rid-1', {} bytes received, first payload=b'{{}}')",
|
||||
wire.len()
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn trailing_undecoded_bytes_are_a_502() {
|
||||
let complete = chunk(&json!({"type": "ping"}));
|
||||
let truncated = chunk(&json!({"type": "message_stop"}));
|
||||
let wire = [complete, truncated[..20].to_vec()].concat();
|
||||
let (events, error) = decoded(wire).await;
|
||||
let (status, body, _) = upstream(error);
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(status, 502);
|
||||
assert!(
|
||||
body.starts_with("Bedrock stream ended with 20 undecoded bytes after 1 events ("),
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decoding_stops_at_the_first_error() {
|
||||
let wire = [
|
||||
frame(
|
||||
&[
|
||||
(":message-type", "exception"),
|
||||
(":exception-type", "throttlingException"),
|
||||
],
|
||||
b"x",
|
||||
),
|
||||
chunk(&json!({"type": "ping"})),
|
||||
]
|
||||
.concat();
|
||||
let bytes = futures_util::stream::iter([Ok(Bytes::from(wire))]).boxed();
|
||||
let items: Vec<Result<PythonValue, Error>> = aiter_bytes(bytes, Vec::new()).collect().await;
|
||||
assert_eq!(items.len(), 1);
|
||||
assert!(items[0].is_err());
|
||||
}
|
||||
|
||||
fn raise_of(error: Option<Error>) -> BedrockStreamRaise {
|
||||
litellm_error::find::<BedrockStreamRaise>(&error.expect("an error"))
|
||||
.cloned()
|
||||
.expect("a botocore parser error")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_json_body_fails_the_prelude_checksum_as_botocore_does() {
|
||||
let body = br#"{"id":"msg_1","type":"message","content":[]}"#;
|
||||
assert_eq!(
|
||||
raise_of(decoded(body.to_vec()).await.1),
|
||||
BedrockStreamRaise::ChecksumMismatch {
|
||||
expected: be_u32(body, 8),
|
||||
calculated: crc32fast::hash(&body[..8]),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
fn with_prelude(total: u32, headers: u32) -> Vec<u8> {
|
||||
let mut prelude = [total.to_be_bytes(), headers.to_be_bytes()].concat();
|
||||
let crc = crc32fast::hash(&prelude);
|
||||
prelude.extend_from_slice(&crc.to_be_bytes());
|
||||
prelude
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::headers_too_long(
|
||||
with_prelude(200_000, 140_000),
|
||||
BedrockStreamRaise::InvalidHeadersLength(140_000)
|
||||
)]
|
||||
#[case::payload_too_long(
|
||||
with_prelude(30_000_000, 0),
|
||||
BedrockStreamRaise::InvalidPayloadLength(30_000_000 - 16)
|
||||
)]
|
||||
#[case::duplicate_header(
|
||||
frame(&[(":message-type", "event"), (":message-type", "event")], b"{}"),
|
||||
BedrockStreamRaise::DuplicateHeader(":message-type".into())
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn a_malformed_frame_raises_the_botocore_parser_error(
|
||||
#[case] wire: Vec<u8>,
|
||||
#[case] expected: BedrockStreamRaise,
|
||||
) {
|
||||
assert_eq!(raise_of(decoded(wire).await.1), expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_corrupt_message_checksum_raises_a_checksum_mismatch() {
|
||||
let mut wire = chunk(&json!({"type": "ping"}));
|
||||
let last = wire.len() - 1;
|
||||
wire[last] ^= 0xff;
|
||||
assert!(matches!(
|
||||
raise_of(decoded(wire).await.1),
|
||||
BedrockStreamRaise::ChecksumMismatch { .. }
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,186 @@
|
|||
use litellm_python_compat::{json as python_json, repr};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::Error;
|
||||
|
||||
pub const GUARDRAIL_CONFIG_PARAM: &str = "guardrailConfig";
|
||||
|
||||
pub(crate) const GUARDRAIL_IDENTIFIER_HEADER: &str = "X-Amzn-Bedrock-GuardrailIdentifier";
|
||||
pub(crate) const GUARDRAIL_VERSION_HEADER: &str = "X-Amzn-Bedrock-GuardrailVersion";
|
||||
const GUARDRAIL_TRACE_HEADER: &str = "X-Amzn-Bedrock-Trace";
|
||||
const GUARDRAIL_CONFIG_EXPECTED_FORMAT: &str = "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}";
|
||||
const GUARDRAIL_TRACE_VALUES: [&str; 3] = ["enabled", "disabled", "enabled_full"];
|
||||
|
||||
fn python_repr(value: &Value) -> String {
|
||||
repr::to_str(&python_json::from_json(value.clone()))
|
||||
}
|
||||
|
||||
fn python_type_name(value: &Value) -> &'static str {
|
||||
match value {
|
||||
Value::Null => "NoneType",
|
||||
Value::Bool(_) => "bool",
|
||||
Value::Number(number) if number.is_f64() => "float",
|
||||
Value::Number(_) => "int",
|
||||
Value::String(_) => "str",
|
||||
Value::Array(_) => "list",
|
||||
Value::Object(_) => "dict",
|
||||
}
|
||||
}
|
||||
|
||||
fn validation_error(location: &str, message: &str, kind: &str, input: &Value) -> String {
|
||||
let location = if location.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("\n{location}")
|
||||
};
|
||||
format!(
|
||||
"1 validation error for GuardrailConfigBlock{location}\n {message} [type={kind}, input_value={}, input_type={}]",
|
||||
python_repr(input),
|
||||
python_type_name(input)
|
||||
)
|
||||
}
|
||||
|
||||
fn guardrail_violation(config: &Map<String, Value>) -> Option<String> {
|
||||
["guardrailIdentifier", "guardrailVersion"]
|
||||
.into_iter()
|
||||
.find_map(|field| {
|
||||
config
|
||||
.get(field)
|
||||
.filter(|value| !value.is_string())
|
||||
.map(|value| {
|
||||
validation_error(
|
||||
field,
|
||||
"Input should be a valid string",
|
||||
"string_type",
|
||||
value,
|
||||
)
|
||||
})
|
||||
})
|
||||
.or_else(|| {
|
||||
config
|
||||
.get("trace")
|
||||
.filter(|value| {
|
||||
!value
|
||||
.as_str()
|
||||
.is_some_and(|trace| GUARDRAIL_TRACE_VALUES.contains(&trace))
|
||||
})
|
||||
.map(|value| {
|
||||
validation_error(
|
||||
"trace",
|
||||
"Input should be 'enabled', 'disabled' or 'enabled_full'",
|
||||
"literal_error",
|
||||
value,
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn guardrail_error(message: String) -> Error {
|
||||
Error::Http(litellm_http::Error::upstream(400, message))
|
||||
}
|
||||
|
||||
/// Port of Python's `bedrock_invoke_guardrail_headers`.
|
||||
pub(crate) fn bedrock_invoke_guardrail_headers(
|
||||
raw: &Value,
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
let Value::Object(config) = raw else {
|
||||
return Err(guardrail_error(format!(
|
||||
"Invalid guardrailConfig={}. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}. Error: {}",
|
||||
python_repr(raw),
|
||||
validation_error("", "Input should be a valid dictionary", "dict_type", raw)
|
||||
)));
|
||||
};
|
||||
if let Some(violation) = guardrail_violation(config) {
|
||||
return Err(guardrail_error(format!(
|
||||
"Invalid guardrailConfig={}. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}. Error: {violation}",
|
||||
python_repr(raw)
|
||||
)));
|
||||
}
|
||||
let Some(identifier) = config.get("guardrailIdentifier").and_then(Value::as_str) else {
|
||||
return Err(guardrail_error(format!(
|
||||
"guardrailConfig={} is missing 'guardrailIdentifier'. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}",
|
||||
python_repr(raw)
|
||||
)));
|
||||
};
|
||||
Ok([
|
||||
(GUARDRAIL_IDENTIFIER_HEADER, Some(identifier.to_string())),
|
||||
(
|
||||
GUARDRAIL_VERSION_HEADER,
|
||||
config
|
||||
.get("guardrailVersion")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string),
|
||||
),
|
||||
(
|
||||
GUARDRAIL_TRACE_HEADER,
|
||||
config
|
||||
.get("trace")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_uppercase),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(name, value)| Some((name.to_string(), value?)))
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn rejection(error: Error) -> (u16, String) {
|
||||
let Error::Http(error) = error else {
|
||||
panic!("expected a 400, got {error:?}");
|
||||
};
|
||||
let (status, body) = error.upstream_response().unwrap();
|
||||
(status, body.to_string())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::enabled("enabled", "ENABLED")]
|
||||
#[case::disabled("disabled", "DISABLED")]
|
||||
#[case::enabled_full("enabled_full", "ENABLED_FULL")]
|
||||
fn guardrail_config_becomes_invoke_headers(#[case] trace: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
bedrock_invoke_guardrail_headers(
|
||||
&json!({"guardrailIdentifier": "gid", "guardrailVersion": "2", "trace": trace})
|
||||
)
|
||||
.unwrap(),
|
||||
vec![
|
||||
(GUARDRAIL_IDENTIFIER_HEADER.to_string(), "gid".to_string()),
|
||||
(GUARDRAIL_VERSION_HEADER.to_string(), "2".to_string()),
|
||||
(GUARDRAIL_TRACE_HEADER.to_string(), expected.to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_guardrail_config_without_an_identifier_is_a_400() {
|
||||
assert_eq!(
|
||||
rejection(bedrock_invoke_guardrail_headers(&json!({"guardrailVersion": "2"})).unwrap_err()),
|
||||
(
|
||||
400,
|
||||
"guardrailConfig={'guardrailVersion': '2'} is missing 'guardrailIdentifier'. Expected format: {'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}".to_string()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bad_trace(json!({"guardrailIdentifier": "gid", "trace": "loud"}))]
|
||||
#[case::not_a_dict(json!("gid"))]
|
||||
#[case::non_string_version(json!({"guardrailIdentifier": "gid", "guardrailVersion": 2}))]
|
||||
fn an_invalid_guardrail_config_is_a_400(#[case] raw: Value) {
|
||||
let (status, body) = rejection(bedrock_invoke_guardrail_headers(&raw).unwrap_err());
|
||||
assert_eq!(status, 400);
|
||||
assert!(
|
||||
body.starts_with(&format!(
|
||||
"Invalid guardrailConfig={}. Expected format: ",
|
||||
python_repr(&raw)
|
||||
)),
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
pub mod base_invoke_transformation;
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
pub mod converse_transformation;
|
||||
pub mod invoke_handler;
|
||||
pub mod invoke_transformations;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use std::sync::LazyLock;
|
|||
use litellm_auth::{AwsParams, CredentialPlacement, ParamKind, ParamSpec};
|
||||
use litellm_secret_value::SecretValue;
|
||||
|
||||
use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
|
||||
use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders, resolve_aws_region};
|
||||
use litellm_core_utils::settings::resolve_non_empty;
|
||||
use litellm_http::request::{header_value, with_header, without_headers};
|
||||
use litellm_llms_types::{
|
||||
|
|
@ -24,7 +24,7 @@ use crate::{
|
|||
base_llm::{
|
||||
auth::{AuthScheme, Headers, ValidatedEnvironment},
|
||||
messages::{
|
||||
context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages},
|
||||
context::{MessagesShapeOptions, MessagesTransformContext},
|
||||
transformation::BaseMessagesConfig,
|
||||
},
|
||||
},
|
||||
|
|
@ -123,6 +123,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
|
|||
region: platform_region(base.as_deref(), params, env)?,
|
||||
service: SERVICE,
|
||||
credentials: Box::new(AwsCredentialSource::from_params(¶ms.aws, env)),
|
||||
caller_headers: CallerSigningHeaders::Botocore,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -196,12 +197,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
|
|||
Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
|
||||
None => headers,
|
||||
};
|
||||
update_headers_with_safeguards_beta(
|
||||
headers,
|
||||
request,
|
||||
BetaProvider::Anthropic,
|
||||
UnparsedMessages::Skipped,
|
||||
)
|
||||
update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic)
|
||||
}
|
||||
|
||||
fn wire_body(&self, body: Value) -> Value {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
use litellm_auth_aws::is_bedrock_region;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::{Error, ErrorDetail};
|
||||
|
||||
pub mod structured_output;
|
||||
|
||||
const ROUTE_PREFIXES: [&str; 3] = ["bedrock/converse/", "bedrock/", "converse/"];
|
||||
const ROUTING_PREFIXES: [&str; 8] = [
|
||||
"bedrock/",
|
||||
|
|
@ -27,7 +30,7 @@ pub fn bedrock_base_model(model: &str) -> &str {
|
|||
if unrouted.starts_with("nova/") {
|
||||
return "amazon.nova-custom";
|
||||
}
|
||||
let named = without_throughput_suffix(model_name_from_arn(without_routing_prefixes(model)));
|
||||
let named = without_throughput_suffix(model_name_from_arn(strip_bedrock_routing_prefix(model)));
|
||||
if let Some((region, rest)) = named.split_once('.')
|
||||
&& CROSS_REGION_INFERENCE_PREFIXES.contains(®ion)
|
||||
{
|
||||
|
|
@ -43,7 +46,8 @@ pub fn is_bedrock_application_inference_profile_arn(model: &str) -> bool {
|
|||
model.contains(":application-inference-profile/")
|
||||
}
|
||||
|
||||
pub fn without_routing_prefixes(model: &str) -> &str {
|
||||
/// Port of Python's `strip_bedrock_routing_prefix`.
|
||||
pub fn strip_bedrock_routing_prefix(model: &str) -> &str {
|
||||
ROUTING_PREFIXES.into_iter().fold(model, |model, prefix| {
|
||||
model.strip_prefix(prefix).unwrap_or(model)
|
||||
})
|
||||
|
|
@ -83,6 +87,181 @@ fn is_digits(value: &str) -> bool {
|
|||
!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
const UNNAMED_TOOL_PREFIX: &str = "litellm_unnamed_tool_";
|
||||
|
||||
pub(crate) fn is_int(value: &Value) -> bool {
|
||||
value.is_i64() || value.is_u64()
|
||||
}
|
||||
|
||||
fn effort_rank(effort: &str) -> Option<u8> {
|
||||
match effort {
|
||||
"low" => Some(0),
|
||||
"medium" => Some(1),
|
||||
"high" => Some(2),
|
||||
"max" => Some(3),
|
||||
"xhigh" => Some(4),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn clamp_effort(effort: &str, ceiling: Option<&str>) -> Option<String> {
|
||||
let ceiling = ceiling?;
|
||||
(effort_rank(effort)? > effort_rank(ceiling)?).then(|| ceiling.to_string())
|
||||
}
|
||||
|
||||
/// Port of Python's `normalize_bedrock_opus_output_config_effort`.
|
||||
pub(crate) fn normalize_bedrock_opus_output_config_effort(
|
||||
output_config: Option<&mut Value>,
|
||||
ceiling: Option<&str>,
|
||||
) {
|
||||
let Some(Value::Object(output_config)) = output_config else {
|
||||
return;
|
||||
};
|
||||
if let Some(clamped) = output_config
|
||||
.get("effort")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|effort| clamp_effort(effort, ceiling))
|
||||
{
|
||||
output_config.insert("effort".into(), Value::String(clamped));
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `strip_unsupported_bedrock_invoke_output_config_keys`.
|
||||
pub(crate) fn strip_unsupported_bedrock_invoke_output_config_keys(
|
||||
model: &str,
|
||||
request_body: &mut Map<String, Value>,
|
||||
accepts_output_config: bool,
|
||||
) {
|
||||
let Some(Value::Object(output_config)) = request_body.get("output_config") else {
|
||||
return;
|
||||
};
|
||||
if output_config.keys().all(|key| key == "format") || accepts_output_config {
|
||||
return;
|
||||
}
|
||||
litellm_tracing::warn!(
|
||||
"Bedrock Invoke: stripping unsupported `output_config` keys for model={model}: neither `supports_output_config` nor any `supports_*_reasoning_effort` flag is set in model_prices_and_context_window.json. Add the capability flag to the model JSON entry if this model accepts `output_config`."
|
||||
);
|
||||
match output_config
|
||||
.get("format")
|
||||
.filter(|format| !format.is_null())
|
||||
.cloned()
|
||||
{
|
||||
Some(format) => {
|
||||
request_body.insert("output_config".into(), json!({"format": format}));
|
||||
}
|
||||
None => {
|
||||
request_body.shift_remove("output_config");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tools_mut(request_body: &mut Map<String, Value>) -> Option<&mut Vec<Value>> {
|
||||
match request_body.get_mut("tools") {
|
||||
Some(Value::Array(tools)) => Some(tools),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `_normalize_custom_field_on_tool`.
|
||||
fn normalize_custom_field_on_tool(tool: &mut Value) {
|
||||
let Value::Object(tool) = tool else {
|
||||
return;
|
||||
};
|
||||
if let Some(Value::Object(custom)) = tool.shift_remove("custom")
|
||||
&& !tool.contains_key("defer_loading")
|
||||
&& let Some(deferred @ Value::Bool(_)) = custom.get("defer_loading")
|
||||
{
|
||||
tool.insert("defer_loading".into(), deferred.clone());
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `normalize_custom_field_on_tools`.
|
||||
pub(crate) fn normalize_custom_field_on_tools(request_body: &mut Map<String, Value>) {
|
||||
if let Some(tools) = tools_mut(request_body) {
|
||||
tools.iter_mut().for_each(normalize_custom_field_on_tool);
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `normalize_json_schema_custom_types_to_object`.
|
||||
fn normalize_json_schema_custom_types_to_object(schema: &mut Value) {
|
||||
let Value::Object(node) = schema else {
|
||||
return;
|
||||
};
|
||||
if node.get("type").and_then(Value::as_str) == Some("custom") {
|
||||
node.insert("type".into(), Value::String("object".into()));
|
||||
}
|
||||
for key in ["items", "additionalProperties"] {
|
||||
if let Some(child) = node.get_mut(key) {
|
||||
normalize_json_schema_custom_types_to_object(child);
|
||||
}
|
||||
}
|
||||
if let Some(Value::Object(properties)) = node.get_mut("properties") {
|
||||
properties
|
||||
.values_mut()
|
||||
.for_each(normalize_json_schema_custom_types_to_object);
|
||||
}
|
||||
for combiner in ["allOf", "anyOf", "oneOf"] {
|
||||
if let Some(Value::Array(branches)) = node.get_mut(combiner) {
|
||||
branches
|
||||
.iter_mut()
|
||||
.for_each(normalize_json_schema_custom_types_to_object);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `normalize_tool_input_schema_types_for_bedrock_invoke`.
|
||||
pub(crate) fn normalize_tool_input_schema_types_for_bedrock_invoke(
|
||||
request_body: &mut Map<String, Value>,
|
||||
) {
|
||||
let Some(tools) = tools_mut(request_body) else {
|
||||
return;
|
||||
};
|
||||
tools
|
||||
.iter_mut()
|
||||
.filter_map(|tool| tool.get_mut("input_schema"))
|
||||
.filter(|input_schema| input_schema.is_object())
|
||||
.for_each(normalize_json_schema_custom_types_to_object);
|
||||
}
|
||||
|
||||
/// Port of Python's `ensure_bedrock_anthropic_messages_tool_names`.
|
||||
pub(crate) fn ensure_bedrock_anthropic_messages_tool_names(request_body: &mut Map<String, Value>) {
|
||||
let Some(tools) = tools_mut(request_body) else {
|
||||
return;
|
||||
};
|
||||
for (index, tool) in tools.iter_mut().enumerate() {
|
||||
let Value::Object(tool) = tool else {
|
||||
continue;
|
||||
};
|
||||
let unnamed = match tool.get("name") {
|
||||
None | Some(Value::Null) => true,
|
||||
Some(Value::String(name)) => name.trim().is_empty(),
|
||||
Some(_) => false,
|
||||
};
|
||||
if unnamed {
|
||||
tool.insert(
|
||||
"name".into(),
|
||||
Value::String(format!("{UNNAMED_TOOL_PREFIX}{index}")),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `tools_without_eager_input_streaming`.
|
||||
pub(crate) fn tools_without_eager_input_streaming(request_body: &mut Map<String, Value>) {
|
||||
let Some(tools) = tools_mut(request_body) else {
|
||||
return;
|
||||
};
|
||||
if !tools.iter().all(Value::is_object) {
|
||||
return;
|
||||
}
|
||||
tools
|
||||
.iter_mut()
|
||||
.filter_map(Value::as_object_mut)
|
||||
.for_each(|tool| {
|
||||
tool.shift_remove("eager_input_streaming");
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn validated_region(region: String) -> Result<String, Error> {
|
||||
if !region.is_empty()
|
||||
&& region
|
||||
|
|
@ -101,8 +280,42 @@ pub(crate) fn validated_region(region: String) -> Result<String, Error> {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::bedrock_base_model;
|
||||
use super::*;
|
||||
use crate::{
|
||||
anthropic::common_utils::supports_effort_param,
|
||||
base_llm::messages::context::{
|
||||
BedrockCapabilities, MessagesModelCapabilities, SupportedEffortTiers,
|
||||
},
|
||||
};
|
||||
|
||||
fn adaptive() -> MessagesModelCapabilities {
|
||||
MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_adaptive_thinking: true,
|
||||
supports_output_config: true,
|
||||
effort_tiers: SupportedEffortTiers {
|
||||
low: true,
|
||||
medium: true,
|
||||
high: true,
|
||||
max: true,
|
||||
xhigh: true,
|
||||
..SupportedEffortTiers::default()
|
||||
},
|
||||
..MessagesModelCapabilities::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn with_bedrock(
|
||||
capabilities: MessagesModelCapabilities,
|
||||
bedrock: BedrockCapabilities,
|
||||
) -> MessagesModelCapabilities {
|
||||
MessagesModelCapabilities {
|
||||
bedrock,
|
||||
..capabilities
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::cross_region("us.anthropic.claude-x-v1:0", "anthropic.claude-x-v1:0")]
|
||||
|
|
@ -132,4 +345,66 @@ mod tests {
|
|||
fn base_model_matches_the_python_resolution(#[case] model: &str, #[case] expected: &str) {
|
||||
assert_eq!(bedrock_base_model(model), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::hoists_bool_defer_loading(
|
||||
json!([{"name": "Read", "input_schema": {"type": "object"}, "custom": {"defer_loading": true}}, {"name": "Write", "input_schema": {"type": "object"}}]),
|
||||
json!([{"name": "Read", "input_schema": {"type": "object"}, "defer_loading": true}, {"name": "Write", "input_schema": {"type": "object"}}]),
|
||||
)]
|
||||
#[case::keeps_an_existing_defer_loading(
|
||||
json!([{"name": "Read", "defer_loading": false, "custom": {"defer_loading": true}}]),
|
||||
json!([{"name": "Read", "defer_loading": false}]),
|
||||
)]
|
||||
#[case::drops_a_non_bool_defer_loading(
|
||||
json!([{"name": "Read", "custom": {"defer_loading": "yes", "other": 1}}]),
|
||||
json!([{"name": "Read"}]),
|
||||
)]
|
||||
#[case::custom_schema_types_become_object(
|
||||
json!([{"name": "t", "input_schema": {"type": "custom", "properties": {"a": {"type": "custom", "items": {"type": "custom"}}}, "anyOf": [{"type": "custom"}]}}]),
|
||||
json!([{"name": "t", "input_schema": {"type": "object", "properties": {"a": {"type": "object", "items": {"type": "object"}}}, "anyOf": [{"type": "object"}]}}]),
|
||||
)]
|
||||
#[case::unnamed_tools_get_their_index(
|
||||
json!([{"name": "first"}, {"input_schema": {"type": "object"}}, {"name": " "}, {"name": null}]),
|
||||
json!([{"name": "first"}, {"input_schema": {"type": "object"}, "name": "litellm_unnamed_tool_1"}, {"name": "litellm_unnamed_tool_2"}, {"name": "litellm_unnamed_tool_3"}]),
|
||||
)]
|
||||
fn tools_are_normalized_for_bedrock(#[case] tools: Value, #[case] expected: Value) {
|
||||
let mut request = json!({"tools": tools}).as_object().unwrap().clone();
|
||||
normalize_custom_field_on_tools(&mut request);
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke(&mut request);
|
||||
ensure_bedrock_anthropic_messages_tool_names(&mut request);
|
||||
assert_eq!(request["tools"], expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::stripped_without_support(
|
||||
MessagesModelCapabilities::default(),
|
||||
json!({"effort": "high"}),
|
||||
None,
|
||||
)]
|
||||
#[case::kept_with_output_config_support(
|
||||
adaptive(),
|
||||
json!({"effort": "high"}),
|
||||
Some(json!({"effort": "high"})),
|
||||
)]
|
||||
#[case::bedrock_flag_keeps_it(
|
||||
with_bedrock(MessagesModelCapabilities::default(), BedrockCapabilities { supports_output_config: true, ..BedrockCapabilities::default() }),
|
||||
json!({"effort": "high"}),
|
||||
Some(json!({"effort": "high"})),
|
||||
)]
|
||||
fn output_config_keys_follow_model_support(
|
||||
#[case] capabilities: MessagesModelCapabilities,
|
||||
#[case] output_config: Value,
|
||||
#[case] expected: Option<Value>,
|
||||
) {
|
||||
let mut request = json!({"output_config": output_config})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone();
|
||||
strip_unsupported_bedrock_invoke_output_config_keys(
|
||||
"model",
|
||||
&mut request,
|
||||
capabilities.bedrock.supports_output_config || supports_effort_param(&capabilities),
|
||||
);
|
||||
assert_eq!(request.get("output_config").cloned(), expected);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,904 @@
|
|||
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
|
||||
use litellm_python_compat::{json as python_json, repr};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::{Error, ErrorDetail, base_llm::messages::context::NativeOutputSupport};
|
||||
|
||||
use litellm_llms_types::recognized::python_truthy;
|
||||
|
||||
use super::is_int;
|
||||
|
||||
type Body = Map<String, Value>;
|
||||
|
||||
const NATIVE_OUTPUT_MAX_PROPERTIES: usize = 64;
|
||||
const NATIVE_OUTPUT_MAX_UNION_PARAMETERS: usize = 16;
|
||||
const NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: usize = 24;
|
||||
const NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: u128 = 2048;
|
||||
const MAX_SCHEMA_NESTING: usize = 1024;
|
||||
const NATIVE_OUTPUT_STRING_FORMATS: [&str; 10] = [
|
||||
"date-time",
|
||||
"time",
|
||||
"date",
|
||||
"duration",
|
||||
"email",
|
||||
"hostname",
|
||||
"uri",
|
||||
"ipv4",
|
||||
"ipv6",
|
||||
"uuid",
|
||||
];
|
||||
const SUBSCHEMA_KEYWORDS: [&str; 11] = [
|
||||
"additionalItems",
|
||||
"additionalProperties",
|
||||
"contains",
|
||||
"else",
|
||||
"if",
|
||||
"items",
|
||||
"not",
|
||||
"propertyNames",
|
||||
"then",
|
||||
"unevaluatedItems",
|
||||
"unevaluatedProperties",
|
||||
];
|
||||
const SUBSCHEMA_LIST_KEYWORDS: [&str; 5] = ["allOf", "anyOf", "items", "oneOf", "prefixItems"];
|
||||
const SUBSCHEMA_MAP_KEYWORDS: [&str; 5] = [
|
||||
"$defs",
|
||||
"definitions",
|
||||
"dependentSchemas",
|
||||
"patternProperties",
|
||||
"properties",
|
||||
];
|
||||
const CONSTRAINT_LABELS: [(&str, &str); 25] = [
|
||||
("minItems", "minimum number of items: {}"),
|
||||
("maxItems", "maximum number of items: {}"),
|
||||
("uniqueItems", "all array items must be unique"),
|
||||
("contains", "array must contain an item matching: {}"),
|
||||
("minContains", "minimum number of matching items: {}"),
|
||||
("maxContains", "maximum number of matching items: {}"),
|
||||
("prefixItems", "leading items must match, in order: {}"),
|
||||
("minimum", "minimum value: {}"),
|
||||
("maximum", "maximum value: {}"),
|
||||
("exclusiveMinimum", "exclusive minimum value: {}"),
|
||||
("exclusiveMaximum", "exclusive maximum value: {}"),
|
||||
("multipleOf", "must be a multiple of {}"),
|
||||
("minLength", "minimum length: {}"),
|
||||
("maxLength", "maximum length: {}"),
|
||||
("minProperties", "minimum number of properties: {}"),
|
||||
("maxProperties", "maximum number of properties: {}"),
|
||||
(
|
||||
"patternProperties",
|
||||
"properties whose names match each pattern must satisfy: {}",
|
||||
),
|
||||
("propertyNames", "property names must satisfy: {}"),
|
||||
("dependentRequired", "dependent required properties: {}"),
|
||||
("dependentSchemas", "dependent schemas: {}"),
|
||||
(
|
||||
"unevaluatedProperties",
|
||||
"unevaluated properties must satisfy: {}",
|
||||
),
|
||||
("if", "conditional (if): {}"),
|
||||
("then", "conditional (then): {}"),
|
||||
("else", "conditional (else): {}"),
|
||||
("not", "must not match: {}"),
|
||||
];
|
||||
|
||||
type Schema = Map<String, Value>;
|
||||
|
||||
fn subschemas(node: &Schema) -> impl Iterator<Item = &Schema> {
|
||||
node.iter().flat_map(|(key, value)| {
|
||||
let key = key.as_str();
|
||||
let children: Vec<&Schema> = match value {
|
||||
Value::Object(entries) if SUBSCHEMA_MAP_KEYWORDS.contains(&key) => {
|
||||
entries.values().filter_map(Value::as_object).collect()
|
||||
}
|
||||
Value::Array(entries) if SUBSCHEMA_LIST_KEYWORDS.contains(&key) => {
|
||||
entries.iter().filter_map(Value::as_object).collect()
|
||||
}
|
||||
Value::Object(child) if SUBSCHEMA_KEYWORDS.contains(&key) => vec![child],
|
||||
_ => Vec::new(),
|
||||
};
|
||||
children
|
||||
})
|
||||
}
|
||||
|
||||
fn schema_nodes(schema: &Schema) -> Vec<&Schema> {
|
||||
let mut nodes = Vec::new();
|
||||
let mut frontier = vec![schema];
|
||||
let mut depth = 0;
|
||||
while !frontier.is_empty() && depth < MAX_SCHEMA_NESTING {
|
||||
let next: Vec<&Schema> = frontier.iter().flat_map(|node| subschemas(node)).collect();
|
||||
nodes.extend(frontier);
|
||||
frontier = next;
|
||||
depth += 1;
|
||||
}
|
||||
nodes
|
||||
}
|
||||
|
||||
fn schema_at<'a>(node: &'a Schema, key: &str) -> Option<&'a Schema> {
|
||||
node.get(key).and_then(Value::as_object)
|
||||
}
|
||||
|
||||
fn list_at<'a>(node: &'a Schema, key: &str) -> &'a [Value] {
|
||||
match node.get(key) {
|
||||
Some(Value::Array(values)) => values,
|
||||
_ => &[],
|
||||
}
|
||||
}
|
||||
|
||||
fn schema_fragment_ref(reference: &str) -> String {
|
||||
if reference.starts_with("/$defs/") || reference.starts_with("/definitions/") {
|
||||
format!("#{reference}")
|
||||
} else {
|
||||
reference.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn refs_in(schema: &Schema) -> BTreeSet<String> {
|
||||
schema_nodes(schema)
|
||||
.into_iter()
|
||||
.filter_map(|node| node.get("$ref").and_then(Value::as_str))
|
||||
.map(schema_fragment_ref)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn root_definitions(schema: &Schema) -> Vec<(String, &Schema)> {
|
||||
["$defs", "definitions"]
|
||||
.into_iter()
|
||||
.filter_map(|defs_key| schema_at(schema, defs_key).map(|defs| (defs_key, defs)))
|
||||
.flat_map(|(defs_key, defs)| {
|
||||
defs.iter().filter_map(move |(name, body)| {
|
||||
body.as_object()
|
||||
.map(|body| (format!("#/{defs_key}/{name}"), body))
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn has_cycle(graph: &BTreeMap<String, BTreeSet<String>>) -> bool {
|
||||
let mut unresolved: BTreeMap<&str, usize> = BTreeMap::new();
|
||||
let mut dependents: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
|
||||
graph.iter().for_each(|(node, edges)| {
|
||||
let known = edges.iter().filter(|edge| graph.contains_key(*edge));
|
||||
unresolved.insert(node, known.clone().count());
|
||||
known.for_each(|edge| dependents.entry(edge).or_default().push(node));
|
||||
});
|
||||
let mut ready: VecDeque<&str> = unresolved
|
||||
.iter()
|
||||
.filter(|(_, count)| **count == 0)
|
||||
.map(|(node, _)| *node)
|
||||
.collect();
|
||||
let mut resolved = 0;
|
||||
while let Some(node) = ready.pop_front() {
|
||||
resolved += 1;
|
||||
dependents
|
||||
.get(node)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.for_each(|dependent| {
|
||||
if let Some(count) = unresolved.get_mut(dependent) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
ready.push_back(*dependent);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
resolved != graph.len()
|
||||
}
|
||||
|
||||
fn schema_has_unresolvable_ref(schema: &Schema) -> bool {
|
||||
let definitions: BTreeMap<String, &Schema> = root_definitions(schema).into_iter().collect();
|
||||
if !refs_in(schema)
|
||||
.iter()
|
||||
.all(|reference| definitions.contains_key(reference))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
has_cycle(
|
||||
&definitions
|
||||
.iter()
|
||||
.map(|(pointer, body)| (pointer.clone(), refs_in(body)))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_object_schema(node: &Schema) -> bool {
|
||||
match node.get("type") {
|
||||
Some(Value::String(kind)) => kind == "object",
|
||||
Some(Value::Array(kinds)) => kinds.iter().any(|kind| kind.as_str() == Some("object")),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_open_object(node: &Schema) -> bool {
|
||||
let additional = node.get("additionalProperties");
|
||||
if additional == Some(&Value::Bool(true))
|
||||
|| additional.is_some_and(Value::is_object)
|
||||
|| node.contains_key("patternProperties")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
is_object_schema(node)
|
||||
&& !node.contains_key("properties")
|
||||
&& additional != Some(&Value::Bool(false))
|
||||
}
|
||||
|
||||
fn uses_regex_lookaround(pattern: &str) -> bool {
|
||||
["(?=", "(?!", "(?<=", "(?<!"]
|
||||
.iter()
|
||||
.any(|lookaround| pattern.contains(lookaround))
|
||||
}
|
||||
|
||||
fn may_be_string(node: &Schema) -> bool {
|
||||
match node.get("type") {
|
||||
None => true,
|
||||
Some(Value::String(kind)) => kind == "string",
|
||||
Some(_) => list_at(node, "type")
|
||||
.iter()
|
||||
.any(|kind| kind.as_str() == Some("string")),
|
||||
}
|
||||
}
|
||||
|
||||
fn optional_property_names(node: &Schema) -> BTreeSet<&str> {
|
||||
let Some(properties) = schema_at(node, "properties") else {
|
||||
return BTreeSet::new();
|
||||
};
|
||||
let required: BTreeSet<&str> = list_at(node, "required")
|
||||
.iter()
|
||||
.filter_map(Value::as_str)
|
||||
.collect();
|
||||
properties
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.filter(|name| !required.contains(name))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_union(node: &Schema) -> bool {
|
||||
node.get("anyOf").is_some_and(Value::is_array)
|
||||
|| node.get("oneOf").is_some_and(Value::is_array)
|
||||
|| list_at(node, "type").len() > 1
|
||||
}
|
||||
|
||||
fn grammar_branches(node: &Schema) -> u128 {
|
||||
let Some(properties) = schema_at(node, "properties") else {
|
||||
return 0;
|
||||
};
|
||||
let optional = optional_property_names(node).len();
|
||||
let required = (properties.len() - optional.min(properties.len())) as u128 + 1;
|
||||
u32::try_from(optional)
|
||||
.ok()
|
||||
.and_then(|shift| 1u128.checked_shl(shift))
|
||||
.and_then(|branches| branches.checked_mul(required))
|
||||
.unwrap_or(u128::MAX)
|
||||
}
|
||||
|
||||
/// Port of Python's `schema_property_count`.
|
||||
pub(crate) fn schema_property_count(schema: &Schema) -> usize {
|
||||
schema_nodes(schema)
|
||||
.into_iter()
|
||||
.filter_map(|node| schema_at(node, "properties"))
|
||||
.map(Map::len)
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn native_output_rejects(schema: &Schema) -> bool {
|
||||
let nodes = schema_nodes(schema);
|
||||
let string_formats_supported = nodes
|
||||
.iter()
|
||||
.filter(|node| may_be_string(node))
|
||||
.filter_map(|node| node.get("format").and_then(Value::as_str))
|
||||
.all(|format| NATIVE_OUTPUT_STRING_FORMATS.contains(&format));
|
||||
let union_properties = nodes
|
||||
.iter()
|
||||
.filter_map(|node| schema_at(node, "properties"))
|
||||
.flat_map(|properties| properties.values().filter_map(Value::as_object))
|
||||
.filter(|property| is_union(property))
|
||||
.count();
|
||||
let optional_properties: usize = nodes
|
||||
.iter()
|
||||
.map(|node| optional_property_names(node).len())
|
||||
.sum();
|
||||
let grammar_branches = nodes
|
||||
.iter()
|
||||
.map(|node| grammar_branches(node))
|
||||
.fold(0u128, u128::saturating_add);
|
||||
schema_has_unresolvable_ref(schema)
|
||||
|| nodes.iter().any(|node| is_open_object(node))
|
||||
|| nodes.iter().any(|node| {
|
||||
node.get("pattern")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(uses_regex_lookaround)
|
||||
})
|
||||
|| !string_formats_supported
|
||||
|| union_properties > NATIVE_OUTPUT_MAX_UNION_PARAMETERS
|
||||
|| optional_properties > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS
|
||||
|| grammar_branches > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES
|
||||
}
|
||||
|
||||
fn enum_conflicts_with_declared_type(schema: &Schema) -> bool {
|
||||
let Some(Value::Array(values)) = schema.get("enum") else {
|
||||
return false;
|
||||
};
|
||||
let declared = match schema.get("type") {
|
||||
None | Some(Value::Null) => return false,
|
||||
Some(Value::Array(_)) => return true,
|
||||
Some(Value::String(kind)) => kind.as_str(),
|
||||
Some(_) => return false,
|
||||
};
|
||||
let check: fn(&Value) -> bool = match declared {
|
||||
"null" => Value::is_null,
|
||||
"boolean" => Value::is_boolean,
|
||||
"integer" => is_int,
|
||||
"number" => Value::is_number,
|
||||
"string" => Value::is_string,
|
||||
"array" => Value::is_array,
|
||||
"object" => Value::is_object,
|
||||
_ => return false,
|
||||
};
|
||||
!values.iter().all(check)
|
||||
}
|
||||
|
||||
fn note_value(value: &Value) -> Result<String, Error> {
|
||||
let python = python_json::from_json(value.clone());
|
||||
match value {
|
||||
Value::Object(_) | Value::Array(_) => python_json::dumps(&python).map_err(|error| {
|
||||
Error::InvalidRequest(ErrorDetail::failed(
|
||||
"output schema note serialization",
|
||||
error,
|
||||
))
|
||||
}),
|
||||
_ => Ok(repr::to_str(&python)),
|
||||
}
|
||||
}
|
||||
|
||||
fn described_with_note(description: Option<&Value>, note: String) -> Result<Value, Error> {
|
||||
match description {
|
||||
Some(Value::String(text)) if !text.is_empty() => {
|
||||
Ok(Value::String(format!("{text} {note}")))
|
||||
}
|
||||
Some(other) if python_truthy(other) => {
|
||||
Err(Error::Declined(litellm_core_utils::decline::Decline::new(
|
||||
"a non-string output schema description next to a dropped constraint",
|
||||
)))
|
||||
}
|
||||
_ => Ok(Value::String(note)),
|
||||
}
|
||||
}
|
||||
|
||||
fn filtered_subschema(value: &Value) -> Result<Value, Error> {
|
||||
match value {
|
||||
Value::Object(schema) => filter_output_schema(schema).map(Value::Object),
|
||||
other => Ok(other.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
fn filtered_subschemas(value: &Value) -> Result<Value, Error> {
|
||||
match value {
|
||||
Value::Object(entries) => entries
|
||||
.iter()
|
||||
.map(|(name, schema)| Ok((name.clone(), filtered_subschema(schema)?)))
|
||||
.collect::<Result<Map<_, _>, Error>>()
|
||||
.map(Value::Object),
|
||||
other => Ok(other.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
fn filtered_branches(branches: &[Value]) -> Result<Vec<Value>, Error> {
|
||||
branches.iter().map(filtered_subschema).collect()
|
||||
}
|
||||
|
||||
fn extended_any_of(result: &Schema, branches: &[Value]) -> Result<Value, Error> {
|
||||
let filtered = filtered_branches(branches)?;
|
||||
match result.get("anyOf") {
|
||||
Some(Value::Array(existing)) => Ok(Value::Array(
|
||||
existing.iter().cloned().chain(filtered).collect(),
|
||||
)),
|
||||
Some(_) => Err(Error::Declined(litellm_core_utils::decline::Decline::new(
|
||||
"an output schema whose anyOf cannot be extended",
|
||||
))),
|
||||
None => Ok(Value::Array(filtered)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn filter_output_schema(schema: &Schema) -> Result<Schema, Error> {
|
||||
let enforced_min_items = schema
|
||||
.get("minItems")
|
||||
.filter(|value| is_int(value))
|
||||
.and_then(Value::as_i64)
|
||||
.is_some_and(|count| count == 0 || count == 1);
|
||||
let unsupported = |field: &str| {
|
||||
CONSTRAINT_LABELS.iter().any(|(label, _)| *label == field)
|
||||
&& !(field == "minItems" && enforced_min_items)
|
||||
};
|
||||
let notes = CONSTRAINT_LABELS
|
||||
.iter()
|
||||
.filter(|(field, _)| unsupported(field))
|
||||
.filter_map(|(field, label)| schema.get(*field).map(|value| (label, value)))
|
||||
.filter(|(_, value)| **value != Value::Bool(false))
|
||||
.map(|(label, value)| Ok(label.replacen("{}", ¬e_value(value)?, 1)))
|
||||
.collect::<Result<Vec<String>, Error>>()?;
|
||||
let mut result = Map::new();
|
||||
if !notes.is_empty() {
|
||||
let note = format!("Note: {}.", notes.join(", "));
|
||||
result.insert(
|
||||
"description".into(),
|
||||
described_with_note(schema.get("description"), note)?,
|
||||
);
|
||||
}
|
||||
let drops_conflicting_type = enum_conflicts_with_declared_type(schema);
|
||||
for (key, value) in schema {
|
||||
let key = key.as_str();
|
||||
if unsupported(key)
|
||||
|| (key == "type" && drops_conflicting_type)
|
||||
|| (key == "description" && result.contains_key("description"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let filtered = match (key, value) {
|
||||
("properties" | "$defs" | "definitions", _) => filtered_subschemas(value)?,
|
||||
("items", _) => filtered_subschema(value)?,
|
||||
("anyOf" | "oneOf", Value::Array(branches)) => {
|
||||
let extended = extended_any_of(&result, branches)?;
|
||||
result.insert("anyOf".into(), extended);
|
||||
continue;
|
||||
}
|
||||
("allOf", Value::Array(branches)) => Value::Array(filtered_branches(branches)?),
|
||||
("$ref", Value::String(reference)) => Value::String(schema_fragment_ref(reference)),
|
||||
_ => value.clone(),
|
||||
};
|
||||
result.insert(key.to_string(), filtered);
|
||||
}
|
||||
if is_object_schema(&result) {
|
||||
result.insert("additionalProperties".into(), Value::Bool(false));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Port of Python's `AnthropicConfig.native_output_schema`.
|
||||
pub(crate) fn native_output_schema(schema: &Schema) -> Result<Option<Schema>, Error> {
|
||||
if native_output_rejects(schema) {
|
||||
return Ok(None);
|
||||
}
|
||||
filter_output_schema(schema).map(Some)
|
||||
}
|
||||
|
||||
/// Port of Python's `bedrock_native_output_format`, with `bedrock_native_output_schema` and
|
||||
/// `bedrock_cleaned_output_schema` inlined.
|
||||
pub(crate) fn bedrock_native_output_format(
|
||||
output_format: &Schema,
|
||||
native: NativeOutputSupport,
|
||||
) -> Result<Option<Schema>, Error> {
|
||||
let Some(Value::Object(schema)) = output_format.get("schema") else {
|
||||
return Ok((native.enabled || native.opt_in).then(|| output_format.clone()));
|
||||
};
|
||||
let cleaned = if !native.mapped && schema_property_count(schema) > NATIVE_OUTPUT_MAX_PROPERTIES
|
||||
{
|
||||
None
|
||||
} else {
|
||||
native_output_schema(schema)?
|
||||
};
|
||||
let native_schema = if native.enabled {
|
||||
Some(cleaned.unwrap_or_else(|| schema.clone()))
|
||||
} else if native.opt_in {
|
||||
cleaned
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(native_schema.map(|native_schema| {
|
||||
let mut format = output_format.clone();
|
||||
format.insert("schema".into(), Value::Object(native_schema));
|
||||
format
|
||||
}))
|
||||
}
|
||||
|
||||
/// Port of Python's `pop_bedrock_invoke_output_config_format`.
|
||||
fn pop_bedrock_invoke_output_config_format(body: &mut Body) -> Option<Schema> {
|
||||
let Some(Value::Object(output_config)) = body.get_mut("output_config") else {
|
||||
return None;
|
||||
};
|
||||
let format = output_config.shift_remove("format");
|
||||
if output_config.is_empty() {
|
||||
body.shift_remove("output_config");
|
||||
}
|
||||
match format {
|
||||
Some(Value::Object(format)) => Some(format),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `convert_bedrock_invoke_output_format_to_inline_schema`.
|
||||
fn convert_bedrock_invoke_output_format_to_inline_schema(
|
||||
output_format: &Schema,
|
||||
body: &mut Body,
|
||||
) -> Result<(), Error> {
|
||||
let Some(schema) = output_format
|
||||
.get("schema")
|
||||
.filter(|schema| python_truthy(schema))
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let text = python_json::dumps(&python_json::from_json(schema.clone())).map_err(|error| {
|
||||
Error::InvalidRequest(ErrorDetail::failed("output schema serialization", error))
|
||||
})?;
|
||||
let schema_block = json!({"type": "text", "text": text});
|
||||
let Some(Value::Array(messages)) = body.get_mut("messages") else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(last_user) = messages
|
||||
.iter_mut()
|
||||
.rev()
|
||||
.find(|message| message.get("role").and_then(Value::as_str) == Some("user"))
|
||||
.and_then(Value::as_object_mut)
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let content = match last_user.get("content") {
|
||||
None => Value::Array(vec![schema_block]),
|
||||
Some(Value::String(text)) => json!([{"type": "text", "text": text}, schema_block]),
|
||||
Some(Value::Array(blocks)) => {
|
||||
Value::Array(blocks.iter().cloned().chain([schema_block]).collect())
|
||||
}
|
||||
Some(_) => return Ok(()),
|
||||
};
|
||||
last_user.insert("content".into(), content);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Port of Python's `apply_bedrock_invoke_structured_output`.
|
||||
pub(crate) fn apply_bedrock_invoke_structured_output(
|
||||
body: &mut Body,
|
||||
model: &str,
|
||||
native: NativeOutputSupport,
|
||||
) -> Result<(), Error> {
|
||||
let legacy = body.shift_remove("output_format");
|
||||
let output_config_format = pop_bedrock_invoke_output_config_format(body);
|
||||
let schema_format = match legacy {
|
||||
Some(Value::Object(legacy)) => Some(legacy),
|
||||
_ => output_config_format,
|
||||
};
|
||||
let Some(schema_format) = schema_format else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(native_format) = bedrock_native_output_format(&schema_format, native)? {
|
||||
match body.get_mut("output_config") {
|
||||
Some(Value::Object(output_config)) => {
|
||||
output_config.insert("format".into(), Value::Object(native_format));
|
||||
}
|
||||
_ => {
|
||||
body.insert("output_config".into(), json!({"format": native_format}));
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
litellm_tracing::warn!(
|
||||
"Bedrock Invoke: model={model} does not take native structured output for this schema per model_prices_and_context_window.json, so the JSON schema was inlined into the last user message and is NOT enforced by the model."
|
||||
);
|
||||
convert_bedrock_invoke_output_format_to_inline_schema(&schema_format, body)
|
||||
}
|
||||
|
||||
/// Port of Python's `requests_output_config_format`.
|
||||
fn requests_output_config_format(params: &Map<String, Value>) -> bool {
|
||||
matches!(params.get("output_config"), Some(Value::Object(output_config)) if output_config.contains_key("format"))
|
||||
}
|
||||
|
||||
pub(crate) fn requested_output_format(params: &Map<String, Value>) -> Option<&Schema> {
|
||||
match params.get("output_format") {
|
||||
Some(Value::Object(legacy)) => Some(legacy),
|
||||
_ => params
|
||||
.get("output_config")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|output_config| output_config.get("format"))
|
||||
.and_then(Value::as_object),
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `bedrock_native_output_retry_requested`.
|
||||
pub(crate) fn bedrock_native_output_retry_requested(
|
||||
params: &Map<String, Value>,
|
||||
native: NativeOutputSupport,
|
||||
) -> bool {
|
||||
(params.get("output_format").is_some_and(Value::is_object)
|
||||
|| requests_output_config_format(params))
|
||||
&& native.retries()
|
||||
}
|
||||
|
||||
pub(crate) fn sent_output_config_format(body: &Value) -> bool {
|
||||
body.as_object().is_some_and(requests_output_config_format)
|
||||
}
|
||||
|
||||
pub(crate) const NATIVE_OUTPUT_UNSUPPORTED: &str =
|
||||
"output_config.format: Extra inputs are not permitted";
|
||||
|
||||
const NATIVE_OUTPUT_REJECTIONS: [&str; 4] = [
|
||||
NATIVE_OUTPUT_UNSUPPORTED,
|
||||
"compilation exceeded limits: Schema is too complex",
|
||||
"The compiled grammar is too large",
|
||||
"output_config.format.schema:",
|
||||
];
|
||||
|
||||
/// Port of Python's `is_bedrock_native_output_rejection`.
|
||||
pub(crate) fn is_bedrock_native_output_rejection(status: u16, error_text: &str) -> bool {
|
||||
status == 400
|
||||
&& NATIVE_OUTPUT_REJECTIONS
|
||||
.iter()
|
||||
.any(|rejection| error_text.contains(rejection))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn schema(value: Value) -> Schema {
|
||||
value.as_object().unwrap().clone()
|
||||
}
|
||||
|
||||
fn legacy() -> NativeOutputSupport {
|
||||
NativeOutputSupport {
|
||||
enabled: true,
|
||||
mapped: true,
|
||||
..NativeOutputSupport::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn bedrock_opt_in() -> NativeOutputSupport {
|
||||
NativeOutputSupport {
|
||||
opt_in: true,
|
||||
..NativeOutputSupport::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn tree() -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {"root": {"$ref": "#/$defs/Node"}},
|
||||
"$defs": {"Node": {"type": "object", "properties": {"children": {"type": "array", "items": {"$ref": "#/$defs/Node"}}}}},
|
||||
})
|
||||
}
|
||||
|
||||
fn open_object() -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {"counts": {"type": "object", "additionalProperties": {"type": "integer"}}},
|
||||
})
|
||||
}
|
||||
|
||||
fn many_properties(count: usize) -> Value {
|
||||
let properties: Map<String, Value> = (0..count)
|
||||
.map(|index| (format!("p{index}"), json!({"type": "string"})))
|
||||
.collect();
|
||||
let required: Vec<Value> = (0..count).map(|index| json!(format!("p{index}"))).collect();
|
||||
json!({"type": "object", "properties": properties, "required": required, "additionalProperties": false})
|
||||
}
|
||||
|
||||
fn sent(native: NativeOutputSupport, raw: Value) -> Body {
|
||||
let mut body = schema(json!({
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"output_config": {"format": {"type": "json_schema", "schema": raw}},
|
||||
}));
|
||||
apply_bedrock_invoke_structured_output(&mut body, "model", native).unwrap();
|
||||
body
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_output_drops_schema_keywords_bedrock_rejects() {
|
||||
let body = sent(
|
||||
bedrock_opt_in(),
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"count": {"type": "integer", "minimum": 1},
|
||||
"ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
},
|
||||
"required": ["count", "ids"],
|
||||
}),
|
||||
);
|
||||
assert_eq!(
|
||||
body["output_config"]["format"]["schema"],
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"count": {"description": "Note: minimum value: 1.", "type": "integer"},
|
||||
"ids": {"description": "Note: all array items must be unique.", "type": "array", "items": {"type": "string"}},
|
||||
},
|
||||
"required": ["count", "ids"],
|
||||
"additionalProperties": false,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::recursive_on_a_bedrock_profile(bedrock_opt_in(), tree(), false)]
|
||||
#[case::open_object_on_a_bedrock_profile(bedrock_opt_in(), open_object(), false)]
|
||||
#[case::recursive_on_a_legacy_profile(legacy(), tree(), true)]
|
||||
#[case::open_object_on_a_legacy_profile(legacy(), open_object(), true)]
|
||||
fn schemas_native_output_cannot_hold(
|
||||
#[case] native: NativeOutputSupport,
|
||||
#[case] raw: Value,
|
||||
#[case] forwarded: bool,
|
||||
) {
|
||||
let body = sent(native, raw.clone());
|
||||
if forwarded {
|
||||
assert_eq!(body["output_config"]["format"]["schema"], raw);
|
||||
} else {
|
||||
assert_eq!(body.get("output_config"), None);
|
||||
let text = body["messages"][0]["content"][1]["text"].as_str().unwrap();
|
||||
assert_eq!(serde_json::from_str::<Value>(text).unwrap(), raw);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_defs_reach_native_output_once() {
|
||||
let party = json!({"$ref": "#/$defs/Party"});
|
||||
let body = sent(
|
||||
legacy(),
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {"buyer": party, "seller": party},
|
||||
"required": ["buyer", "seller"],
|
||||
"additionalProperties": false,
|
||||
"$defs": {"Party": {"type": "object", "properties": {"email": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, "required": ["email"], "additionalProperties": false}},
|
||||
}),
|
||||
);
|
||||
let sent_schema = &body["output_config"]["format"]["schema"];
|
||||
assert_eq!(sent_schema["properties"]["buyer"], party);
|
||||
assert_eq!(sent_schema.to_string().matches("\"anyOf\"").count(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unmapped_over_the_limit(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, 65, false)]
|
||||
#[case::unmapped_at_the_limit(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, 24, true)]
|
||||
#[case::mapped_over_the_limit(NativeOutputSupport { opt_in: true, mapped: true, ..NativeOutputSupport::default() }, 65, true)]
|
||||
fn property_count_rule_applies_to_unmapped_models(
|
||||
#[case] native: NativeOutputSupport,
|
||||
#[case] properties: usize,
|
||||
#[case] native_sent: bool,
|
||||
) {
|
||||
assert_eq!(
|
||||
sent(native, many_properties(properties)).contains_key("output_config"),
|
||||
native_sent
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_output_format_wins() {
|
||||
let mut body = schema(json!({
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
|
||||
"output_format": {"type": "json_schema", "schema": {"type": "object", "properties": {"legacy": {"type": "string"}}}},
|
||||
"output_config": {"effort": "high", "format": {"type": "json_schema", "schema": {"type": "object", "properties": {"new": {"type": "string"}}}}},
|
||||
}));
|
||||
apply_bedrock_invoke_structured_output(&mut body, "model", NativeOutputSupport::default())
|
||||
.unwrap();
|
||||
assert_eq!(body["output_config"], json!({"effort": "high"}));
|
||||
assert_eq!(
|
||||
body["messages"][0]["content"][1]["text"],
|
||||
json!(r#"{"type": "object", "properties": {"legacy": {"type": "string"}}}"#)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::one_of_becomes_any_of(
|
||||
json!({"oneOf": [{"type": "string"}, {"type": "integer", "maximum": 3}]}),
|
||||
json!({"anyOf": [{"type": "string"}, {"description": "Note: maximum value: 3.", "type": "integer"}]}),
|
||||
)]
|
||||
#[case::small_min_items_is_kept(
|
||||
json!({"type": "array", "minItems": 1, "maxItems": 4}),
|
||||
json!({"description": "Note: maximum number of items: 4.", "type": "array", "minItems": 1}),
|
||||
)]
|
||||
#[case::false_constraints_add_no_note(
|
||||
json!({"type": "array", "uniqueItems": false}),
|
||||
json!({"type": "array"}),
|
||||
)]
|
||||
#[case::conflicting_enum_type_is_dropped(
|
||||
json!({"type": "string", "enum": [1, 2]}),
|
||||
json!({"enum": [1, 2]}),
|
||||
)]
|
||||
#[case::sub_schema_constraints_note_their_json(
|
||||
json!({"type": "array", "contains": {"type": "string"}, "description": "Items"}),
|
||||
json!({"description": "Items Note: array must contain an item matching: {\"type\": \"string\"}.", "type": "array"}),
|
||||
)]
|
||||
#[case::ref_becomes_a_fragment(
|
||||
json!({"$ref": "/$defs/Node"}),
|
||||
json!({"$ref": "#/$defs/Node"}),
|
||||
)]
|
||||
fn output_schema_filter_matches_the_anthropic_sdk(#[case] raw: Value, #[case] expected: Value) {
|
||||
assert_eq!(
|
||||
Value::Object(filter_output_schema(&schema(raw)).unwrap()),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::lookaround(json!({"type": "string", "pattern": "^(?!x)"}), true)]
|
||||
#[case::unsupported_format(json!({"type": "string", "format": "regex"}), true)]
|
||||
#[case::supported_format(json!({"type": "string", "format": "email"}), false)]
|
||||
#[case::unresolvable_ref(json!({"$ref": "#/$defs/Missing"}), true)]
|
||||
#[case::too_many_optional(json!({"type": "object", "properties": (0..25).map(|i| (format!("p{i}"), json!({"type": "string"}))).collect::<Map<String, Value>>(), "additionalProperties": false}), true)]
|
||||
fn native_output_rejects_what_the_grammar_cannot_compile(
|
||||
#[case] raw: Value,
|
||||
#[case] rejected: bool,
|
||||
) {
|
||||
assert_eq!(native_output_rejects(&schema(raw)), rejected);
|
||||
}
|
||||
|
||||
fn graph(edges: Vec<(&str, Vec<&str>)>) -> BTreeMap<String, BTreeSet<String>> {
|
||||
edges
|
||||
.into_iter()
|
||||
.map(|(node, refs)| {
|
||||
(
|
||||
node.to_string(),
|
||||
refs.into_iter().map(str::to_string).collect(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::none(vec![], false)]
|
||||
#[case::chain(vec![("a", vec!["b"]), ("b", vec!["c"]), ("c", vec![])], false)]
|
||||
#[case::diamond(vec![("a", vec!["b", "c"]), ("b", vec!["d"]), ("c", vec!["d"]), ("d", vec![])], false)]
|
||||
#[case::unknown_edges_are_ignored(vec![("a", vec!["missing"])], false)]
|
||||
#[case::self_reference(vec![("a", vec!["a"])], true)]
|
||||
#[case::two_node_cycle(vec![("a", vec!["b"]), ("b", vec!["a"])], true)]
|
||||
#[case::cycle_behind_a_resolved_node(vec![("a", vec![]), ("b", vec!["a", "c"]), ("c", vec!["b"])], true)]
|
||||
fn has_cycle_finds_only_reference_loops(
|
||||
#[case] edges: Vec<(&str, Vec<&str>)>,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
assert_eq!(has_cycle(&graph(edges)), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_cycle_stays_linear_on_a_long_reference_chain() {
|
||||
let size = 50_000;
|
||||
let chain: BTreeMap<String, BTreeSet<String>> = (0..size)
|
||||
.map(|i| {
|
||||
(
|
||||
format!("n{i}"),
|
||||
(i + 1 < size)
|
||||
.then(|| format!("n{}", i + 1))
|
||||
.into_iter()
|
||||
.collect(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let started = std::time::Instant::now();
|
||||
assert!(!has_cycle(&chain));
|
||||
assert!(started.elapsed() < std::time::Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::extra_inputs(400, "output_config.format: Extra inputs are not permitted", true)]
|
||||
#[case::too_complex(400, "compilation exceeded limits: Schema is too complex", true)]
|
||||
#[case::grammar_too_large(400, "The compiled grammar is too large", true)]
|
||||
#[case::schema_error(400, "output_config.format.schema: bad", true)]
|
||||
#[case::other_400(400, "messages: bad", false)]
|
||||
#[case::not_a_400(500, "output_config.format: Extra inputs are not permitted", false)]
|
||||
fn native_output_rejections_are_recognized(
|
||||
#[case] status: u16,
|
||||
#[case] text: &str,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
assert_eq!(is_bedrock_native_output_rejection(status, text), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::enabled(legacy(), true)]
|
||||
#[case::opt_in(bedrock_opt_in(), true)]
|
||||
#[case::replaces_unforceable_tool(NativeOutputSupport { replaces_unforceable_tool: true, ..NativeOutputSupport::default() }, true)]
|
||||
#[case::unsupported(NativeOutputSupport::default(), false)]
|
||||
fn native_output_retry_needs_native_support(
|
||||
#[case] native: NativeOutputSupport,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
let params = schema(json!({"output_config": {"format": {"type": "json_schema"}}}));
|
||||
assert_eq!(
|
||||
bedrock_native_output_retry_requested(¶ms, native),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
# rules
|
||||
|
||||
- Claude on Bedrock InvokeModel: reuses `anthropic/messages` shaping, moves the model into the URL, and decodes the AWS event stream
|
||||
- Claude on Bedrock InvokeModel: reuses `anthropic/messages` shaping, moves the model into the URL, and decodes the AWS event stream through `chat/invoke_handler.rs`
|
||||
- `invoke_transformations/` holds only what Python's `AmazonAnthropicClaudeMessagesConfig` holds, under its method names. Helpers Python keeps in `bedrock/common_utils.py` and `bedrock/request_metadata.py` live in the matching Rust modules
|
||||
- Invocation metrics in the response map onto Messages usage here
|
||||
|
||||
# references
|
||||
|
|
|
|||
|
|
@ -1,57 +1,63 @@
|
|||
use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error;
|
||||
use litellm_core_utils::decline::Decline;
|
||||
use litellm_host::interceptors::RetriedRequest;
|
||||
use std::convert::Infallible;
|
||||
|
||||
use crate::{
|
||||
anthropic::messages::handler::shape_anthropic_messages_request,
|
||||
base_llm::messages::context::{MessagesShapeOptions, MessagesTransformContext},
|
||||
base_llm::messages::context::{
|
||||
MessagesShapeOptions, MessagesTransformContext, NativeOutputSupport,
|
||||
},
|
||||
};
|
||||
use futures_util::StreamExt;
|
||||
use litellm_auth::AwsParams;
|
||||
use litellm_auth::CredentialPlacement;
|
||||
use litellm_secret_value::SecretValue;
|
||||
|
||||
use litellm_auth_aws::{
|
||||
AwsCredentialSource, bedrock_model_id_and_region,
|
||||
AwsCredentialSource, CallerSigningHeaders, bedrock_runtime_endpoint,
|
||||
constants::{
|
||||
AWS_BEARER_TOKEN_BEDROCK, AWS_BEDROCK_RUNTIME_ENDPOINT, AWS_DEFAULT_REGION, AWS_REGION,
|
||||
AWS_REGION_NAME, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE,
|
||||
AWS_REGION_NAME, BEDROCK_SERVICE,
|
||||
},
|
||||
resolve_bedrock_region,
|
||||
};
|
||||
use litellm_llms_types::formats::messages::{
|
||||
MessagesOptionalParams, MessagesRequest,
|
||||
streaming::{MessagesStreamEvent, MessagesStreamUsage},
|
||||
};
|
||||
use litellm_llms_types::providers::bedrock::{
|
||||
INVOCATION_METRICS_KEY, INVOKE_PATH, INVOKE_STREAM_PATH,
|
||||
};
|
||||
use litellm_llms_types::formats::messages::{MessagesOptionalParams, MessagesRequest};
|
||||
use litellm_llms_types::providers::bedrock::{INVOKE_PATH, INVOKE_STREAM_PATH};
|
||||
use litellm_router_types::LitellmParams;
|
||||
use serde_json::{Map, Value};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
Error,
|
||||
base_llm::{
|
||||
auth::AuthScheme,
|
||||
base_model_iterator::{StreamError, StreamTransformer, transform_stream},
|
||||
messages::{
|
||||
streaming::{ByteStream, EventStream, StreamDecoder},
|
||||
streaming::StreamDecoder,
|
||||
transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment},
|
||||
},
|
||||
},
|
||||
bedrock::chat::invoke_handler::{decode_invoke_anthropic_chunk, invoke_chunk_stream},
|
||||
};
|
||||
|
||||
const METRICS_USAGE_KEYS: [(&str, &str); 4] = [
|
||||
("input_tokens", "inputTokenCount"),
|
||||
("output_tokens", "outputTokenCount"),
|
||||
("cache_read_input_tokens", "cacheReadInputTokenCount"),
|
||||
("cache_creation_input_tokens", "cacheWriteInputTokenCount"),
|
||||
];
|
||||
use crate::bedrock::{
|
||||
chat::invoke_transformations::base_invoke_transformation::{
|
||||
GUARDRAIL_CONFIG_PARAM, bedrock_invoke_guardrail_headers,
|
||||
},
|
||||
common_utils::{
|
||||
strip_bedrock_routing_prefix,
|
||||
structured_output::{
|
||||
NATIVE_OUTPUT_UNSUPPORTED, bedrock_native_output_format,
|
||||
bedrock_native_output_retry_requested, is_bedrock_native_output_rejection,
|
||||
requested_output_format, sent_output_config_format,
|
||||
},
|
||||
validated_region,
|
||||
},
|
||||
request_metadata::{bedrock_request_metadata_headers, merge_bedrock_invoke_headers},
|
||||
};
|
||||
|
||||
use super::{request, stream};
|
||||
|
||||
const INVOKE_MODEL_PREFIX: &str = "invoke/";
|
||||
|
||||
const MODEL_ID_PARAM: &str = "model_id";
|
||||
|
||||
const MODEL_SPEC_PREFIXES: [&str; 2] = ["nova-2/", "nova/"];
|
||||
|
||||
const SECRET_NAMES: &[&str] = &[
|
||||
AWS_BEARER_TOKEN_BEDROCK,
|
||||
AWS_BEDROCK_RUNTIME_ENDPOINT,
|
||||
|
|
@ -76,16 +82,89 @@ fn bearer_token(
|
|||
.filter(|token| !token.is_empty())
|
||||
}
|
||||
|
||||
/// Port of Python's `BaseAWSLLM.encode_model_id`.
|
||||
pub fn encode_model_id(text: &str) -> String {
|
||||
text.bytes()
|
||||
.map(|byte| match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' => {
|
||||
char::from(byte).to_string()
|
||||
}
|
||||
other => format!("%{other:02X}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Port of Python's `BaseAWSLLM.get_bedrock_model_id` for Claude.
|
||||
fn get_bedrock_model_id(model: &str, model_id: Option<&str>) -> String {
|
||||
if let Some(model_id) = model_id {
|
||||
return encode_model_id(model_id).replacen(INVOKE_MODEL_PREFIX, "", 1);
|
||||
}
|
||||
let stripped = strip_bedrock_routing_prefix(model);
|
||||
if stripped.starts_with("arn:") {
|
||||
return encode_model_id(stripped);
|
||||
}
|
||||
let model_id = stripped.replacen(INVOKE_MODEL_PREFIX, "", 1);
|
||||
match MODEL_SPEC_PREFIXES
|
||||
.iter()
|
||||
.find(|spec| model_id.contains(**spec))
|
||||
{
|
||||
Some(spec) => encode_model_id(&model_id.replace(spec, "")),
|
||||
None => model_id,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_arn_partition_char(character: char) -> bool {
|
||||
character.is_ascii_lowercase() || character.is_ascii_digit() || character == '-'
|
||||
}
|
||||
|
||||
/// Port of Python's `contains_bedrock_arn`.
|
||||
fn contains_bedrock_arn(model: &str) -> bool {
|
||||
model.match_indices("arn:aws").any(|(index, prefix)| {
|
||||
let rest = &model[index + prefix.len()..];
|
||||
let after_partition = match rest.strip_prefix('-') {
|
||||
Some(partition) => {
|
||||
let trimmed = partition.trim_start_matches(is_arn_partition_char);
|
||||
if trimmed.len() == partition.len() {
|
||||
return false;
|
||||
}
|
||||
trimmed
|
||||
}
|
||||
None => rest,
|
||||
};
|
||||
after_partition.starts_with(":bedrock")
|
||||
})
|
||||
}
|
||||
|
||||
/// Port of Python's `BaseAWSLLM._get_aws_region_from_model_arn`.
|
||||
fn get_aws_region_from_model_arn(model: &str) -> Option<String> {
|
||||
contains_bedrock_arn(model)
|
||||
.then(|| model.split(':').nth(3))
|
||||
.flatten()
|
||||
.filter(|region| !region.is_empty() && region.chars().all(is_arn_partition_char))
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// Port of Python's `AmazonInvokeConfig.get_complete_url`.
|
||||
fn invoke_url(
|
||||
api_base: Option<&str>,
|
||||
model: &str,
|
||||
aws: &AwsParams,
|
||||
litellm_params: &LitellmParams,
|
||||
stream: bool,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> String {
|
||||
let (model_id, model_region) =
|
||||
bedrock_model_id_and_region(model.strip_prefix(INVOKE_MODEL_PREFIX).unwrap_or(model));
|
||||
let region = resolve_bedrock_region(model_region.as_deref(), aws, env_lookup);
|
||||
) -> Result<String, Error> {
|
||||
let aws = &litellm_params.aws;
|
||||
let model_id = get_bedrock_model_id(
|
||||
model,
|
||||
litellm_params
|
||||
.extra
|
||||
.get(MODEL_ID_PARAM)
|
||||
.and_then(Value::as_str),
|
||||
);
|
||||
let region = resolve_bedrock_region(
|
||||
get_aws_region_from_model_arn(model).as_deref(),
|
||||
aws,
|
||||
env_lookup,
|
||||
);
|
||||
let path = if stream {
|
||||
INVOKE_STREAM_PATH
|
||||
} else {
|
||||
|
|
@ -97,11 +176,41 @@ fn invoke_url(
|
|||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
};
|
||||
let endpoint = configured(api_base)
|
||||
let endpoint = match configured(api_base)
|
||||
.or_else(|| configured(aws.aws_bedrock_runtime_endpoint.as_deref()))
|
||||
.or_else(|| env_lookup(AWS_BEDROCK_RUNTIME_ENDPOINT))
|
||||
.unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion));
|
||||
format!("{}/model/{model_id}/{path}", endpoint.trim_end_matches('/'))
|
||||
{
|
||||
Some(endpoint) => endpoint,
|
||||
None => bedrock_runtime_endpoint(&validated_region(region)?),
|
||||
};
|
||||
Ok(format!(
|
||||
"{}/model/{model_id}/{path}",
|
||||
endpoint.trim_end_matches('/')
|
||||
))
|
||||
}
|
||||
|
||||
/// The header half of Python's `validate_anthropic_messages_environment`.
|
||||
fn invoke_headers(headers: Headers, litellm_params: &LitellmParams) -> Result<Headers, Error> {
|
||||
let guardrail = match litellm_params.extra.get(GUARDRAIL_CONFIG_PARAM) {
|
||||
None | Some(Value::Null) => Vec::new(),
|
||||
Some(raw) => bedrock_invoke_guardrail_headers(raw)?,
|
||||
};
|
||||
Ok(merge_bedrock_invoke_headers(
|
||||
headers,
|
||||
guardrail,
|
||||
bedrock_request_metadata_headers(&litellm_params.extra),
|
||||
))
|
||||
}
|
||||
|
||||
fn without_native_output(context: &MessagesTransformContext) -> MessagesTransformContext {
|
||||
let mut fallback = *context;
|
||||
fallback.thinking.capabilities.bedrock.native_output = NativeOutputSupport {
|
||||
enabled: false,
|
||||
opt_in: false,
|
||||
replaces_unforceable_tool: false,
|
||||
..fallback.thinking.capabilities.bedrock.native_output
|
||||
};
|
||||
fallback
|
||||
}
|
||||
|
||||
impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
|
||||
|
|
@ -131,23 +240,72 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
|
|||
stream: bool,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
Ok(invoke_url(
|
||||
api_base,
|
||||
model,
|
||||
&litellm_params.aws,
|
||||
stream,
|
||||
env_lookup,
|
||||
))
|
||||
invoke_url(api_base, model, litellm_params, stream, env_lookup)
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
/// Python's `transform_anthropic_messages_request`, which picks the betas from the caller's
|
||||
/// header and the request and sends them in the body.
|
||||
fn finalize_request(
|
||||
&self,
|
||||
_request: MessagesRequest,
|
||||
_context: &MessagesTransformContext,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
Err(Error::Declined(Decline::new(
|
||||
"Bedrock invoke messages request shaping",
|
||||
)))
|
||||
request: MessagesRequest,
|
||||
headers: Headers,
|
||||
context: &MessagesTransformContext,
|
||||
) -> Result<(MessagesRequest, Headers), Error> {
|
||||
request::transform_anthropic_messages_request(request, context, headers)
|
||||
}
|
||||
|
||||
/// Port of Python's `native_output_fallback_litellm_params`.
|
||||
fn native_output_fallback(
|
||||
&self,
|
||||
request: &MessagesRequest,
|
||||
context: &MessagesTransformContext,
|
||||
) -> Option<MessagesTransformContext> {
|
||||
let native = context.thinking.capabilities.bedrock.native_output;
|
||||
let Ok(Value::Object(params)) = serde_json::to_value(&request.params) else {
|
||||
return None;
|
||||
};
|
||||
if !bedrock_native_output_retry_requested(¶ms, native) {
|
||||
return None;
|
||||
}
|
||||
let output_format = requested_output_format(¶ms)?;
|
||||
bedrock_native_output_format(output_format, native)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|_| without_native_output(context))
|
||||
}
|
||||
|
||||
/// Port of Python's `should_retry_anthropic_messages_without_native_output`.
|
||||
fn retries_without_native_output(
|
||||
&self,
|
||||
status: u16,
|
||||
error_text: &str,
|
||||
request_body: &Value,
|
||||
model: &str,
|
||||
) -> bool {
|
||||
let retries = sent_output_config_format(request_body)
|
||||
&& is_bedrock_native_output_rejection(status, error_text);
|
||||
if retries {
|
||||
let opt_out_advice = if error_text.contains(NATIVE_OUTPUT_UNSUPPORTED) {
|
||||
" Set `supports_bedrock_native_structured_output: false` in this deployment's model_info to skip the rejected call."
|
||||
} else {
|
||||
""
|
||||
};
|
||||
litellm_tracing::warn!(
|
||||
"Bedrock rejected native structured output for model={model}, so litellm sent the request again without it.{opt_out_advice}"
|
||||
);
|
||||
}
|
||||
retries
|
||||
}
|
||||
|
||||
fn wire_body(&self, body: Value) -> Value {
|
||||
match body {
|
||||
Value::Object(mut fields) => {
|
||||
fields.shift_remove("model");
|
||||
fields.shift_remove("stream");
|
||||
Value::Object(fields)
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> &'static [&'static str] {
|
||||
|
|
@ -165,6 +323,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
|
|||
litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
let headers = invoke_headers(headers, litellm_params)?;
|
||||
if let Some(token) = bearer_token(api_key, env_lookup) {
|
||||
return Ok(ValidatedEnvironment {
|
||||
headers,
|
||||
|
|
@ -174,15 +333,18 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
|
|||
},
|
||||
});
|
||||
}
|
||||
let (_, model_region) =
|
||||
bedrock_model_id_and_region(model.strip_prefix(INVOKE_MODEL_PREFIX).unwrap_or(model));
|
||||
let params = &litellm_params.aws;
|
||||
Ok(ValidatedEnvironment {
|
||||
headers,
|
||||
auth: AuthScheme::AwsSigV4 {
|
||||
region: resolve_bedrock_region(model_region.as_deref(), params, env_lookup),
|
||||
region: resolve_bedrock_region(
|
||||
get_aws_region_from_model_arn(model).as_deref(),
|
||||
params,
|
||||
env_lookup,
|
||||
),
|
||||
service: BEDROCK_SERVICE,
|
||||
credentials: Box::new(AwsCredentialSource::from_params(params, env_lookup)),
|
||||
caller_headers: CallerSigningHeaders::Botocore,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -191,343 +353,20 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
|
|||
&[("content-type", "application/json")]
|
||||
}
|
||||
|
||||
/// Port of Python's `get_async_streaming_response_iterator`.
|
||||
fn stream_decoder(&self) -> Option<StreamDecoder> {
|
||||
Some(bedrock_anthropic_messages_event_stream)
|
||||
}
|
||||
}
|
||||
|
||||
fn with_invocation_usage(chunk: Value) -> Value {
|
||||
match chunk {
|
||||
Value::Object(fields) => Value::Object(with_metrics_usage(fields)),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_metrics_usage(mut fields: Map<String, Value>) -> Map<String, Value> {
|
||||
let Some(Value::Object(metrics)) = fields.remove(INVOCATION_METRICS_KEY) else {
|
||||
return fields;
|
||||
};
|
||||
if metrics.is_empty() {
|
||||
return fields;
|
||||
}
|
||||
let preserved = match fields.remove("usage") {
|
||||
Some(Value::Object(usage)) => usage,
|
||||
_ => Map::new(),
|
||||
};
|
||||
let usage: Map<String, Value> = METRICS_USAGE_KEYS
|
||||
.iter()
|
||||
.filter_map(|(anthropic, metric)| {
|
||||
Some((anthropic.to_string(), metrics.get(*metric)?.clone()))
|
||||
})
|
||||
.chain(preserved)
|
||||
.collect();
|
||||
fields.insert("usage".to_string(), Value::Object(usage));
|
||||
fields
|
||||
}
|
||||
|
||||
pub fn bedrock_anthropic_messages_event_stream(bytes: ByteStream) -> EventStream {
|
||||
let events = invoke_chunk_stream(bytes)
|
||||
.map(|chunk| decode_invoke_anthropic_chunk(with_invocation_usage(chunk?)));
|
||||
Box::pin(
|
||||
transform_stream(events, MessageStopUsagePromoter::default())
|
||||
.map(|item| item.map_err(StreamError::into_decode)),
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MessageStopUsagePromoter {
|
||||
pending_delta: Option<MessagesStreamEvent>,
|
||||
start_usage: Option<MessagesStreamUsage>,
|
||||
}
|
||||
|
||||
fn promoted_usage(
|
||||
delta: Option<MessagesStreamUsage>,
|
||||
stop: Option<&MessagesStreamUsage>,
|
||||
start: Option<&MessagesStreamUsage>,
|
||||
) -> Option<MessagesStreamUsage> {
|
||||
let delta = delta.unwrap_or_default();
|
||||
let merged = MessagesStreamUsage {
|
||||
input_tokens: stop
|
||||
.and_then(|stop| stop.input_tokens)
|
||||
.or(delta.input_tokens),
|
||||
cache_creation_input_tokens: stop
|
||||
.and_then(|stop| stop.cache_creation_input_tokens)
|
||||
.or(delta.cache_creation_input_tokens)
|
||||
.or_else(|| start.and_then(|start| start.cache_creation_input_tokens)),
|
||||
cache_read_input_tokens: stop
|
||||
.and_then(|stop| stop.cache_read_input_tokens)
|
||||
.or(delta.cache_read_input_tokens)
|
||||
.or_else(|| start.and_then(|start| start.cache_read_input_tokens)),
|
||||
extra: delta
|
||||
.extra
|
||||
.into_iter()
|
||||
.chain(
|
||||
start
|
||||
.and_then(|start| start.extra.get_key_value("cache_creation"))
|
||||
.map(|(key, value)| (key.clone(), value.clone())),
|
||||
)
|
||||
.fold(Map::new(), |mut extra, (key, value)| {
|
||||
extra.entry(key).or_insert(value);
|
||||
extra
|
||||
}),
|
||||
..delta
|
||||
};
|
||||
(merged != MessagesStreamUsage::default()).then_some(merged)
|
||||
}
|
||||
|
||||
fn promoted(
|
||||
event: MessagesStreamEvent,
|
||||
stop: Option<&MessagesStreamUsage>,
|
||||
start: Option<&MessagesStreamUsage>,
|
||||
) -> MessagesStreamEvent {
|
||||
match event {
|
||||
MessagesStreamEvent::MessageDelta {
|
||||
delta,
|
||||
usage,
|
||||
context_management,
|
||||
} => MessagesStreamEvent::MessageDelta {
|
||||
delta,
|
||||
usage: promoted_usage(usage, stop, start),
|
||||
context_management,
|
||||
},
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamTransformer for MessageStopUsagePromoter {
|
||||
type Input = MessagesStreamEvent;
|
||||
type Output = MessagesStreamEvent;
|
||||
type Error = Infallible;
|
||||
|
||||
fn transform(
|
||||
&mut self,
|
||||
input: MessagesStreamEvent,
|
||||
) -> Result<Vec<MessagesStreamEvent>, Infallible> {
|
||||
let pending = self.pending_delta.take();
|
||||
match input {
|
||||
MessagesStreamEvent::MessageDelta { .. } => {
|
||||
self.pending_delta = Some(input);
|
||||
Ok(pending.into_iter().collect())
|
||||
}
|
||||
MessagesStreamEvent::MessageStop { usage } => Ok(pending
|
||||
.map(|delta| promoted(delta, usage.as_ref(), self.start_usage.as_ref()))
|
||||
.into_iter()
|
||||
.chain([MessagesStreamEvent::MessageStop { usage }])
|
||||
.collect()),
|
||||
MessagesStreamEvent::MessageStart { message } => {
|
||||
self.start_usage = Some(message.usage.clone());
|
||||
Ok(pending
|
||||
.into_iter()
|
||||
.chain([MessagesStreamEvent::MessageStart { message }])
|
||||
.collect())
|
||||
}
|
||||
other => Ok(pending.into_iter().chain([other]).collect()),
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<Vec<MessagesStreamEvent>, Infallible> {
|
||||
Ok(self
|
||||
.pending_delta
|
||||
.take()
|
||||
.map(|delta| promoted(delta, None, self.start_usage.as_ref()))
|
||||
.into_iter()
|
||||
.collect())
|
||||
Some(stream::get_async_streaming_response_iterator)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use aws_smithy_eventstream::frame::write_message_to;
|
||||
use aws_smithy_types::event_stream::{Header, HeaderValue, Message};
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use futures_util::TryStreamExt;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use litellm_auth::AwsParams;
|
||||
use litellm_auth_aws::constants::DEFAULT_BEDROCK_REGION;
|
||||
|
||||
use super::*;
|
||||
use crate::base_llm::messages::streaming;
|
||||
|
||||
fn event(value: Value) -> MessagesStreamEvent {
|
||||
serde_json::from_value(value).unwrap()
|
||||
}
|
||||
|
||||
fn message_start(usage: Value) -> MessagesStreamEvent {
|
||||
event(json!({
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
|
||||
"content": [], "stop_reason": null, "stop_sequence": null, "usage": usage
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
fn message_delta(usage: Value) -> MessagesStreamEvent {
|
||||
event(json!({
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn"},
|
||||
"usage": usage
|
||||
}))
|
||||
}
|
||||
|
||||
fn message_stop(usage: Option<Value>) -> MessagesStreamEvent {
|
||||
match usage {
|
||||
Some(usage) => event(json!({"type": "message_stop", "usage": usage})),
|
||||
None => event(json!({"type": "message_stop"})),
|
||||
}
|
||||
}
|
||||
|
||||
fn promote(events: Vec<MessagesStreamEvent>) -> Vec<MessagesStreamEvent> {
|
||||
let mut promoter = MessageStopUsagePromoter::default();
|
||||
let mut output: Vec<_> = events
|
||||
.into_iter()
|
||||
.flat_map(|event| promoter.transform(event).unwrap())
|
||||
.collect();
|
||||
output.extend(promoter.finish().unwrap());
|
||||
output
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::cache_fields_on_message_stop(
|
||||
json!({"input_tokens": 10, "output_tokens": 0}),
|
||||
json!({"output_tokens": 5}),
|
||||
Some(json!({"input_tokens": 3, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20})),
|
||||
json!({"input_tokens": 3, "output_tokens": 5, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20}),
|
||||
)]
|
||||
#[case::cache_only_on_message_start(
|
||||
json!({"input_tokens": 10, "output_tokens": 0, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}),
|
||||
json!({"output_tokens": 5}),
|
||||
Some(json!({"input_tokens": 10})),
|
||||
json!({"input_tokens": 10, "output_tokens": 5, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}),
|
||||
)]
|
||||
#[case::message_stop_wins_over_message_start(
|
||||
json!({"input_tokens": 10, "cache_read_input_tokens": 80}),
|
||||
json!({"output_tokens": 5}),
|
||||
Some(json!({"cache_read_input_tokens": 100})),
|
||||
json!({"output_tokens": 5, "cache_read_input_tokens": 100}),
|
||||
)]
|
||||
#[case::delta_cache_fields_are_kept(
|
||||
json!({"input_tokens": 10, "cache_read_input_tokens": 80}),
|
||||
json!({"output_tokens": 5, "cache_read_input_tokens": 7}),
|
||||
None,
|
||||
json!({"output_tokens": 5, "cache_read_input_tokens": 7}),
|
||||
)]
|
||||
fn message_delta_usage_is_completed_from_stop_then_start(
|
||||
#[case] start: Value,
|
||||
#[case] delta: Value,
|
||||
#[case] stop: Option<Value>,
|
||||
#[case] expected: Value,
|
||||
) {
|
||||
let output = promote(vec![
|
||||
message_start(start),
|
||||
message_delta(delta),
|
||||
message_stop(stop.clone()),
|
||||
]);
|
||||
|
||||
assert_eq!(output.len(), 3);
|
||||
assert_eq!(output[1], message_delta(expected));
|
||||
assert_eq!(output[2], message_stop(stop));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_delta_is_flushed_with_start_usage_when_the_stream_ends_without_a_stop() {
|
||||
let output = promote(vec![
|
||||
message_start(json!({"input_tokens": 10, "cache_read_input_tokens": 80})),
|
||||
message_delta(json!({"output_tokens": 5})),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
output[1],
|
||||
message_delta(json!({"output_tokens": 5, "cache_read_input_tokens": 80}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn events_around_the_delta_keep_their_order() {
|
||||
let ping = event(json!({"type": "ping"}));
|
||||
let output = promote(vec![
|
||||
message_delta(json!({"output_tokens": 5})),
|
||||
ping.clone(),
|
||||
message_stop(None),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
output,
|
||||
vec![
|
||||
message_delta(json!({"output_tokens": 5})),
|
||||
ping,
|
||||
message_stop(None)
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::metrics_fill_missing_usage(
|
||||
json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "outputTokenCount": 9}}),
|
||||
json!({"type": "message_stop", "usage": {"input_tokens": 3, "output_tokens": 9}}),
|
||||
)]
|
||||
#[case::the_chunks_own_usage_wins(
|
||||
json!({"type": "message_stop", "usage": {"input_tokens": 1}, "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}}),
|
||||
json!({"type": "message_stop", "usage": {"cache_read_input_tokens": 40, "input_tokens": 1}}),
|
||||
)]
|
||||
#[case::no_metrics_leaves_the_chunk(
|
||||
json!({"type": "message_stop"}),
|
||||
json!({"type": "message_stop"}),
|
||||
)]
|
||||
#[case::empty_metrics_are_dropped(
|
||||
json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {}}),
|
||||
json!({"type": "message_stop"}),
|
||||
)]
|
||||
fn invocation_metrics_become_anthropic_usage(#[case] chunk: Value, #[case] expected: Value) {
|
||||
assert_eq!(with_invocation_usage(chunk), expected);
|
||||
}
|
||||
|
||||
fn aws_frame(chunk: &Value) -> Vec<u8> {
|
||||
let payload = json!({"bytes": STANDARD.encode(chunk.to_string())});
|
||||
let message = Message::new(Bytes::from(serde_json::to_vec(&payload).unwrap())).add_header(
|
||||
Header::new(":event-type", HeaderValue::String("chunk".into())),
|
||||
);
|
||||
let mut wire = Vec::new();
|
||||
write_message_to(&message, &mut wire).unwrap();
|
||||
wire
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bedrock_stream_yields_the_sse_an_anthropic_client_reads() {
|
||||
let chunks = [
|
||||
json!({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}}),
|
||||
json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}}),
|
||||
];
|
||||
let wire: Vec<u8> = chunks.iter().flat_map(aws_frame).collect();
|
||||
let bytes: ByteStream = futures_util::stream::iter(
|
||||
wire.chunks(7)
|
||||
.map(|chunk| Ok(Bytes::copy_from_slice(chunk)))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.boxed();
|
||||
|
||||
let sse = bedrock_anthropic_messages_event_stream(bytes)
|
||||
.map_ok(|event| streaming::encode(&event).unwrap())
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap()
|
||||
.concat();
|
||||
|
||||
let expected: Vec<u8> = [
|
||||
message_delta(
|
||||
json!({"output_tokens": 5, "cache_read_input_tokens": 40, "input_tokens": 3}),
|
||||
),
|
||||
message_stop(Some(
|
||||
json!({"input_tokens": 3, "cache_read_input_tokens": 40}),
|
||||
)),
|
||||
]
|
||||
.iter()
|
||||
.flat_map(|event| streaming::encode(event).unwrap())
|
||||
.collect();
|
||||
assert_eq!(sse, expected);
|
||||
}
|
||||
|
||||
fn in_region(region: &str) -> LitellmParams {
|
||||
LitellmParams {
|
||||
|
|
@ -619,6 +458,37 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::host_injection("evil.example.com/x?")]
|
||||
#[case::uppercase("US-WEST-2")]
|
||||
#[case::port("us-west-2:8443")]
|
||||
fn a_region_outside_the_aws_alphabet_never_builds_a_url(#[case] region: &str) {
|
||||
let url = AmazonAnthropicClaudeMessagesConfig.get_complete_url(
|
||||
None,
|
||||
"anthropic.claude-3",
|
||||
&in_region(region),
|
||||
false,
|
||||
&|_| None,
|
||||
);
|
||||
assert!(matches!(url, Err(Error::InvalidRequest(_))), "{url:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_configured_endpoint_does_not_read_the_region() {
|
||||
assert_eq!(
|
||||
AmazonAnthropicClaudeMessagesConfig
|
||||
.get_complete_url(
|
||||
Some("https://base.test"),
|
||||
"anthropic.claude-3",
|
||||
&in_region("evil.example.com/x?"),
|
||||
false,
|
||||
&|_| None,
|
||||
)
|
||||
.unwrap(),
|
||||
"https://base.test/model/anthropic.claude-3/invoke"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sigv4_scope_and_credentials_come_from_the_litellm_params() {
|
||||
let litellm_params = LitellmParams {
|
||||
|
|
@ -702,4 +572,276 @@ mod tests {
|
|||
(other, _) => panic!("unexpected auth {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
const ARN: &str = "arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0";
|
||||
const ENCODED_ARN: &str = "arn%3Aaws%3Abedrock%3Aus-east-1%3A086734376398%3Ainference-profile%2Fglobal.anthropic.claude-sonnet-4-5-20250929-v1%3A0";
|
||||
|
||||
#[rstest]
|
||||
#[case::bedrock_prefixed_arn(
|
||||
"bedrock/arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
None,
|
||||
ENCODED_ARN
|
||||
)]
|
||||
#[case::compound_prefixed_arn(
|
||||
"bedrock/invoke/arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
None,
|
||||
ENCODED_ARN
|
||||
)]
|
||||
#[case::bare_arn(ARN, None, ENCODED_ARN)]
|
||||
#[case::regular_id(
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
None,
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0"
|
||||
)]
|
||||
#[case::invoke_prefix(
|
||||
"invoke/anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
None,
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0"
|
||||
)]
|
||||
#[case::model_id_override(
|
||||
"anthropic.claude-3",
|
||||
Some("arn:aws:bedrock:us-east-1:1:application-inference-profile/abc"),
|
||||
"arn%3Aaws%3Abedrock%3Aus-east-1%3A1%3Aapplication-inference-profile%2Fabc"
|
||||
)]
|
||||
fn invoke_model_ids_match_python(
|
||||
#[case] model: &str,
|
||||
#[case] model_id: Option<&str>,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(get_bedrock_model_id(model, model_id), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::region_segment_stays_in_the_path(
|
||||
"bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
None,
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke",
|
||||
"us-west-2"
|
||||
)]
|
||||
#[case::region_segment_does_not_pick_the_region(
|
||||
"invoke/eu-central-1/anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
None,
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/eu-central-1/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke",
|
||||
"us-west-2"
|
||||
)]
|
||||
#[case::params_region_hosts_a_region_segment(
|
||||
"invoke/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
Some("us-east-1"),
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke",
|
||||
"us-east-1"
|
||||
)]
|
||||
#[case::arn_after_a_region_segment_picks_the_region(
|
||||
"invoke/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
None,
|
||||
"https://bedrock-runtime.eu-west-1.amazonaws.com/model/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke",
|
||||
"eu-west-1"
|
||||
)]
|
||||
#[case::arn_after_any_segment_picks_the_region(
|
||||
"invoke/foo/arn:aws:bedrock:eu-west-1:1:x",
|
||||
None,
|
||||
"https://bedrock-runtime.eu-west-1.amazonaws.com/model/foo/arn:aws:bedrock:eu-west-1:1:x/invoke",
|
||||
"eu-west-1"
|
||||
)]
|
||||
#[case::partitioned_arn(
|
||||
"invoke/arn:aws-us-gov:bedrock:us-gov-west-1:1:inference-profile/x",
|
||||
None,
|
||||
"https://bedrock-runtime.us-gov-west-1.amazonaws.com/model/arn%3Aaws-us-gov%3Abedrock%3Aus-gov-west-1%3A1%3Ainference-profile%2Fx/invoke",
|
||||
"us-gov-west-1"
|
||||
)]
|
||||
#[case::empty_partition_is_not_an_arn(
|
||||
"invoke/arn:aws-:bedrock:eu-west-1:1:x",
|
||||
None,
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws-%3Abedrock%3Aeu-west-1%3A1%3Ax/invoke",
|
||||
"us-west-2"
|
||||
)]
|
||||
#[case::uppercase_arn_region_is_ignored(
|
||||
"invoke/arn:aws:bedrock:EU:1:x",
|
||||
None,
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3AEU%3A1%3Ax/invoke",
|
||||
"us-west-2"
|
||||
)]
|
||||
#[case::nova_spec_is_removed_and_the_id_encoded(
|
||||
"invoke/us-west-2/nova/anthropic.claude-3",
|
||||
None,
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2%2Fanthropic.claude-3/invoke",
|
||||
"us-west-2"
|
||||
)]
|
||||
#[case::one_inner_invoke_segment_is_removed(
|
||||
"invoke/us-west-2/invoke/anthropic.claude-3",
|
||||
None,
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2/anthropic.claude-3/invoke",
|
||||
"us-west-2"
|
||||
)]
|
||||
fn url_and_signing_region_match_python_for_every_model_shape(
|
||||
#[case] model: &str,
|
||||
#[case] region_param: Option<&str>,
|
||||
#[case] expected_url: &str,
|
||||
#[case] expected_region: &str,
|
||||
) {
|
||||
let litellm_params = region_param.map_or_else(LitellmParams::default, in_region);
|
||||
let url = AmazonAnthropicClaudeMessagesConfig
|
||||
.get_complete_url(None, model, &litellm_params, false, &|_| None)
|
||||
.unwrap();
|
||||
let validated = AmazonAnthropicClaudeMessagesConfig
|
||||
.validate_environment(
|
||||
Vec::new(),
|
||||
None,
|
||||
model,
|
||||
&MessagesOptionalParams::default(),
|
||||
&litellm_params,
|
||||
&|_| None,
|
||||
)
|
||||
.unwrap();
|
||||
let AuthScheme::AwsSigV4 {
|
||||
region,
|
||||
caller_headers,
|
||||
..
|
||||
} = validated.auth
|
||||
else {
|
||||
panic!("expected SigV4, got {:?}", validated.auth);
|
||||
};
|
||||
assert_eq!(
|
||||
(url.as_str(), region.as_str(), caller_headers),
|
||||
(
|
||||
expected_url,
|
||||
expected_region,
|
||||
CallerSigningHeaders::Botocore
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_arn_url_is_percent_encoded_in_the_arn_region() {
|
||||
assert_eq!(
|
||||
AmazonAnthropicClaudeMessagesConfig
|
||||
.get_complete_url(
|
||||
None,
|
||||
&format!("bedrock/{ARN}"),
|
||||
&LitellmParams::default(),
|
||||
true,
|
||||
&|_| None
|
||||
)
|
||||
.unwrap(),
|
||||
format!(
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com/model/{ENCODED_ARN}/invoke-with-response-stream"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_model_id_param_overrides_the_url_model() {
|
||||
let litellm_params: LitellmParams = serde_json::from_value(serde_json::json!({
|
||||
"model": "bedrock/anthropic.claude-3",
|
||||
"aws_region_name": "us-west-2",
|
||||
"model_id": "my/profile",
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
AmazonAnthropicClaudeMessagesConfig
|
||||
.get_complete_url(None, "anthropic.claude-3", &litellm_params, false, &|_| {
|
||||
None
|
||||
})
|
||||
.unwrap(),
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/my%2Fprofile/invoke"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guardrail_and_metadata_headers_are_added_for_both_auth_schemes() {
|
||||
let litellm_params: LitellmParams = serde_json::from_value(serde_json::json!({
|
||||
"model": "bedrock/anthropic.claude-3",
|
||||
"guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "2", "trace": "enabled"},
|
||||
}))
|
||||
.unwrap();
|
||||
let validated = AmazonAnthropicClaudeMessagesConfig
|
||||
.validate_environment(
|
||||
Vec::new(),
|
||||
Some("token"),
|
||||
"anthropic.claude-3",
|
||||
&MessagesOptionalParams::default(),
|
||||
&litellm_params,
|
||||
&|_| None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
validated.headers,
|
||||
vec![
|
||||
(
|
||||
"X-Amzn-Bedrock-GuardrailIdentifier".to_string(),
|
||||
"gid".to_string()
|
||||
),
|
||||
(
|
||||
"X-Amzn-Bedrock-GuardrailVersion".to_string(),
|
||||
"2".to_string()
|
||||
),
|
||||
("X-Amzn-Bedrock-Trace".to_string(), "ENABLED".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
fn native_context(native_output: NativeOutputSupport) -> MessagesTransformContext {
|
||||
let mut context = MessagesTransformContext::default();
|
||||
context.thinking.capabilities.bedrock.native_output = native_output;
|
||||
context
|
||||
}
|
||||
|
||||
fn structured_request() -> MessagesRequest {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"model": "anthropic.claude-sonnet-5",
|
||||
"max_tokens": 10,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"output_config": {"format": {"type": "json_schema", "schema": {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}}},
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::native_enabled(NativeOutputSupport { enabled: true, mapped: true, ..NativeOutputSupport::default() }, true)]
|
||||
#[case::native_opt_in(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, true)]
|
||||
#[case::inline_only(NativeOutputSupport::default(), false)]
|
||||
fn a_native_output_request_carries_an_inline_fallback(
|
||||
#[case] native_output: NativeOutputSupport,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
let fallback = AmazonAnthropicClaudeMessagesConfig
|
||||
.native_output_fallback(&structured_request(), &native_context(native_output));
|
||||
assert_eq!(fallback.is_some(), expected);
|
||||
if let Some(fallback) = fallback {
|
||||
let (rebuilt, _) = AmazonAnthropicClaudeMessagesConfig
|
||||
.finalize_request(structured_request(), Vec::new(), &fallback)
|
||||
.unwrap();
|
||||
let body = serde_json::to_value(rebuilt).unwrap();
|
||||
assert_eq!(body.get("output_config"), None);
|
||||
assert_eq!(
|
||||
body["messages"][0]["content"][1]["type"],
|
||||
serde_json::json!("text")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::rejected_native_output(400, "output_config.format: Extra inputs are not permitted", serde_json::json!({"output_config": {"format": {}}}), true)]
|
||||
#[case::inline_body_is_not_retried(400, "output_config.format: Extra inputs are not permitted", serde_json::json!({"messages": []}), false)]
|
||||
#[case::other_error(429, "Too many requests", serde_json::json!({"output_config": {"format": {}}}), false)]
|
||||
fn native_output_rejections_retry_without_it(
|
||||
#[case] status: u16,
|
||||
#[case] text: &str,
|
||||
#[case] body: Value,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
assert_eq!(
|
||||
AmazonAnthropicClaudeMessagesConfig
|
||||
.retries_without_native_output(status, text, &body, "m"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_wire_body_drops_model_and_stream() {
|
||||
assert_eq!(
|
||||
AmazonAnthropicClaudeMessagesConfig
|
||||
.wire_body(serde_json::json!({"model": "m", "stream": true, "max_tokens": 1})),
|
||||
serde_json::json!({"max_tokens": 1})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,702 @@
|
|||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_llms_types::{
|
||||
providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet},
|
||||
recognized::python_truthy,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
Error, ErrorDetail,
|
||||
anthropic::beta_headers::BetaPolicy,
|
||||
base_llm::{auth::Headers, messages::context::MessagesModelCapabilities},
|
||||
};
|
||||
|
||||
use super::request::Body;
|
||||
|
||||
const PER_TURN_CONTROL: &str = "per-turn-control-2026-07-01";
|
||||
const DANGEROUS_TOOL_USE: &str = "dangerous-tool-use-2026-09-03";
|
||||
const TOOL_SEARCH_TOOL: &str = "tool-search-tool-2025-10-19";
|
||||
const ADVANCED_TOOL_USE: &str = "advanced-tool-use-2025-11-20";
|
||||
const TOOL_EXAMPLES: &str = "tool-examples-2025-10-29";
|
||||
const COMPACT_2026_09_04: &str = "compact-2026-09-04";
|
||||
const THINKING_BINDING_CONTROLS: &str = "thinking-binding-controls-2026-08-01";
|
||||
const THINKING_DISPLAY_UPDATES: &str = "thinking-display-updates-2026-08-18";
|
||||
const MID_CONVERSATION_OUTPUT_CONFIG: &str = "mid-conversation-output-config-2026-07-01";
|
||||
const MID_CONVERSATION_TOOL_CHANGES: &str = "mid-conversation-tool-changes-2026-07-01";
|
||||
const PROGRAMMATIC_CALLER: &str = "code_execution_20250825";
|
||||
const TOOL_SEARCH_TYPES: [&str; 2] = [
|
||||
"tool_search_tool_regex_20251119",
|
||||
"tool_search_tool_bm25_20251119",
|
||||
];
|
||||
const CONTEXT_MANAGEMENT_EDIT_BETAS: [(&str, &str); 2] = [
|
||||
("compact_20260112", "compact-2026-01-12"),
|
||||
("clear_tool_uses_20250919", "context-management-2025-06-27"),
|
||||
];
|
||||
const COMPUTER_TOOL_BETAS: [(&str, &str); 3] = [
|
||||
("computer_20251124", "computer-use-2025-11-24"),
|
||||
("computer_20250124", "computer-use-2025-01-24"),
|
||||
("computer_20241022", "computer-use-2024-10-22"),
|
||||
];
|
||||
const OPUS_4_5_EFFORT_PATTERNS: [&str; 2] = ["opus-4-5", "opus_4_5"];
|
||||
|
||||
fn on_bedrock(beta: &str) -> Option<String> {
|
||||
beta.parse::<AnthropicBeta>()
|
||||
.unwrap_or_else(|never| match never {})
|
||||
.on(BetaProvider::Bedrock)
|
||||
.map(|beta| beta.as_str().to_string())
|
||||
}
|
||||
|
||||
/// Port of Python's `model_specific_anthropic_betas`.
|
||||
fn model_specific_anthropic_betas(
|
||||
capabilities: &MessagesModelCapabilities,
|
||||
) -> BTreeSet<&'static str> {
|
||||
if capabilities.bedrock.supports_per_turn_control {
|
||||
BTreeSet::from([PER_TURN_CONTROL])
|
||||
} else {
|
||||
BTreeSet::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn header_value<'a>(headers: &'a Headers, name: &str) -> Option<&'a str> {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
}
|
||||
|
||||
fn user_betas(headers: &Headers) -> BTreeSet<String> {
|
||||
let Some(header) = header_value(headers, BETA_HEADER).filter(|value| !value.is_empty()) else {
|
||||
return BTreeSet::new();
|
||||
};
|
||||
if header.starts_with('[')
|
||||
&& header.ends_with(']')
|
||||
&& let Ok(Value::Array(values)) = serde_json::from_str::<Value>(header)
|
||||
{
|
||||
return values
|
||||
.iter()
|
||||
.map(|value| match value {
|
||||
Value::String(text) => text.trim().to_string(),
|
||||
other => other.to_string(),
|
||||
})
|
||||
.filter(|value| !value.is_empty())
|
||||
.collect();
|
||||
}
|
||||
header
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn tools(value: Option<&Value>) -> &[Value] {
|
||||
match value {
|
||||
Some(Value::Array(tools)) => tools,
|
||||
_ => &[],
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_type(tool: &Value) -> Option<&str> {
|
||||
tool.get("type").and_then(Value::as_str)
|
||||
}
|
||||
|
||||
fn lists_programmatic_caller(value: Option<&Value>) -> bool {
|
||||
matches!(value, Some(Value::Array(callers)) if callers.iter().any(|caller| caller.as_str() == Some(PROGRAMMATIC_CALLER)))
|
||||
}
|
||||
|
||||
fn programmatic_tool_calling_used(tools: &[Value]) -> bool {
|
||||
tools.iter().any(|tool| {
|
||||
lists_programmatic_caller(tool.get("allowed_callers"))
|
||||
|| lists_programmatic_caller(
|
||||
tool.get("function")
|
||||
.and_then(|function| function.get("allowed_callers")),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn non_empty_list(value: Option<&Value>) -> bool {
|
||||
matches!(value, Some(Value::Array(items)) if !items.is_empty())
|
||||
}
|
||||
|
||||
fn input_examples_used(tools: &[Value]) -> bool {
|
||||
tools.iter().any(|tool| {
|
||||
non_empty_list(tool.get("input_examples"))
|
||||
|| non_empty_list(
|
||||
tool.get("function")
|
||||
.and_then(|function| function.get("input_examples")),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn eager_input_streaming_flag(tool: &Value) -> Result<Option<bool>, Error> {
|
||||
let Value::Object(tool) = tool else {
|
||||
return Ok(None);
|
||||
};
|
||||
let strict_bool = |value: Option<&Value>| match value {
|
||||
None | Some(Value::Null) => Ok(None),
|
||||
Some(Value::Bool(flag)) => Ok(Some(*flag)),
|
||||
Some(_) => Err(Error::InvalidRequest(ErrorDetail::Expected {
|
||||
subject: "eager_input_streaming",
|
||||
expected: "a boolean",
|
||||
})),
|
||||
};
|
||||
let top_level = strict_bool(tool.get("eager_input_streaming"))?;
|
||||
let function = match tool.get("function") {
|
||||
None | Some(Value::Null) => None,
|
||||
Some(Value::Object(function)) => strict_bool(function.get("eager_input_streaming"))?,
|
||||
Some(_) => {
|
||||
return Err(Error::InvalidRequest(ErrorDetail::Expected {
|
||||
subject: "eager_input_streaming",
|
||||
expected: "a boolean",
|
||||
}));
|
||||
}
|
||||
};
|
||||
Ok(top_level.or(function))
|
||||
}
|
||||
|
||||
fn eager_input_streaming_used(tools: &[Value]) -> Result<bool, Error> {
|
||||
tools.iter().try_fold(false, |used, tool| {
|
||||
Ok(eager_input_streaming_flag(tool)? == Some(true) || used)
|
||||
})
|
||||
}
|
||||
|
||||
fn effort_used(params: &Body, model: &str, capabilities: &MessagesModelCapabilities) -> bool {
|
||||
if params.is_empty() || capabilities.supports_adaptive_thinking {
|
||||
return false;
|
||||
}
|
||||
let lowered = model.to_lowercase();
|
||||
let opus_4_5_effort = OPUS_4_5_EFFORT_PATTERNS
|
||||
.iter()
|
||||
.any(|pattern| lowered.contains(pattern))
|
||||
&& params
|
||||
.get("reasoning_effort")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|effort| !effort.is_empty());
|
||||
let output_config_effort = params
|
||||
.get("output_config")
|
||||
.and_then(|output_config| output_config.get("effort"))
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|effort| !effort.is_empty());
|
||||
opus_4_5_effort || output_config_effort
|
||||
}
|
||||
|
||||
fn system_messages(messages: &[Value]) -> impl Iterator<Item = &Value> {
|
||||
messages
|
||||
.iter()
|
||||
.filter(|message| message.get("role").and_then(Value::as_str) == Some("system"))
|
||||
}
|
||||
|
||||
fn mid_conversation_tool_change_used(messages: &[Value]) -> bool {
|
||||
system_messages(messages).any(|message| {
|
||||
tools(message.get("content")).iter().any(|block| {
|
||||
matches!(tool_type(block), Some("tool_addition" | "tool_removal"))
|
||||
&& block.get("tool").and_then(tool_type) == Some("tool_reference")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn thinking_betas(thinking: Option<&Value>) -> impl Iterator<Item = &'static str> {
|
||||
let display = super::request::is_thinking_display_updates_used(thinking)
|
||||
.then_some(THINKING_DISPLAY_UPDATES);
|
||||
let binding =
|
||||
matches!(thinking, Some(Value::Object(thinking)) if thinking.contains_key("block_binding"))
|
||||
.then_some(THINKING_BINDING_CONTROLS);
|
||||
display.into_iter().chain(binding)
|
||||
}
|
||||
|
||||
fn carries_signed_compaction_block(messages: &[Value]) -> bool {
|
||||
messages
|
||||
.iter()
|
||||
.flat_map(|message| tools(message.get("content")))
|
||||
.any(|block| {
|
||||
tool_type(block) == Some("compaction")
|
||||
&& block
|
||||
.get("signature")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|signature| !signature.is_empty())
|
||||
})
|
||||
}
|
||||
|
||||
/// Port of Python's `_filter_context_management_for_bedrock_invoke`.
|
||||
fn filter_context_management_for_bedrock_invoke(body: &mut Body, betas: &mut BTreeSet<String>) {
|
||||
let Some(Value::Object(context_management)) = body.get("context_management") else {
|
||||
return;
|
||||
};
|
||||
let Some(Value::Array(edits)) = context_management.get("edits") else {
|
||||
body.shift_remove("context_management");
|
||||
return;
|
||||
};
|
||||
let retained: Vec<(Value, &'static str)> = edits
|
||||
.iter()
|
||||
.filter_map(|edit| {
|
||||
let edit_type = edit.as_object()?.get("type")?.as_str()?;
|
||||
CONTEXT_MANAGEMENT_EDIT_BETAS
|
||||
.iter()
|
||||
.find(|(supported, _)| *supported == edit_type)
|
||||
.map(|(_, beta)| (edit.clone(), *beta))
|
||||
})
|
||||
.collect();
|
||||
if retained.is_empty() {
|
||||
body.shift_remove("context_management");
|
||||
return;
|
||||
}
|
||||
let mut filtered = context_management.clone();
|
||||
betas.extend(retained.iter().map(|(_, beta)| (*beta).to_string()));
|
||||
filtered.insert(
|
||||
"edits".into(),
|
||||
Value::Array(retained.into_iter().map(|(edit, _)| edit).collect()),
|
||||
);
|
||||
body.insert("context_management".into(), Value::Object(filtered));
|
||||
}
|
||||
|
||||
/// Port of the handler's `update_headers_with_filtered_beta`, keeping
|
||||
/// `model_specific_anthropic_betas`.
|
||||
fn update_headers_with_filtered_beta(headers: Headers, keep: &BTreeSet<&'static str>) -> Headers {
|
||||
BetaPolicy::Filter(BetaProvider::Bedrock).apply_keeping(
|
||||
headers,
|
||||
&keep
|
||||
.iter()
|
||||
.map(|beta| {
|
||||
beta.parse::<AnthropicBeta>()
|
||||
.unwrap_or_else(|never| match never {})
|
||||
})
|
||||
.collect::<BetaSet>(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Port of Python's `AnthropicModelInfo.get_anthropic_beta_list` for `custom_llm_provider="bedrock"`.
|
||||
fn get_anthropic_beta_list(
|
||||
body: &Body,
|
||||
params: &Body,
|
||||
model: &str,
|
||||
capabilities: &MessagesModelCapabilities,
|
||||
outgoing_tools: &[Value],
|
||||
messages: &[Value],
|
||||
) -> BTreeSet<String> {
|
||||
let mut betas = BTreeSet::new();
|
||||
if effort_used(params, model, capabilities) {
|
||||
betas.insert("effort-2025-11-24".into());
|
||||
}
|
||||
if let Some(beta) = outgoing_tools
|
||||
.iter()
|
||||
.find_map(|tool| tool_type(tool).filter(|kind| kind.starts_with("computer_")))
|
||||
.and_then(|kind| COMPUTER_TOOL_BETAS.iter().find(|(tool, _)| *tool == kind))
|
||||
.map(|(_, beta)| *beta)
|
||||
{
|
||||
betas.insert(beta.into());
|
||||
}
|
||||
if params.get("mcp_servers").is_some_and(python_truthy) {
|
||||
betas.insert("mcp-client-2025-04-04".into());
|
||||
}
|
||||
if messages.iter().any(|message| {
|
||||
message
|
||||
.as_object()
|
||||
.is_some_and(|message| message.contains_key("output_config"))
|
||||
}) {
|
||||
betas.insert(MID_CONVERSATION_OUTPUT_CONFIG.into());
|
||||
}
|
||||
betas.extend(thinking_betas(body.get("thinking")).map(str::to_string));
|
||||
if mid_conversation_tool_change_used(messages) {
|
||||
betas.insert(MID_CONVERSATION_TOOL_CHANGES.into());
|
||||
}
|
||||
betas
|
||||
}
|
||||
|
||||
/// Port of Python's `_add_compaction_beta_for_bedrock_invoke`.
|
||||
fn add_compaction_beta_for_bedrock_invoke(
|
||||
body: &Body,
|
||||
messages: &[Value],
|
||||
betas: &mut BTreeSet<String>,
|
||||
) {
|
||||
if body.get("compaction").is_some_and(Value::is_object)
|
||||
|| carries_signed_compaction_block(messages)
|
||||
{
|
||||
betas.insert(COMPACT_2026_09_04.into());
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `_get_tool_search_beta_header_for_bedrock`.
|
||||
fn get_tool_search_beta_header_for_bedrock(
|
||||
capabilities: &MessagesModelCapabilities,
|
||||
tool_search_used: bool,
|
||||
programmatic_tool_calling_used: bool,
|
||||
input_examples_used: bool,
|
||||
betas: &mut BTreeSet<String>,
|
||||
) {
|
||||
if tool_search_used && !(programmatic_tool_calling_used || input_examples_used) {
|
||||
betas.remove(ADVANCED_TOOL_USE);
|
||||
if capabilities.bedrock.supports_tool_search {
|
||||
betas.insert(TOOL_SEARCH_TOOL.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Port of Python's `_get_bedrock_invoke_anthropic_beta_headers`.
|
||||
pub(super) fn get_bedrock_invoke_anthropic_beta_headers(
|
||||
body: &mut Body,
|
||||
params: &Body,
|
||||
headers: Headers,
|
||||
model: &str,
|
||||
capabilities: &MessagesModelCapabilities,
|
||||
injected_thinking: bool,
|
||||
) -> Result<(Vec<String>, Headers), Error> {
|
||||
let keep = model_specific_anthropic_betas(capabilities);
|
||||
let headers = update_headers_with_filtered_beta(headers, &keep);
|
||||
let user = user_betas(&headers);
|
||||
let mut betas = user.clone();
|
||||
let outgoing_tools = tools(body.get("tools")).to_vec();
|
||||
let messages = tools(body.get("messages")).to_vec();
|
||||
betas.extend(get_anthropic_beta_list(
|
||||
body,
|
||||
params,
|
||||
model,
|
||||
capabilities,
|
||||
&outgoing_tools,
|
||||
&messages,
|
||||
));
|
||||
if injected_thinking {
|
||||
betas.insert("interleaved-thinking-2025-05-14".into());
|
||||
}
|
||||
if eager_input_streaming_used(&outgoing_tools)? {
|
||||
betas.insert("fine-grained-tool-streaming-2025-05-14".into());
|
||||
}
|
||||
if params
|
||||
.get("safeguards")
|
||||
.is_some_and(|safeguards| !safeguards.is_null())
|
||||
{
|
||||
betas.insert(DANGEROUS_TOOL_USE.into());
|
||||
} else {
|
||||
betas.remove(DANGEROUS_TOOL_USE);
|
||||
}
|
||||
filter_context_management_for_bedrock_invoke(body, &mut betas);
|
||||
add_compaction_beta_for_bedrock_invoke(body, &messages, &mut betas);
|
||||
let tool_search_used = outgoing_tools
|
||||
.iter()
|
||||
.any(|tool| tool_type(tool).is_some_and(|kind| TOOL_SEARCH_TYPES.contains(&kind)));
|
||||
get_tool_search_beta_header_for_bedrock(
|
||||
capabilities,
|
||||
tool_search_used,
|
||||
programmatic_tool_calling_used(&outgoing_tools),
|
||||
input_examples_used(&outgoing_tools),
|
||||
&mut betas,
|
||||
);
|
||||
if betas.contains(TOOL_SEARCH_TOOL) {
|
||||
betas.insert(TOOL_EXAMPLES.into());
|
||||
}
|
||||
let model_supported = user.iter().filter(|beta| keep.contains(beta.as_str()));
|
||||
let filtered: BTreeSet<String> = betas
|
||||
.iter()
|
||||
.filter_map(|beta| on_bedrock(beta))
|
||||
.chain(model_supported.cloned())
|
||||
.collect();
|
||||
let dropped: Vec<&String> = user
|
||||
.iter()
|
||||
.filter(|beta| !keep.contains(beta.as_str()) && on_bedrock(beta).is_none())
|
||||
.collect();
|
||||
if !dropped.is_empty() {
|
||||
litellm_tracing::warn!(
|
||||
"Bedrock Invoke: dropping unsupported anthropic-beta values from client headers: {dropped:?}. Bedrock has no mapping entry for these; forwarding them would cause a 400."
|
||||
);
|
||||
}
|
||||
Ok((filtered.into_iter().collect(), headers))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_llms_types::formats::messages::MessagesRequest;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
use crate::{
|
||||
base_llm::messages::{
|
||||
context::{BedrockCapabilities, MessagesTransformContext},
|
||||
transformation::BaseMessagesConfig,
|
||||
},
|
||||
bedrock::messages::invoke_transformations::anthropic_claude_transformation::AmazonAnthropicClaudeMessagesConfig,
|
||||
};
|
||||
|
||||
fn sent(
|
||||
params: Value,
|
||||
capabilities: MessagesModelCapabilities,
|
||||
header: Option<&str>,
|
||||
drop_params: bool,
|
||||
) -> (Value, Headers) {
|
||||
let mut request = json!({
|
||||
"model": "anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
|
||||
"max_tokens": 128,
|
||||
});
|
||||
request
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(params.as_object().unwrap().clone());
|
||||
let request: MessagesRequest = serde_json::from_value(request).unwrap();
|
||||
let context = MessagesTransformContext {
|
||||
drops_unsupported_params: drop_params,
|
||||
..MessagesTransformContext::new(capabilities, drop_params)
|
||||
};
|
||||
let headers = header
|
||||
.map(|value| vec![("anthropic-beta".to_string(), value.to_string())])
|
||||
.unwrap_or_default();
|
||||
let (request, headers) = AmazonAnthropicClaudeMessagesConfig
|
||||
.finalize_request(request, headers, &context)
|
||||
.unwrap();
|
||||
(serde_json::to_value(request).unwrap(), headers)
|
||||
}
|
||||
|
||||
fn betas(
|
||||
params: Value,
|
||||
capabilities: MessagesModelCapabilities,
|
||||
header: Option<&str>,
|
||||
) -> Value {
|
||||
sent(params, capabilities, header, false)
|
||||
.0
|
||||
.get("anthropic_beta")
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null)
|
||||
}
|
||||
|
||||
fn bedrock(bedrock: BedrockCapabilities) -> MessagesModelCapabilities {
|
||||
MessagesModelCapabilities {
|
||||
bedrock,
|
||||
..MessagesModelCapabilities::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unlisted_beta_dropped("advisor-tool-2026-03-01,context-1m-2025-08-07", json!(["context-1m-2025-08-07"]))]
|
||||
#[case::aliased_beta_renamed("advanced-tool-use-2025-11-20", json!(["tool-search-tool-2025-10-19"]))]
|
||||
#[case::unknown_beta_dropped("not-a-real-beta-2099-01-01", Value::Null)]
|
||||
#[case::dangerous_tool_use_dropped_without_safeguards(
|
||||
"dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14",
|
||||
Value::Null
|
||||
)]
|
||||
fn user_betas_go_through_the_bedrock_column(#[case] header: &str, #[case] expected: Value) {
|
||||
assert_eq!(
|
||||
betas(
|
||||
json!({}),
|
||||
MessagesModelCapabilities::default(),
|
||||
Some(header)
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::client_sends_beta("dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")]
|
||||
#[case::client_omits_beta("interleaved-thinking-2025-05-14")]
|
||||
fn safeguards_carry_the_dangerous_tool_use_beta(#[case] header: &str) {
|
||||
let (body, _) = sent(
|
||||
json!({"safeguards": {"mode": "block"}}),
|
||||
MessagesModelCapabilities::default(),
|
||||
Some(header),
|
||||
false,
|
||||
);
|
||||
assert_eq!(
|
||||
body["anthropic_beta"],
|
||||
json!(["dangerous-tool-use-2026-09-03"])
|
||||
);
|
||||
assert_eq!(body["safeguards"], json!({"mode": "block"}));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::flagged(true, json!(["per-turn-control-2026-07-01"]), Some("per-turn-control-2026-07-01"))]
|
||||
#[case::not_flagged(false, Value::Null, None)]
|
||||
fn per_turn_control_is_kept_only_where_the_model_flags_it(
|
||||
#[case] supports_per_turn_control: bool,
|
||||
#[case] expected: Value,
|
||||
#[case] header: Option<&str>,
|
||||
) {
|
||||
let (body, headers) = sent(
|
||||
json!({}),
|
||||
bedrock(BedrockCapabilities {
|
||||
supports_per_turn_control,
|
||||
..BedrockCapabilities::default()
|
||||
}),
|
||||
Some("per-turn-control-2026-07-01"),
|
||||
false,
|
||||
);
|
||||
assert_eq!(
|
||||
body.get("anthropic_beta").cloned().unwrap_or(Value::Null),
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
headers
|
||||
.iter()
|
||||
.find(|(name, _)| name == "anthropic-beta")
|
||||
.map(|(_, value)| value.as_str()),
|
||||
header
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::supported(true, json!(["tool-search-tool-2025-10-19"]))]
|
||||
#[case::unsupported(false, Value::Null)]
|
||||
fn tool_search_beta_follows_the_model_map(
|
||||
#[case] supports_tool_search: bool,
|
||||
#[case] expected: Value,
|
||||
) {
|
||||
assert_eq!(
|
||||
betas(
|
||||
json!({"tools": [{"type": "tool_search_tool_regex_20251119", "name": "tool_search"}]}),
|
||||
bedrock(BedrockCapabilities {
|
||||
supports_tool_search,
|
||||
..BedrockCapabilities::default()
|
||||
}),
|
||||
None,
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_search_with_programmatic_calling_keeps_the_users_beta() {
|
||||
assert_eq!(
|
||||
betas(
|
||||
json!({"tools": [
|
||||
{"type": "tool_search_tool_regex_20251119", "name": "tool_search"},
|
||||
{"name": "t", "input_schema": {"type": "object"}, "allowed_callers": ["code_execution_20250825"]},
|
||||
]}),
|
||||
MessagesModelCapabilities::default(),
|
||||
Some("advanced-tool-use-2025-11-20"),
|
||||
),
|
||||
json!(["tool-search-tool-2025-10-19"])
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::computer_20250124("computer_20250124", json!(["computer-use-2025-01-24"]))]
|
||||
#[case::computer_20251124("computer_20251124", json!(["computer-use-2025-11-24"]))]
|
||||
#[case::computer_20241022_is_rejected("computer_20241022", Value::Null)]
|
||||
fn computer_tools_add_their_beta(#[case] tool_type: &str, #[case] expected: Value) {
|
||||
assert_eq!(
|
||||
betas(
|
||||
json!({"tools": [{"type": tool_type, "name": "computer", "display_width_px": 1, "display_height_px": 1}]}),
|
||||
MessagesModelCapabilities::default(),
|
||||
None,
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_config_effort_adds_the_effort_beta_on_non_adaptive_models() {
|
||||
assert_eq!(
|
||||
betas(
|
||||
json!({"output_config": {"effort": "high"}}),
|
||||
MessagesModelCapabilities {
|
||||
supports_output_config: true,
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
None,
|
||||
),
|
||||
json!(["effort-2025-11-24"])
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::display_updates(json!({"type": "adaptive", "display": "updates"}), json!(["thinking-display-updates-2026-08-18"]))]
|
||||
#[case::block_binding(json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}), json!(["thinking-binding-controls-2026-08-01"]))]
|
||||
fn thinking_controls_add_their_betas(#[case] thinking: Value, #[case] expected: Value) {
|
||||
let capabilities = MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_adaptive_thinking: true,
|
||||
..MessagesModelCapabilities::default()
|
||||
};
|
||||
assert_eq!(
|
||||
betas(json!({"thinking": thinking}), capabilities, None),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_conversation_output_config_keeps_its_auto_added_beta() {
|
||||
let (body, _) = sent(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Hello"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Hi"}]},
|
||||
{"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}},
|
||||
{"role": "user", "content": [{"type": "text", "text": "Go on"}]},
|
||||
]}),
|
||||
MessagesModelCapabilities {
|
||||
supports_mid_conversation_system: true,
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
None,
|
||||
true,
|
||||
);
|
||||
assert_eq!(
|
||||
body["anthropic_beta"],
|
||||
json!(["mid-conversation-output-config-2026-07-01"])
|
||||
);
|
||||
assert_eq!(
|
||||
body["messages"][2]["output_config"],
|
||||
json!({"effort": "high"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_conversation_tool_changes_add_their_beta() {
|
||||
let (body, _) = sent(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi"},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "tool_name": "x"}}]},
|
||||
{"role": "user", "content": "Go on"},
|
||||
]}),
|
||||
MessagesModelCapabilities {
|
||||
supports_mid_conversation_system: true,
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
None,
|
||||
true,
|
||||
);
|
||||
assert_eq!(
|
||||
body["anthropic_beta"],
|
||||
json!(["mid-conversation-tool-changes-2026-07-01"])
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::clear_at_stripped(
|
||||
json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}),
|
||||
true,
|
||||
json!([
|
||||
{"role": "user", "content": "a"},
|
||||
{"role": "system", "content": "note"},
|
||||
{"role": "user", "content": "b"}
|
||||
])
|
||||
)]
|
||||
#[case::emptied_turn_dropped(
|
||||
json!({"role": "system", "content": "", "clear_at": {"type": "turns"}}),
|
||||
true,
|
||||
json!([{"role": "user", "content": "a"}, {"role": "user", "content": "b"}])
|
||||
)]
|
||||
#[case::kept_without_drop_params(
|
||||
json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}),
|
||||
false,
|
||||
json!([
|
||||
{"role": "user", "content": "a"},
|
||||
{"role": "system", "content": "note", "clear_at": {"type": "turns"}},
|
||||
{"role": "user", "content": "b"}
|
||||
])
|
||||
)]
|
||||
fn a_system_field_the_body_betas_do_not_unlock_is_stripped_under_drop_params(
|
||||
#[case] system: Value,
|
||||
#[case] drop_params: bool,
|
||||
#[case] expected: Value,
|
||||
) {
|
||||
let (body, _) = sent(
|
||||
json!({"messages": [{"role": "user", "content": "a"}, system, {"role": "user", "content": "b"}]}),
|
||||
MessagesModelCapabilities {
|
||||
supports_mid_conversation_system: true,
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
None,
|
||||
drop_params,
|
||||
);
|
||||
assert_eq!(body["messages"], expected);
|
||||
}
|
||||
}
|
||||
|
|
@ -1 +1,4 @@
|
|||
pub mod anthropic_claude_transformation;
|
||||
mod betas;
|
||||
pub mod request;
|
||||
mod stream;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,443 @@
|
|||
use std::collections::VecDeque;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::{Stream, StreamExt, stream};
|
||||
use litellm_llms_types::providers::bedrock::INVOCATION_METRICS_KEY;
|
||||
use litellm_python_compat::{Value, json, repr, truthy::truthy};
|
||||
|
||||
use crate::{
|
||||
Error, ErrorDetail,
|
||||
base_llm::messages::streaming::{ByteStream, FrameStream},
|
||||
};
|
||||
|
||||
use crate::bedrock::chat::invoke_handler::{BedrockStreamRaise, aiter_bytes, raised};
|
||||
|
||||
const METRICS_USAGE_KEYS: [(&str, &str); 4] = [
|
||||
("input_tokens", "inputTokenCount"),
|
||||
("output_tokens", "outputTokenCount"),
|
||||
("cache_read_input_tokens", "cacheReadInputTokenCount"),
|
||||
("cache_creation_input_tokens", "cacheWriteInputTokenCount"),
|
||||
];
|
||||
|
||||
const CACHE_FIELDS: [&str; 2] = ["cache_creation_input_tokens", "cache_read_input_tokens"];
|
||||
|
||||
type Fields = Vec<(Value, Value)>;
|
||||
|
||||
fn key(name: &str) -> Value {
|
||||
Value::Str(name.to_string())
|
||||
}
|
||||
|
||||
fn position(fields: &Fields, name: &str) -> Option<usize> {
|
||||
fields
|
||||
.iter()
|
||||
.position(|(key, _)| matches!(key, Value::Str(key) if key == name))
|
||||
}
|
||||
|
||||
fn get<'a>(fields: &'a Fields, name: &str) -> Option<&'a Value> {
|
||||
position(fields, name).map(|at| &fields[at].1)
|
||||
}
|
||||
|
||||
fn set(fields: &mut Fields, name: &str, value: Value) {
|
||||
match position(fields, name) {
|
||||
Some(at) => fields[at].1 = value,
|
||||
None => fields.push((key(name), value)),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_pair(fields: &mut Fields, name: Value, value: Value) {
|
||||
match fields.iter().position(|(key, _)| *key == name) {
|
||||
Some(at) => fields[at].1 = value,
|
||||
None => fields.push((name, value)),
|
||||
}
|
||||
}
|
||||
|
||||
fn pop(fields: &mut Fields, name: &str) -> Option<Value> {
|
||||
position(fields, name).map(|at| fields.remove(at).1)
|
||||
}
|
||||
|
||||
fn is_type(fields: &Fields, event_type: &str) -> bool {
|
||||
matches!(get(fields, "type"), Some(Value::Str(value)) if value == event_type)
|
||||
}
|
||||
|
||||
fn dict_or_empty(value: Option<&Value>) -> Fields {
|
||||
match value {
|
||||
Some(Value::Dict(fields)) => fields.clone(),
|
||||
_ => Fields::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `AmazonAnthropicClaudeMessagesStreamDecoder._chunk_parser`: Bedrock's invocation metrics
|
||||
/// become Anthropic usage, under the chunk's own usage.
|
||||
fn chunk_parser(chunk: Value) -> Result<Fields, Error> {
|
||||
let mut fields = match chunk {
|
||||
Value::Dict(fields) => fields,
|
||||
other => return Err(raised(BedrockStreamRaise::NotAnObject(dumped(&other)?))),
|
||||
};
|
||||
let Some(metrics) = pop(&mut fields, INVOCATION_METRICS_KEY).filter(truthy) else {
|
||||
return Ok(fields);
|
||||
};
|
||||
let metrics = dict_or_empty(Some(&metrics));
|
||||
let preserved = dict_or_empty(get(&fields, "usage"));
|
||||
let usage = preserved.into_iter().fold(
|
||||
METRICS_USAGE_KEYS
|
||||
.iter()
|
||||
.filter_map(|(anthropic, metric)| {
|
||||
Some((key(anthropic), get(&metrics, metric)?.clone()))
|
||||
})
|
||||
.collect::<Fields>(),
|
||||
|mut usage, (name, value)| {
|
||||
set_pair(&mut usage, name, value);
|
||||
usage
|
||||
},
|
||||
);
|
||||
set(&mut fields, "usage", Value::Dict(usage));
|
||||
Ok(fields)
|
||||
}
|
||||
|
||||
/// `AmazonAnthropicClaudeMessagesConfig._promote_message_stop_usage`, on the decoded chunks.
|
||||
#[derive(Default)]
|
||||
struct UsagePromoter {
|
||||
pending_delta: Option<Fields>,
|
||||
start_usage: Option<Fields>,
|
||||
}
|
||||
|
||||
impl UsagePromoter {
|
||||
/// Port of Python's `_merge_message_start_cache_into_delta_usage`.
|
||||
fn merge_message_start_cache_into_delta_usage(&self, delta_usage: &mut Fields) {
|
||||
let Some(start) = self.start_usage.as_ref().filter(|start| !start.is_empty()) else {
|
||||
return;
|
||||
};
|
||||
for field in CACHE_FIELDS.iter().chain(["cache_creation"].iter()) {
|
||||
if get(delta_usage, field).is_none()
|
||||
&& let Some(value) = get(start, field).filter(|value| **value != Value::None)
|
||||
{
|
||||
set(delta_usage, field, value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn completed(&self, mut delta: Fields, stop: Option<&Fields>) -> Fields {
|
||||
let mut delta_usage = dict_or_empty(get(&delta, "usage").filter(|usage| truthy(usage)));
|
||||
if let Some(stop) = stop {
|
||||
for field in CACHE_FIELDS {
|
||||
if let Some(value) = get(stop, field) {
|
||||
set(&mut delta_usage, field, value.clone());
|
||||
}
|
||||
}
|
||||
if let Some(raw) = get(stop, "input_tokens").filter(|raw| **raw != Value::None) {
|
||||
let tokens = match raw {
|
||||
Value::Int(_) | Value::Bool(_) => raw.clone(),
|
||||
_ => Value::from(0),
|
||||
};
|
||||
set(&mut delta_usage, "input_tokens", tokens);
|
||||
}
|
||||
}
|
||||
self.merge_message_start_cache_into_delta_usage(&mut delta_usage);
|
||||
if !delta_usage.is_empty() {
|
||||
set(&mut delta, "usage", Value::Dict(delta_usage));
|
||||
}
|
||||
delta
|
||||
}
|
||||
|
||||
fn feed(&mut self, chunk: Fields) -> Vec<Fields> {
|
||||
if is_type(&chunk, "message_start") {
|
||||
let message = dict_or_empty(get(&chunk, "message").filter(|message| truthy(message)));
|
||||
if let Some(Value::Dict(usage)) = get(&message, "usage") {
|
||||
self.start_usage = Some(usage.clone());
|
||||
}
|
||||
return self
|
||||
.pending_delta
|
||||
.take()
|
||||
.into_iter()
|
||||
.chain([chunk])
|
||||
.collect();
|
||||
}
|
||||
if is_type(&chunk, "message_delta") {
|
||||
self.pending_delta = Some(chunk);
|
||||
return Vec::new();
|
||||
}
|
||||
if is_type(&chunk, "message_stop")
|
||||
&& let Some(delta) = self.pending_delta.take()
|
||||
{
|
||||
let stop_usage = dict_or_empty(get(&chunk, "usage").filter(|usage| truthy(usage)));
|
||||
return vec![self.completed(delta, Some(&stop_usage)), chunk];
|
||||
}
|
||||
self.pending_delta
|
||||
.take()
|
||||
.into_iter()
|
||||
.chain([chunk])
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Option<Fields> {
|
||||
let delta = self.pending_delta.take()?;
|
||||
Some(self.completed(delta, None))
|
||||
}
|
||||
}
|
||||
|
||||
fn dumped(value: &Value) -> Result<String, Error> {
|
||||
json::dumps(value)
|
||||
.map_err(|error| Error::InvalidResponse(ErrorDetail::failed("SSE event", error)))
|
||||
}
|
||||
|
||||
/// `BaseAnthropicMessagesStreamingIterator._convert_chunk_to_sse_format` for a decoded chunk.
|
||||
fn sse_frame(chunk: Fields) -> Result<Bytes, Error> {
|
||||
let event_type = get(&chunk, "type").map_or_else(|| "message".to_string(), repr::to_str);
|
||||
let data = dumped(&Value::Dict(chunk))?;
|
||||
Ok(Bytes::from(format!(
|
||||
"event: {event_type}\ndata: {data}\n\n"
|
||||
)))
|
||||
}
|
||||
|
||||
struct Relay<S> {
|
||||
chunks: S,
|
||||
promoter: UsagePromoter,
|
||||
queued: VecDeque<Bytes>,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl<S: Stream<Item = Result<Value, Error>> + Unpin> Relay<S> {
|
||||
fn queue(&mut self, chunks: impl IntoIterator<Item = Fields>) -> Result<(), Error> {
|
||||
for chunk in chunks {
|
||||
self.queued.push_back(sse_frame(chunk)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn advance(&mut self) -> Result<(), Error> {
|
||||
match self.chunks.next().await {
|
||||
Some(chunk) => {
|
||||
let chunk = chunk_parser(chunk?)?;
|
||||
let promoted = self.promoter.feed(chunk);
|
||||
self.queue(promoted)
|
||||
}
|
||||
None => {
|
||||
self.finished = true;
|
||||
let flushed = self.promoter.finish();
|
||||
self.queue(flushed)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bedrock InvokeModel's event stream as the SSE frames Python's Bedrock Messages stream yields:
|
||||
/// each decoded chunk re-serialized with `json.dumps`, with usage promoted onto `message_delta`.
|
||||
pub fn get_async_streaming_response_iterator(
|
||||
bytes: ByteStream,
|
||||
headers: Vec<(String, String)>,
|
||||
) -> FrameStream {
|
||||
let relay = Relay {
|
||||
chunks: Box::pin(aiter_bytes(bytes, headers)),
|
||||
promoter: UsagePromoter::default(),
|
||||
queued: VecDeque::new(),
|
||||
finished: false,
|
||||
};
|
||||
stream::unfold(Some(relay), |relay| async move {
|
||||
let mut relay = relay?;
|
||||
loop {
|
||||
if let Some(frame) = relay.queued.pop_front() {
|
||||
return Some((Ok(frame), Some(relay)));
|
||||
}
|
||||
if relay.finished {
|
||||
return None;
|
||||
}
|
||||
if let Err(error) = relay.advance().await {
|
||||
return Some((Err(error), None));
|
||||
}
|
||||
}
|
||||
})
|
||||
.boxed()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use aws_smithy_eventstream::frame::write_message_to;
|
||||
use aws_smithy_types::event_stream::{Header, HeaderValue, Message};
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use futures_util::TryStreamExt;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn aws_frame(chunk: &str) -> Vec<u8> {
|
||||
let payload = format!(r#"{{"bytes": "{}"}}"#, STANDARD.encode(chunk));
|
||||
let message = Message::new(Bytes::from(payload))
|
||||
.add_header(Header::new(
|
||||
":event-type",
|
||||
HeaderValue::String("chunk".into()),
|
||||
))
|
||||
.add_header(Header::new(
|
||||
":message-type",
|
||||
HeaderValue::String("event".into()),
|
||||
));
|
||||
let mut wire = Vec::new();
|
||||
write_message_to(&message, &mut wire).unwrap();
|
||||
wire
|
||||
}
|
||||
|
||||
fn in_pieces(chunks: &[&str]) -> ByteStream {
|
||||
let wire: Vec<u8> = chunks.iter().flat_map(|chunk| aws_frame(chunk)).collect();
|
||||
stream::iter(
|
||||
wire.chunks(7)
|
||||
.map(|piece| Ok(Bytes::copy_from_slice(piece)))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.boxed()
|
||||
}
|
||||
|
||||
async fn relayed(chunks: &[&str]) -> Result<String, Error> {
|
||||
let frames: Vec<Bytes> =
|
||||
get_async_streaming_response_iterator(in_pieces(chunks), Vec::new())
|
||||
.try_collect()
|
||||
.await?;
|
||||
Ok(frames
|
||||
.iter()
|
||||
.map(|frame| String::from_utf8(frame.to_vec()).unwrap())
|
||||
.collect())
|
||||
}
|
||||
|
||||
const MESSAGE_STOP: &str = "event: message_stop\ndata: {\"type\": \"message_stop\"}\n\n";
|
||||
|
||||
#[rstest]
|
||||
#[case::stop_sequence_null_is_kept(
|
||||
r#"{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":4}}"#,
|
||||
"event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\", \"stop_sequence\": null}, \"usage\": {\"output_tokens\": 4}}\n\n"
|
||||
)]
|
||||
#[case::an_absent_stop_sequence_stays_absent(
|
||||
r#"{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":4}}"#,
|
||||
"event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 4}}\n\n"
|
||||
)]
|
||||
#[case::unknown_keys_and_non_ascii_are_dumped_as_python_does(
|
||||
r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hé"},"extra":[1.0,2]}"#,
|
||||
"event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"h\\u00e9\"}, \"extra\": [1.0, 2]}\n\n"
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn each_chunk_is_relayed_as_json_dumps_writes_it(
|
||||
#[case] chunk: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
relayed(&[chunk, r#"{"type":"message_stop"}"#])
|
||||
.await
|
||||
.unwrap(),
|
||||
format!("{expected}{MESSAGE_STOP}")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_chunk_without_a_type_is_named_message() {
|
||||
assert_eq!(
|
||||
relayed(&[r#"{"message":"x"}"#, r#"{"type":"message_stop"}"#])
|
||||
.await
|
||||
.unwrap(),
|
||||
format!("event: message\ndata: {{\"message\": \"x\"}}\n\n{MESSAGE_STOP}")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::cache_fields_on_message_stop(
|
||||
r#"{"input_tokens": 10, "output_tokens": 0}"#,
|
||||
r#"{"output_tokens": 5}"#,
|
||||
r#"{"input_tokens": 3, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20}"#,
|
||||
r#"{"output_tokens": 5, "cache_creation_input_tokens": 20, "cache_read_input_tokens": 100, "input_tokens": 3}"#
|
||||
)]
|
||||
#[case::cache_only_on_message_start(
|
||||
r#"{"input_tokens": 10, "output_tokens": 0, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}"#,
|
||||
r#"{"output_tokens": 5}"#,
|
||||
r#"{"input_tokens": 10}"#,
|
||||
r#"{"output_tokens": 5, "input_tokens": 10, "cache_creation_input_tokens": 4, "cache_read_input_tokens": 80, "cache_creation": {"ephemeral_5m_input_tokens": 4}}"#
|
||||
)]
|
||||
#[case::a_float_input_count_is_zeroed(
|
||||
r#"{"input_tokens": 10}"#,
|
||||
r#"{"output_tokens": 5}"#,
|
||||
r#"{"input_tokens": 3.5}"#,
|
||||
r#"{"output_tokens": 5, "input_tokens": 0}"#
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn message_delta_usage_is_completed_from_stop_then_start(
|
||||
#[case] start: &str,
|
||||
#[case] delta: &str,
|
||||
#[case] stop: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let start = format!(r#"{{"type": "message_start", "message": {{"usage": {start}}}}}"#);
|
||||
let delta = format!(r#"{{"type": "message_delta", "usage": {delta}}}"#);
|
||||
let stop = format!(r#"{{"type": "message_stop", "usage": {stop}}}"#);
|
||||
|
||||
let relayed = relayed(&[&start, &delta, &stop]).await.unwrap();
|
||||
|
||||
assert!(
|
||||
relayed.contains(&format!(
|
||||
"event: message_delta\ndata: {{\"type\": \"message_delta\", \"usage\": {expected}}}\n\n"
|
||||
)),
|
||||
"{relayed}"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::metrics_fill_missing_usage(
|
||||
r#"{"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "outputTokenCount": 9}}"#,
|
||||
r#"{"type": "message_stop", "usage": {"input_tokens": 3, "output_tokens": 9}}"#
|
||||
)]
|
||||
#[case::the_chunks_own_usage_wins_in_place(
|
||||
r#"{"type": "message_stop", "usage": {"input_tokens": 1}, "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}, "amazon-bedrock-trace": {}}"#,
|
||||
r#"{"type": "message_stop", "usage": {"input_tokens": 1, "cache_read_input_tokens": 40}, "amazon-bedrock-trace": {}}"#
|
||||
)]
|
||||
#[case::empty_metrics_are_dropped(
|
||||
r#"{"type": "message_stop", "amazon-bedrock-invocationMetrics": {}}"#,
|
||||
r#"{"type": "message_stop"}"#
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn invocation_metrics_become_anthropic_usage(
|
||||
#[case] chunk: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
relayed(&[chunk]).await.unwrap(),
|
||||
format!("event: message_stop\ndata: {expected}\n\n")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_message_stop(
|
||||
&[r#"{"type":"message_start","message":{}}"#],
|
||||
"event: message_start\ndata: {\"type\": \"message_start\", \"message\": {}}\n\n"
|
||||
)]
|
||||
#[case::a_held_delta_is_flushed(
|
||||
&[r#"{"type":"message_delta","usage":{"output_tokens":1}}"#],
|
||||
"event: message_delta\ndata: {\"type\": \"message_delta\", \"usage\": {\"output_tokens\": 1}}\n\n"
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn a_stream_without_message_stop_ends_with_its_own_frames(
|
||||
#[case] chunks: &[&str],
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(relayed(chunks).await.unwrap(), expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_provider_error_event_is_terminal() {
|
||||
let error = r#"{"type":"error","error":{"type":"overloaded_error","message":"busy"}}"#;
|
||||
|
||||
assert_eq!(
|
||||
relayed(&[error]).await.unwrap(),
|
||||
"event: error\ndata: {\"type\": \"error\", \"error\": {\"type\": \"overloaded_error\", \"message\": \"busy\"}}\n\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::a_list(r#"[1]"#, BedrockStreamRaise::NotAnObject("[1]".into()))]
|
||||
#[case::not_json(r#"{"type": "#, BedrockStreamRaise::InvalidJson(r#"{"type": "#.into()))]
|
||||
#[tokio::test]
|
||||
async fn a_payload_python_cannot_parse_raises_what_python_raises(
|
||||
#[case] chunk: &str,
|
||||
#[case] expected: BedrockStreamRaise,
|
||||
) {
|
||||
let error = relayed(&[chunk]).await.unwrap_err();
|
||||
|
||||
assert_eq!(
|
||||
litellm_error::find::<BedrockStreamRaise>(&error),
|
||||
Some(&expected)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -3,3 +3,4 @@ pub mod chat;
|
|||
pub mod claude_platform;
|
||||
pub mod common_utils;
|
||||
pub mod messages;
|
||||
pub mod request_metadata;
|
||||
|
|
|
|||
336
litellm/rust/crates/llms/src/bedrock/request_metadata.rs
Normal file
336
litellm/rust/crates/llms/src/bedrock/request_metadata.rs
Normal file
|
|
@ -0,0 +1,336 @@
|
|||
use litellm_python_compat::{json as python_json, repr};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::base_llm::auth::Headers;
|
||||
|
||||
pub const REQUEST_METADATA_FIELDS_PARAM: &str = "bedrock_request_metadata_fields";
|
||||
pub const METADATA_PARAMS: [&str; 2] = ["metadata", "litellm_metadata"];
|
||||
|
||||
pub const REQUEST_METADATA_HEADER: &str = "X-Amzn-Bedrock-Request-Metadata";
|
||||
const REQUEST_METADATA_MAX_PAIRS: usize = 16;
|
||||
const REQUEST_METADATA_IDENTITY_PREFIX: &str = "user_api_key_";
|
||||
const REQUEST_METADATA_CLIENT_FIELD: &str = "spend_logs_metadata";
|
||||
const REQUEST_METADATA_HEADER_MAX_CHARS: usize = 8500;
|
||||
const REQUEST_METADATA_MAX_CHARS: usize = 256;
|
||||
|
||||
fn is_allowed_character(character: char) -> bool {
|
||||
character.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
character,
|
||||
' ' | '\t'
|
||||
| '\n'
|
||||
| '\r'
|
||||
| '\x0b'
|
||||
| '\x0c'
|
||||
| ':'
|
||||
| '_'
|
||||
| '@'
|
||||
| '$'
|
||||
| '#'
|
||||
| '='
|
||||
| '/'
|
||||
| '+'
|
||||
| ','
|
||||
| '.'
|
||||
| '-'
|
||||
)
|
||||
}
|
||||
|
||||
fn matches_constraint(text: &str, minimum: usize) -> bool {
|
||||
let count = text.chars().count();
|
||||
(minimum..=REQUEST_METADATA_MAX_CHARS).contains(&count)
|
||||
&& text.chars().all(is_allowed_character)
|
||||
}
|
||||
|
||||
fn is_forwardable(key: &str, value: &str) -> bool {
|
||||
matches_constraint(key, 1) && matches_constraint(value, 0)
|
||||
}
|
||||
|
||||
fn is_reserved(key: &str) -> bool {
|
||||
key.trim()
|
||||
.to_lowercase()
|
||||
.starts_with(REQUEST_METADATA_IDENTITY_PREFIX)
|
||||
}
|
||||
|
||||
fn text_pairs(source: Option<&Value>) -> Vec<(&str, &str)> {
|
||||
match source {
|
||||
Some(Value::Object(fields)) => fields
|
||||
.iter()
|
||||
.filter_map(|(key, value)| Some((key.as_str(), value.as_str()?)))
|
||||
.collect(),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn allowed_fields(configured: Option<&Value>) -> Vec<String> {
|
||||
let Some(Value::Array(fields)) = configured else {
|
||||
return Vec::new();
|
||||
};
|
||||
let fields: Vec<String> = fields
|
||||
.iter()
|
||||
.map(|field| match field {
|
||||
Value::String(text) => text.clone(),
|
||||
other => repr::to_str(&python_json::from_json(other.clone())),
|
||||
})
|
||||
.collect();
|
||||
fields
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(index, field)| !fields[..*index].contains(field))
|
||||
.map(|(_, field)| field.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Port of Python's `resolve_bedrock_request_metadata`.
|
||||
fn resolve_bedrock_request_metadata(
|
||||
sources: &[&Map<String, Value>],
|
||||
allowed: &[String],
|
||||
) -> Vec<(String, String)> {
|
||||
let identity: Vec<(String, String)> = allowed
|
||||
.iter()
|
||||
.filter(|field| field.starts_with(REQUEST_METADATA_IDENTITY_PREFIX))
|
||||
.filter_map(|field| {
|
||||
let value = sources
|
||||
.iter()
|
||||
.find_map(|source| source.get(field.as_str()).and_then(Value::as_str))?;
|
||||
is_forwardable(field, value).then(|| (field.clone(), value.to_string()))
|
||||
})
|
||||
.take(REQUEST_METADATA_MAX_PAIRS)
|
||||
.collect();
|
||||
let spend_logs_pairs: Vec<(&str, &str)> = if allowed
|
||||
.iter()
|
||||
.any(|field| field == REQUEST_METADATA_CLIENT_FIELD)
|
||||
{
|
||||
sources
|
||||
.iter()
|
||||
.flat_map(|source| text_pairs(source.get(REQUEST_METADATA_CLIENT_FIELD)))
|
||||
.collect()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut seen: Vec<&str> = Vec::new();
|
||||
let client: Vec<(String, String)> = spend_logs_pairs
|
||||
.into_iter()
|
||||
.filter(|(key, value)| !is_reserved(key) && is_forwardable(key, value))
|
||||
.filter(|(key, _)| {
|
||||
let first = !seen.contains(key);
|
||||
seen.push(key);
|
||||
first
|
||||
})
|
||||
.take(REQUEST_METADATA_MAX_PAIRS - identity.len())
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect();
|
||||
identity
|
||||
.into_iter()
|
||||
.chain(client)
|
||||
.fold(Vec::new(), |mut resolved, (key, value)| {
|
||||
match resolved.iter_mut().find(|(existing, _)| *existing == key) {
|
||||
Some(slot) => slot.1 = value,
|
||||
None => resolved.push((key, value)),
|
||||
}
|
||||
resolved
|
||||
})
|
||||
}
|
||||
|
||||
fn header_value(pairs: &[(String, String)]) -> String {
|
||||
let object: Map<String, Value> = pairs
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), Value::String(value.clone())))
|
||||
.collect();
|
||||
python_json::dumps(&python_json::from_json(Value::Object(object)))
|
||||
.map(|text| compact_python_json(&text))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn compact_python_json(text: &str) -> String {
|
||||
let mut compact = String::with_capacity(text.len());
|
||||
let mut in_string = false;
|
||||
let mut escaped = false;
|
||||
let mut characters = text.chars().peekable();
|
||||
while let Some(character) = characters.next() {
|
||||
if in_string {
|
||||
compact.push(character);
|
||||
match (escaped, character) {
|
||||
(true, _) => escaped = false,
|
||||
(false, '\\') => escaped = true,
|
||||
(false, '"') => in_string = false,
|
||||
_ => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match character {
|
||||
'"' => {
|
||||
in_string = true;
|
||||
compact.push(character);
|
||||
}
|
||||
',' | ':' if characters.peek() == Some(&' ') => {
|
||||
compact.push(character);
|
||||
characters.next();
|
||||
}
|
||||
_ => compact.push(character),
|
||||
}
|
||||
}
|
||||
compact
|
||||
}
|
||||
|
||||
/// Port of Python's `_longest_fitting_prefix`.
|
||||
fn longest_fitting_prefix(pairs: &[(String, String)]) -> Option<String> {
|
||||
(1..=pairs.len())
|
||||
.rev()
|
||||
.map(|count| header_value(&pairs[..count]))
|
||||
.find(|value| value.chars().count() <= REQUEST_METADATA_HEADER_MAX_CHARS)
|
||||
}
|
||||
|
||||
/// Port of Python's `bedrock_request_metadata_headers`.
|
||||
pub(crate) fn bedrock_request_metadata_headers(
|
||||
extra: &std::collections::BTreeMap<String, Value>,
|
||||
) -> Option<Option<(String, String)>> {
|
||||
let allowed = allowed_fields(extra.get(REQUEST_METADATA_FIELDS_PARAM));
|
||||
if allowed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let sources: Vec<&Map<String, Value>> = METADATA_PARAMS
|
||||
.iter()
|
||||
.filter_map(|name| extra.get(*name).and_then(Value::as_object))
|
||||
.collect();
|
||||
let resolved = resolve_bedrock_request_metadata(&sources, &allowed);
|
||||
Some(
|
||||
longest_fitting_prefix(&resolved).map(|value| (REQUEST_METADATA_HEADER.to_string(), value)),
|
||||
)
|
||||
}
|
||||
|
||||
/// Port of Python's `merge_bedrock_invoke_headers`.
|
||||
pub(crate) fn merge_bedrock_invoke_headers(
|
||||
headers: Headers,
|
||||
caller_owned: Vec<(String, String)>,
|
||||
proxy_owned: Option<Option<(String, String)>>,
|
||||
) -> Headers {
|
||||
let owned_names: Vec<String> = proxy_owned
|
||||
.as_ref()
|
||||
.map(|_| vec![REQUEST_METADATA_HEADER.to_lowercase()])
|
||||
.unwrap_or_default();
|
||||
if caller_owned.is_empty() && owned_names.is_empty() {
|
||||
return headers;
|
||||
}
|
||||
let existing: Vec<String> = headers
|
||||
.iter()
|
||||
.map(|(name, _)| name.to_lowercase())
|
||||
.collect();
|
||||
let kept = headers
|
||||
.into_iter()
|
||||
.filter(|(name, _)| !owned_names.contains(&name.to_lowercase()));
|
||||
let added = caller_owned
|
||||
.into_iter()
|
||||
.filter(|(name, _)| !existing.contains(&name.to_lowercase()));
|
||||
kept.chain(added).chain(proxy_owned.flatten()).fold(
|
||||
Vec::new(),
|
||||
|mut merged: Headers, (name, value)| {
|
||||
match merged.iter_mut().find(|(existing, _)| *existing == name) {
|
||||
Some(slot) => slot.1 = value,
|
||||
None => merged.push((name, value)),
|
||||
}
|
||||
merged
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
use crate::bedrock::chat::invoke_transformations::base_invoke_transformation::{
|
||||
GUARDRAIL_IDENTIFIER_HEADER, GUARDRAIL_VERSION_HEADER,
|
||||
};
|
||||
|
||||
fn extra(value: Value) -> std::collections::BTreeMap<String, Value> {
|
||||
value
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_callers_guardrail_header_wins() {
|
||||
let merged = merge_bedrock_invoke_headers(
|
||||
vec![("x-amzn-bedrock-guardrailidentifier".into(), "caller".into())],
|
||||
vec![
|
||||
(GUARDRAIL_IDENTIFIER_HEADER.into(), "deployment".into()),
|
||||
(GUARDRAIL_VERSION_HEADER.into(), "1".into()),
|
||||
],
|
||||
None,
|
||||
);
|
||||
assert_eq!(
|
||||
merged,
|
||||
vec![
|
||||
(
|
||||
"x-amzn-bedrock-guardrailidentifier".to_string(),
|
||||
"caller".to_string()
|
||||
),
|
||||
(GUARDRAIL_VERSION_HEADER.to_string(), "1".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::disabled_by_default(json!({}), None)]
|
||||
#[case::identity_then_spend_logs(
|
||||
json!({
|
||||
"bedrock_request_metadata_fields": ["user_api_key_team_id", "spend_logs_metadata", "user_api_key_team_id"],
|
||||
"metadata": {"user_api_key_team_id": "team-1", "spend_logs_metadata": {"project": "alpha", "user_api_key_alias": "forged", "bad key!": "x"}},
|
||||
}),
|
||||
Some(Some(r#"{"user_api_key_team_id":"team-1","project":"alpha"}"#)),
|
||||
)]
|
||||
#[case::litellm_metadata_is_read_too(
|
||||
json!({"bedrock_request_metadata_fields": ["user_api_key_user_id"], "litellm_metadata": {"user_api_key_user_id": "u 1"}}),
|
||||
Some(Some(r#"{"user_api_key_user_id":"u 1"}"#)),
|
||||
)]
|
||||
#[case::owned_but_empty(
|
||||
json!({"bedrock_request_metadata_fields": ["user_api_key_team_id"], "metadata": {}}),
|
||||
Some(None),
|
||||
)]
|
||||
fn request_metadata_header_follows_the_allow_list(
|
||||
#[case] params: Value,
|
||||
#[case] expected: Option<Option<&str>>,
|
||||
) {
|
||||
assert_eq!(
|
||||
bedrock_request_metadata_headers(&extra(params)),
|
||||
expected.map(|header| header
|
||||
.map(|value| (REQUEST_METADATA_HEADER.to_string(), value.to_string())))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_proxy_owns_the_request_metadata_header() {
|
||||
let merged = merge_bedrock_invoke_headers(
|
||||
vec![
|
||||
("x-amzn-bedrock-request-metadata".into(), "forged".into()),
|
||||
("x-keep".into(), "1".into()),
|
||||
],
|
||||
Vec::new(),
|
||||
Some(None),
|
||||
);
|
||||
assert_eq!(merged, vec![("x-keep".to_string(), "1".to_string())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_header_keeps_the_longest_prefix_that_fits() {
|
||||
let long = "v".repeat(256);
|
||||
let spend_logs: Map<String, Value> = (0..40)
|
||||
.map(|index| (format!("key{index:02}"), Value::String(long.clone())))
|
||||
.collect();
|
||||
let params = extra(json!({
|
||||
"bedrock_request_metadata_fields": ["spend_logs_metadata"],
|
||||
"metadata": {"spend_logs_metadata": spend_logs},
|
||||
}));
|
||||
let Some(Some((_, value))) = bedrock_request_metadata_headers(¶ms) else {
|
||||
panic!("expected a header");
|
||||
};
|
||||
let pairs: Map<String, Value> = serde_json::from_str(&value).unwrap();
|
||||
assert_eq!(pairs.len(), 16);
|
||||
assert!(value.len() <= REQUEST_METADATA_HEADER_MAX_CHARS);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,8 +5,8 @@ use std::sync::LazyLock;
|
|||
use litellm_auth::{AwsParams, CredentialPlacement};
|
||||
use litellm_secret_value::SecretValue;
|
||||
|
||||
use litellm_auth_aws::AwsCredentialSource;
|
||||
use litellm_auth_aws::bedrock_model_id_and_region;
|
||||
use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders};
|
||||
use litellm_core_utils::settings::resolve_non_empty;
|
||||
use litellm_http::request::{with_header, without_headers};
|
||||
use litellm_llms_types::{
|
||||
|
|
@ -102,6 +102,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
|
|||
region: mantle_region(params.api_base.as_deref(), model, params, env)?,
|
||||
service: "bedrock",
|
||||
credentials: Box::new(AwsCredentialSource::from_params(¶ms.aws, env)),
|
||||
caller_headers: CallerSigningHeaders::Botocore,
|
||||
},
|
||||
};
|
||||
Ok(ValidatedEnvironment { headers, auth })
|
||||
|
|
|
|||
|
|
@ -220,6 +220,10 @@ pub enum ErrorDetail {
|
|||
#[source]
|
||||
cause: Source,
|
||||
},
|
||||
#[error(
|
||||
"Bedrock does not support Anthropic's server-side web search tool (tool type '{tool_type}', model '{model}'). To use web search with this model, enable LiteLLM's web search interception so the proxy executes the search instead: https://docs.litellm.ai/docs/integrations/websearch_interception. Alternatively, remove the web_search tool from the request."
|
||||
)]
|
||||
UnsupportedBedrockWebSearchTool { tool_type: String, model: String },
|
||||
#[error("{operation} failed: {detail}")]
|
||||
RemoteFailure {
|
||||
operation: &'static str,
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ tempfile.workspace = true
|
|||
tokio-tungstenite.workspace = true
|
||||
wiremock.workspace = true
|
||||
aws-sdk-secretsmanager = "1.117.0"
|
||||
aws-credential-types = "1.3.0"
|
||||
|
||||
[[bench]]
|
||||
name = "serialization"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::error::Error as StdError;
|
||||
|
||||
use litellm_error::Report;
|
||||
use litellm_error::{Report, Source};
|
||||
use litellm_inference::Error;
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyTypeError, PyValueError},
|
||||
|
|
@ -42,28 +42,81 @@ const MESSAGES_PYTHON_RAISED: &str = "messages_python_raised";
|
|||
/// The text of a refusal Python's provider config raises from its request transform, which its
|
||||
/// HTTP handler maps as a provider error.
|
||||
const MESSAGES_PROVIDER_REJECTION: &str = "messages_provider_rejection";
|
||||
/// Python's provider config raises the refusal as litellm's `BadRequestError`.
|
||||
const MESSAGES_PROVIDER_BAD_REQUEST: &str = "messages_provider_bad_request";
|
||||
/// Which failure Python's AWS signing raises for a credential Rust could not acquire, or `replay`
|
||||
/// when only running Python's signing again gives its exception.
|
||||
const MESSAGES_CREDENTIAL_FAILURE: &str = "messages_credential_failure";
|
||||
|
||||
fn provider_rejection(rejection: &(dyn StdError + 'static)) -> Option<String> {
|
||||
struct ProviderRejection {
|
||||
text: String,
|
||||
bad_request: bool,
|
||||
}
|
||||
|
||||
fn credential_failure(error: &Error) -> Option<&'static str> {
|
||||
let (Error::Auth(auth) | Error::Llm(litellm_llms::Error::Auth(auth))) = error else {
|
||||
return None;
|
||||
};
|
||||
match auth {
|
||||
litellm_auth::Error::Provider(_) => Some(aws_credential_failure(auth)),
|
||||
litellm_auth::Error::CredentialAcquisition(_)
|
||||
| litellm_auth::Error::CredentialChain { .. } => Some("replay"),
|
||||
litellm_auth::Error::InvalidConfiguration(_)
|
||||
| litellm_auth::Error::EmptyCallerCredential { .. }
|
||||
| litellm_auth::Error::MissingApiKey { .. }
|
||||
| litellm_auth::Error::MissingParam { .. }
|
||||
| litellm_auth::Error::MissingApiBase { .. }
|
||||
| litellm_auth::Error::InvalidHeader => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn aws_credential_failure(error: &(dyn StdError + 'static)) -> &'static str {
|
||||
let innermost = std::iter::successors(Some(error), |&cause| cause.source())
|
||||
.filter_map(|cause| {
|
||||
cause
|
||||
.downcast_ref::<litellm_auth_aws::Error>()
|
||||
.or_else(|| cause.downcast_ref::<Source>()?.downcast_ref())
|
||||
})
|
||||
.last();
|
||||
match innermost {
|
||||
Some(litellm_auth_aws::Error::WebIdentityTokenReference) => "web_identity_reference",
|
||||
Some(litellm_auth_aws::Error::MissingWebIdentityToken) => "missing_web_identity_token",
|
||||
Some(failure) if failure.is_empty_default_chain() => "no_credentials",
|
||||
_ => "replay",
|
||||
}
|
||||
}
|
||||
|
||||
fn provider_rejection(rejection: &(dyn StdError + 'static)) -> Option<ProviderRejection> {
|
||||
match rejection.downcast_ref::<litellm_llms::Error>() {
|
||||
Some(litellm_llms::Error::MissingField("max_tokens")) => {
|
||||
return Some("max_tokens is required for Anthropic /v1/messages API".to_owned());
|
||||
return Some(ProviderRejection {
|
||||
text: "max_tokens is required for Anthropic /v1/messages API".to_owned(),
|
||||
bad_request: false,
|
||||
});
|
||||
}
|
||||
Some(_) => return None,
|
||||
None => {}
|
||||
}
|
||||
match rejection.downcast_ref::<litellm_llms::ErrorDetail>() {
|
||||
Some(
|
||||
detail @ litellm_llms::ErrorDetail::InvalidChoice {
|
||||
detail @ (litellm_llms::ErrorDetail::InvalidChoice {
|
||||
field: "reasoning effort",
|
||||
..
|
||||
},
|
||||
) => Some(format!("litellm.BadRequestError: {detail}")),
|
||||
}
|
||||
| litellm_llms::ErrorDetail::UnsupportedBedrockWebSearchTool { .. }),
|
||||
) => Some(ProviderRejection {
|
||||
text: detail.to_string(),
|
||||
bad_request: true,
|
||||
}),
|
||||
Some(
|
||||
detail @ (litellm_llms::ErrorDetail::InvalidReasoningEffort { .. }
|
||||
| litellm_llms::ErrorDetail::UnsupportedValue {
|
||||
field: "effort", ..
|
||||
}),
|
||||
) => Some(detail.to_string()),
|
||||
) => Some(ProviderRejection {
|
||||
text: detail.to_string(),
|
||||
bad_request: false,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
|
@ -182,8 +235,11 @@ pub(crate) fn route_failure(
|
|||
) {
|
||||
value.setattr(MESSAGES_LATE_REJECTION, true)?;
|
||||
}
|
||||
if let Some(text) = provider_rejection(rejection) {
|
||||
value.setattr(MESSAGES_PROVIDER_REJECTION, text)?;
|
||||
if let Some(provider) = provider_rejection(rejection) {
|
||||
value.setattr(MESSAGES_PROVIDER_REJECTION, provider.text)?;
|
||||
if provider.bad_request {
|
||||
value.setattr(MESSAGES_PROVIDER_BAD_REQUEST, true)?;
|
||||
}
|
||||
}
|
||||
native
|
||||
}
|
||||
|
|
@ -204,6 +260,9 @@ pub(crate) fn route_failure(
|
|||
if error.is_missing_credential() {
|
||||
value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?;
|
||||
}
|
||||
if let Some(failure) = credential_failure(&error) {
|
||||
value.setattr(MESSAGES_CREDENTIAL_FAILURE, failure)?;
|
||||
}
|
||||
native
|
||||
}
|
||||
(RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error),
|
||||
|
|
@ -389,7 +448,7 @@ mod tests {
|
|||
#[rstest::rstest]
|
||||
#[case::missing_max_tokens(
|
||||
litellm_llms::Error::MissingField("max_tokens").into(),
|
||||
Some("max_tokens is required for Anthropic /v1/messages API")
|
||||
Some(("max_tokens is required for Anthropic /v1/messages API", false))
|
||||
)]
|
||||
#[case::unmapped_effort(
|
||||
Error::invalid_request(litellm_llms::ErrorDetail::InvalidChoice {
|
||||
|
|
@ -397,7 +456,17 @@ mod tests {
|
|||
actual: "'ultra'".into(),
|
||||
choices: vec!["low", "none"],
|
||||
}),
|
||||
Some("litellm.BadRequestError: Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'.")
|
||||
Some(("Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'.", true))
|
||||
)]
|
||||
#[case::bedrock_web_search(
|
||||
Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedBedrockWebSearchTool {
|
||||
tool_type: "web_search_20250305".into(),
|
||||
model: "claude".into(),
|
||||
}),
|
||||
Some((
|
||||
"Bedrock does not support Anthropic's server-side web search tool (tool type 'web_search_20250305', model 'claude'). To use web search with this model, enable LiteLLM's web search interception so the proxy executes the search instead: https://docs.litellm.ai/docs/integrations/websearch_interception. Alternatively, remove the web_search tool from the request.",
|
||||
true
|
||||
))
|
||||
)]
|
||||
#[case::other_missing_field(litellm_llms::Error::MissingField("model").into(), None)]
|
||||
#[case::unsupported_param(
|
||||
|
|
@ -411,17 +480,169 @@ mod tests {
|
|||
)]
|
||||
fn provider_config_rejections_carry_python_text(
|
||||
#[case] error: Error,
|
||||
#[case] expected: Option<&str>,
|
||||
#[case] expected: Option<(&str, bool)>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap();
|
||||
let text = native
|
||||
.value(py)
|
||||
let value = native.value(py);
|
||||
let text = value
|
||||
.getattr_opt(MESSAGES_PROVIDER_REJECTION)
|
||||
.unwrap()
|
||||
.map(|value| value.extract::<String>().unwrap());
|
||||
assert_eq!(text.as_deref(), expected);
|
||||
let bad_request = value
|
||||
.getattr_opt(MESSAGES_PROVIDER_BAD_REQUEST)
|
||||
.unwrap()
|
||||
.is_some_and(|value| value.extract::<bool>().unwrap());
|
||||
assert_eq!(text.as_deref().map(|text| (text, bad_request)), expected);
|
||||
assert!(text.is_some() || !bad_request);
|
||||
});
|
||||
}
|
||||
|
||||
fn web_identity_failure(failure: litellm_auth_aws::Error) -> litellm_auth_aws::Error {
|
||||
litellm_auth_aws::Error::Credentials {
|
||||
flow: litellm_auth_aws::CredentialFlow::WebIdentity,
|
||||
source: Source::new(
|
||||
aws_credential_types::provider::error::CredentialsError::provider_error(failure),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::environment_reference(
|
||||
litellm_auth_aws::Error::WebIdentityTokenReference,
|
||||
"web_identity_reference"
|
||||
)]
|
||||
#[case::unresolved_token(
|
||||
web_identity_failure(litellm_auth_aws::Error::MissingWebIdentityToken),
|
||||
"missing_web_identity_token"
|
||||
)]
|
||||
#[case::empty_default_chain(
|
||||
litellm_auth_aws::Error::Credentials {
|
||||
flow: litellm_auth_aws::CredentialFlow::DefaultChain,
|
||||
source: Source::new(aws_credential_types::provider::error::CredentialsError::not_loaded(
|
||||
"no providers in chain provided credentials",
|
||||
)),
|
||||
},
|
||||
"no_credentials"
|
||||
)]
|
||||
#[case::unreadable_token(
|
||||
web_identity_failure(litellm_auth_aws::Error::WebIdentityToken(Source::new(
|
||||
std::io::Error::other("secret source unavailable"),
|
||||
))),
|
||||
"replay"
|
||||
)]
|
||||
#[case::sts_rejection(
|
||||
web_identity_failure(litellm_auth_aws::Error::MissingWebIdentityCredentials),
|
||||
"replay"
|
||||
)]
|
||||
#[case::assume_role(
|
||||
litellm_auth_aws::Error::Credentials {
|
||||
flow: litellm_auth_aws::CredentialFlow::AssumeRole,
|
||||
source: Source::new(aws_credential_types::provider::error::CredentialsError::provider_error(
|
||||
std::io::Error::other("AccessDenied"),
|
||||
)),
|
||||
},
|
||||
"replay"
|
||||
)]
|
||||
#[case::empty_profile(
|
||||
litellm_auth_aws::Error::Credentials {
|
||||
flow: litellm_auth_aws::CredentialFlow::Profile,
|
||||
source: Source::new(aws_credential_types::provider::error::CredentialsError::not_loaded(
|
||||
"profile not found",
|
||||
)),
|
||||
},
|
||||
"replay"
|
||||
)]
|
||||
#[case::expired(litellm_auth_aws::Error::ExpiredCredentials, "replay")]
|
||||
fn an_aws_credential_failure_names_what_python_signing_raises(
|
||||
#[case] failure: litellm_auth_aws::Error,
|
||||
#[case] expected: &str,
|
||||
#[values(false, true)] through_llm: bool,
|
||||
) {
|
||||
let auth = litellm_auth::Error::from(failure);
|
||||
let error = match through_llm {
|
||||
true => Error::Llm(litellm_llms::Error::Auth(auth)),
|
||||
false => Error::Auth(auth),
|
||||
};
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap();
|
||||
assert!(native.is_instance_of::<PyRuntimeError>(py));
|
||||
let marked = native
|
||||
.value(py)
|
||||
.getattr(MESSAGES_CREDENTIAL_FAILURE)
|
||||
.unwrap()
|
||||
.extract::<String>()
|
||||
.unwrap();
|
||||
assert_eq!(marked, expected);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::acquisition(
|
||||
litellm_auth::Error::CredentialAcquisition(litellm_auth::ErrorDetail::Empty {
|
||||
subject: "token"
|
||||
}),
|
||||
Some("replay")
|
||||
)]
|
||||
#[case::chain(
|
||||
litellm_auth::Error::CredentialChain {
|
||||
last: Box::new(litellm_auth::Error::CredentialAcquisition(
|
||||
litellm_auth::ErrorDetail::Empty { subject: "token" },
|
||||
)),
|
||||
earlier: Vec::new(),
|
||||
},
|
||||
Some("replay")
|
||||
)]
|
||||
#[case::missing_key(
|
||||
litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
},
|
||||
None
|
||||
)]
|
||||
#[case::invalid_header(litellm_auth::Error::InvalidHeader, None)]
|
||||
fn only_credential_acquisition_failures_carry_the_credential_marker(
|
||||
#[case] auth: litellm_auth::Error,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let unmarked =
|
||||
route_failure(py, Error::Auth(auth.clone()), RequestMarker::Unmarked, Ok).unwrap();
|
||||
assert!(
|
||||
unmarked
|
||||
.value(py)
|
||||
.getattr_opt(MESSAGES_CREDENTIAL_FAILURE)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let native = route_failure(py, Error::Auth(auth), RequestMarker::Messages, Ok).unwrap();
|
||||
let marked = native
|
||||
.value(py)
|
||||
.getattr_opt(MESSAGES_CREDENTIAL_FAILURE)
|
||||
.unwrap()
|
||||
.map(|value| value.extract::<String>().unwrap());
|
||||
assert_eq!(marked.as_deref(), expected);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wrapped_credential_failure_keeps_the_native_mapping() {
|
||||
let error = Error::post_call(Error::Auth(litellm_auth::Error::from(
|
||||
litellm_auth_aws::Error::WebIdentityTokenReference,
|
||||
)));
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap();
|
||||
assert!(
|
||||
native
|
||||
.value(py)
|
||||
.getattr_opt(MESSAGES_CREDENTIAL_FAILURE)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -80,11 +80,16 @@ fn decode_ssl_verify(field: &Field<'_>) -> Result<Option<SslVerify>, ProjectionE
|
|||
Err(field.invalid("a Boolean, Boolean string, CA path, or None"))
|
||||
}
|
||||
|
||||
static RESOURCES: LazyLock<litellm_inference::resources::CoreResources> = LazyLock::new(|| {
|
||||
litellm_inference::resources::CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(
|
||||
PublicDnsResolver,
|
||||
))))
|
||||
});
|
||||
static RESOURCES: LazyLock<litellm_inference::resources::CoreResources> =
|
||||
LazyLock::new(|| litellm_inference::resources::CoreResources {
|
||||
pool: Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))),
|
||||
auth: Arc::new(litellm_auth::AuthServices {
|
||||
aws: litellm_auth_aws::AwsAuthService::with_web_identity_tokens(Arc::new(
|
||||
crate::secrets::HostWebIdentityTokens,
|
||||
)),
|
||||
..litellm_auth::AuthServices::default()
|
||||
}),
|
||||
});
|
||||
|
||||
pub(crate) fn resources() -> &'static litellm_inference::resources::CoreResources {
|
||||
&RESOURCES
|
||||
|
|
|
|||
|
|
@ -26,7 +26,9 @@ impl<M: Machine> Machine for LoggedMachine<M> {
|
|||
type Complete = M::Complete;
|
||||
|
||||
fn resume(&mut self) -> Step<'_, Self> {
|
||||
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
|
||||
let logger = self
|
||||
.logger
|
||||
.get_or_init(|| Python::attach(super::capture_call));
|
||||
Box::pin(logger.instrument(logger.scope(|| self.machine.resume())))
|
||||
}
|
||||
|
||||
|
|
@ -34,7 +36,9 @@ impl<M: Machine> Machine for LoggedMachine<M> {
|
|||
&mut self,
|
||||
failure: HostFailure<<Self::Protocol as Protocol>::Error>,
|
||||
) -> Interrupted<'_, Self> {
|
||||
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
|
||||
let logger = self
|
||||
.logger
|
||||
.get_or_init(|| Python::attach(super::capture_call));
|
||||
Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure))))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ mod machine;
|
|||
|
||||
pub(crate) use machine::LoggedMachine;
|
||||
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
|
||||
use litellm_host_python::{Pythonized, release_gil};
|
||||
use litellm_tracing::{
|
||||
DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink, TokenRedactor,
|
||||
|
|
@ -118,6 +120,40 @@ fn processing_error(error: fancy_regex::Error) -> PyErr {
|
|||
|
||||
struct PythonSink {
|
||||
correlation: (String, String),
|
||||
levels: Option<Levels>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Levels([AtomicU8; 4]);
|
||||
|
||||
const UNKNOWN: u8 = 0;
|
||||
const DISABLED: u8 = 1;
|
||||
const ENABLED: u8 = 2;
|
||||
|
||||
impl Levels {
|
||||
fn enabled(&self, level: u8, query: impl FnOnce() -> bool) -> bool {
|
||||
let slot = &self.0[usize::from(level / 10 - 1)];
|
||||
match slot.load(Ordering::Relaxed) {
|
||||
UNKNOWN => {
|
||||
let enabled = query();
|
||||
slot.store(if enabled { ENABLED } else { DISABLED }, Ordering::Relaxed);
|
||||
enabled
|
||||
}
|
||||
state => state == ENABLED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn python_enabled(level: u8) -> bool {
|
||||
Python::try_attach(|py| {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method1("enabled", (level,)))
|
||||
.and_then(|enabled| enabled.extract()),
|
||||
)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn level(level: &Level) -> u8 {
|
||||
|
|
@ -145,15 +181,11 @@ impl Sink for PythonSink {
|
|||
{
|
||||
return false;
|
||||
}
|
||||
Python::try_attach(|py| {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method1("enabled", (level(metadata.level()),)))
|
||||
.and_then(|enabled| enabled.extract()),
|
||||
)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
let level = level(metadata.level());
|
||||
match &self.levels {
|
||||
Some(levels) => levels.enabled(level, || python_enabled(level)),
|
||||
None => python_enabled(level),
|
||||
}
|
||||
}
|
||||
|
||||
fn emit(&self, record: &Record) {
|
||||
|
|
@ -182,12 +214,25 @@ impl Sink for PythonSink {
|
|||
}
|
||||
|
||||
pub(crate) fn capture(py: Python<'_>) -> Logger {
|
||||
logger(py, None)
|
||||
}
|
||||
|
||||
pub(crate) fn capture_call(py: Python<'_>) -> Logger {
|
||||
logger(py, Some(Levels::default()))
|
||||
}
|
||||
|
||||
fn logger(py: Python<'_>, levels: Option<Levels>) -> Logger {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method0("context"))
|
||||
.and_then(|value| value.extract())
|
||||
.map(|correlation| Logger::new(PythonSink { correlation })),
|
||||
.map(|correlation| {
|
||||
Logger::new(PythonSink {
|
||||
correlation,
|
||||
levels,
|
||||
})
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::{process::Command, task::Poll};
|
||||
use std::{cell::Cell, process::Command, task::Poll};
|
||||
|
||||
use litellm_host::{
|
||||
machine::{HostFailure, Interrupted, Machine, MachineStep, Step},
|
||||
|
|
@ -39,6 +39,51 @@ impl Machine for DiagnosticMachine {
|
|||
}
|
||||
}
|
||||
|
||||
struct ChunkMachine {
|
||||
raise_level: bool,
|
||||
}
|
||||
|
||||
impl Machine for ChunkMachine {
|
||||
type Protocol = DiagnosticMachine;
|
||||
type Complete = ();
|
||||
|
||||
fn resume(&mut self) -> Step<'_, Self> {
|
||||
let raise_level = self.raise_level;
|
||||
Box::pin(async move {
|
||||
for index in 0..3 {
|
||||
litellm_tracing::debug!(index, "stream chunk");
|
||||
if raise_level {
|
||||
Python::attach(|py| {
|
||||
py.import("litellm._logging")?
|
||||
.getattr("verbose_logger")?
|
||||
.call_method1("setLevel", (10,))
|
||||
.map(|_| ())
|
||||
})
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
Ok(MachineStep::Complete(()))
|
||||
})
|
||||
}
|
||||
|
||||
fn interrupt(&mut self, _: HostFailure<String>) -> Interrupted<'_, Self> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn machine_chunks(py: Python<'_>, raise_level: bool) -> PyResult<Bound<'_, PyAny>> {
|
||||
let mut machine = super::LoggedMachine::new(ChunkMachine { raise_level });
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
machine
|
||||
.resume()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(pyo3::exceptions::PyValueError::new_err)
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn machine_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let mut machine = super::LoggedMachine::new(DiagnosticMachine);
|
||||
|
|
@ -122,6 +167,24 @@ fn http_warning(py: Python<'_>) -> PyResult<()> {
|
|||
crate::http::call_config(py, &PyDict::new(py), false).map(|_| ())
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::debug(10)]
|
||||
#[case::info(20)]
|
||||
#[case::warning(30)]
|
||||
#[case::error(40)]
|
||||
fn call_levels_ask_python_once_per_level(#[case] level: u8, #[values(true, false)] enabled: bool) {
|
||||
let levels = super::Levels::default();
|
||||
let queries = Cell::new(0);
|
||||
let query = || {
|
||||
queries.set(queries.get() + 1);
|
||||
enabled
|
||||
};
|
||||
assert_eq!(levels.enabled(level, query), enabled);
|
||||
assert_eq!(levels.enabled(level, query), enabled);
|
||||
assert_eq!(levels.enabled(level, query), enabled);
|
||||
assert_eq!(queries.get(), 1);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() {
|
||||
if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() {
|
||||
|
|
@ -157,6 +220,12 @@ fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication
|
|||
wrap_pyfunction!(machine_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"machine_chunks",
|
||||
wrap_pyfunction!(machine_chunks, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"synchronous_failure",
|
||||
|
|
@ -281,6 +350,42 @@ try:
|
|||
('warn', logging.WARNING), ('error', logging.ERROR),
|
||||
]
|
||||
|
||||
import litellm.rust_bridge.host.logger as host_logger
|
||||
original_enabled = host_logger.enabled
|
||||
queries = []
|
||||
def counting_enabled(level):
|
||||
queries.append(level)
|
||||
return original_enabled(level)
|
||||
host_logger.enabled = counting_enabled
|
||||
async def chunks(raise_level):
|
||||
await machine_chunks(raise_level)
|
||||
try:
|
||||
verbose_logger.setLevel(logging.WARNING)
|
||||
before_chunks = len(capture.records)
|
||||
asyncio.run(chunks(False))
|
||||
assert capture.records[before_chunks:] == []
|
||||
assert queries == [logging.DEBUG]
|
||||
|
||||
queries.clear()
|
||||
verbose_logger.setLevel(logging.DEBUG)
|
||||
asyncio.run(chunks(False))
|
||||
assert [(r.getMessage(), r.levelno, r.rust_fields) for r in capture.records[before_chunks:]] == [
|
||||
('stream chunk', logging.DEBUG, {'index': 0}),
|
||||
('stream chunk', logging.DEBUG, {'index': 1}),
|
||||
('stream chunk', logging.DEBUG, {'index': 2}),
|
||||
]
|
||||
assert queries == [logging.DEBUG] * 4
|
||||
|
||||
queries.clear()
|
||||
verbose_logger.setLevel(logging.WARNING)
|
||||
before_raise = len(capture.records)
|
||||
asyncio.run(chunks(True))
|
||||
assert capture.records[before_raise:] == []
|
||||
assert queries == [logging.DEBUG]
|
||||
finally:
|
||||
host_logger.enabled = original_enabled
|
||||
verbose_logger.setLevel(logging.DEBUG)
|
||||
|
||||
before_spans = len(capture.records)
|
||||
async def traced_request():
|
||||
session = session_id_var.set('span-session')
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ use litellm_llms::{
|
|||
CALLABLE_TOKEN_PROVIDER, CALLER_TOKEN_PROVIDER_PARAM, ENTRA_ID_PARAMS,
|
||||
},
|
||||
base_llm::messages::context::MessagesModelCapabilities,
|
||||
bedrock::chat::invoke_handler::BedrockStreamRaise,
|
||||
};
|
||||
use litellm_llms_types::headers::ProviderSpecificHeaders;
|
||||
use litellm_router_types::LitellmParams;
|
||||
|
|
@ -51,14 +52,14 @@ type MergedHeaders = (Option<Map<String, Value>>, Option<Vec<u8>>);
|
|||
|
||||
const ANTHROPIC_BETA_HEADER: &str = "anthropic-beta";
|
||||
|
||||
/// Python filters `anthropic-beta` before its client encodes it, so the wire check decides it.
|
||||
/// Only for `azure_ai`: by the owner's ruling `anthropic` keeps main's hold-back of the header.
|
||||
/// A config whose `should_filter_anthropic_beta_headers` holds filters `anthropic-beta` before
|
||||
/// Python's client encodes it, so the wire check decides it.
|
||||
fn is_filtered_beta_header(
|
||||
name: &Bound<'_, PyAny>,
|
||||
value: &Bound<'_, PyAny>,
|
||||
provider_filters_beta: bool,
|
||||
filters_beta: bool,
|
||||
) -> bool {
|
||||
provider_filters_beta
|
||||
filters_beta
|
||||
&& name
|
||||
.extract::<String>()
|
||||
.is_ok_and(|name| name.eq_ignore_ascii_case(ANTHROPIC_BETA_HEADER))
|
||||
|
|
@ -68,7 +69,7 @@ fn is_filtered_beta_header(
|
|||
fn ascii_headers<'py>(
|
||||
headers: &Bound<'py, PyAny>,
|
||||
unencodable: &mut Option<Vec<u8>>,
|
||||
provider_filters_beta: bool,
|
||||
filters_beta: bool,
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let Ok(headers) = headers.cast::<PyDict>() else {
|
||||
return Ok(headers.clone());
|
||||
|
|
@ -77,7 +78,7 @@ fn ascii_headers<'py>(
|
|||
let kept = PyDict::new(py);
|
||||
for (name, value) in headers.iter() {
|
||||
if value.is_instance_of::<PyString>()
|
||||
&& !is_filtered_beta_header(&name, &value, provider_filters_beta)
|
||||
&& !is_filtered_beta_header(&name, &value, filters_beta)
|
||||
&& let Err(error) = value.call_method1("encode", ("ascii",))
|
||||
{
|
||||
if !error.is_instance_of::<PyUnicodeEncodeError>(py) {
|
||||
|
|
@ -99,7 +100,7 @@ const PROVIDER_SPECIFIC_HEADERS_MODULE: &str =
|
|||
|
||||
fn ascii_scoped_headers<'py>(
|
||||
value: &Bound<'py, PyAny>,
|
||||
provider_filters_beta: bool,
|
||||
filters_beta: bool,
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let ascii_entry = |entry: Bound<'py, PyAny>| -> PyResult<Bound<'py, PyAny>> {
|
||||
let Ok(entry) = entry.cast::<PyDict>() else {
|
||||
|
|
@ -111,7 +112,7 @@ fn ascii_scoped_headers<'py>(
|
|||
let kept = entry.copy()?;
|
||||
kept.set_item(
|
||||
"extra_headers",
|
||||
ascii_headers(&headers, &mut None, provider_filters_beta)?,
|
||||
ascii_headers(&headers, &mut None, filters_beta)?,
|
||||
)?;
|
||||
Ok(kept.into_any())
|
||||
};
|
||||
|
|
@ -146,7 +147,8 @@ fn json_messages(messages: &Bound<'_, PyAny>) -> PyResult<Value> {
|
|||
}
|
||||
|
||||
pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages";
|
||||
const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities";
|
||||
const MESSAGES_STREAM_FAILURE: &str = "messages_stream_failure";
|
||||
pub(super) const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities";
|
||||
|
||||
fn merge_headers(
|
||||
forwarded: Option<Map<String, Value>>,
|
||||
|
|
@ -162,6 +164,63 @@ const FEDERATING_PROVIDER: &str = "anthropic";
|
|||
const AZURE_AI_PROVIDER: &str = "azure_ai";
|
||||
const ENABLE_TOKEN_REFRESH: &str = "enable_azure_ad_token_refresh";
|
||||
|
||||
const BEDROCK_PROVIDER: &str = "bedrock";
|
||||
const BEDROCK_PASSTHROUGH_PARAMS: [&str; 2] = ["model_id", "guardrailConfig"];
|
||||
const BEDROCK_METADATA_PARAMS: [&str; 2] = ["metadata", "litellm_metadata"];
|
||||
const BEDROCK_SPEND_LOGS_METADATA: &str = "spend_logs_metadata";
|
||||
const BEDROCK_REQUEST_METADATA_FIELDS: &str = "bedrock_request_metadata_fields";
|
||||
|
||||
fn text_pairs(value: &Bound<'_, PyAny>) -> PyResult<Map<String, Value>> {
|
||||
let Ok(mapping) = value.cast::<PyMapping>() else {
|
||||
return Ok(Map::new());
|
||||
};
|
||||
mapping
|
||||
.items()?
|
||||
.iter()
|
||||
.filter_map(|item| {
|
||||
let (key, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = match item.extract() {
|
||||
Ok(pair) => pair,
|
||||
Err(error) => return Some(Err(error)),
|
||||
};
|
||||
let key = key.cast::<PyString>().ok()?.to_str().ok()?.to_string();
|
||||
if key == BEDROCK_SPEND_LOGS_METADATA {
|
||||
return Some(text_pairs(&value).map(|pairs| (key, Value::Object(pairs))));
|
||||
}
|
||||
let value = value.cast::<PyString>().ok()?.to_str().ok()?.to_string();
|
||||
Some(Ok((key, Value::String(value))))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bedrock_litellm_params<'py>(
|
||||
py: Python<'py>,
|
||||
argument: impl Fn(&str) -> PyResult<Option<Bound<'py, PyAny>>>,
|
||||
) -> PyResult<Map<String, Value>> {
|
||||
let mut extra = project_optional_fields(BEDROCK_PASSTHROUGH_PARAMS, &argument)?;
|
||||
for name in BEDROCK_METADATA_PARAMS {
|
||||
if let Some(value) = argument(name)? {
|
||||
extra.insert(name.to_string(), Value::Object(text_pairs(&value)?));
|
||||
}
|
||||
}
|
||||
if let Some(configured) = module_global(py, BEDROCK_REQUEST_METADATA_FIELDS)?
|
||||
&& (configured.is_instance_of::<PyList>() || configured.is_instance_of::<PyTuple>())
|
||||
{
|
||||
let fields = configured
|
||||
.try_iter()?
|
||||
.map(|field| Ok(Value::String(field?.str()?.to_str()?.to_string())))
|
||||
.collect::<PyResult<Vec<Value>>>()?;
|
||||
extra.insert(
|
||||
BEDROCK_REQUEST_METADATA_FIELDS.to_string(),
|
||||
Value::Array(fields),
|
||||
);
|
||||
}
|
||||
Ok(extra)
|
||||
}
|
||||
|
||||
fn global_drop_params(py: Python<'_>) -> PyResult<bool> {
|
||||
Ok(module_global(py, "drop_params")?.is_some_and(|value| value.is(PyBool::new(py, true))))
|
||||
}
|
||||
|
||||
fn workload_identity_fields(py: Python<'_>) -> PyResult<&'static [String]> {
|
||||
static FIELDS: PyOnceLock<Vec<String>> = PyOnceLock::new();
|
||||
FIELDS
|
||||
|
|
@ -266,6 +325,16 @@ impl litellm_host::protocol::Protocol for MessagesHosted {
|
|||
type HostCall = MessagesHostCall;
|
||||
type Chunk = Bytes;
|
||||
type StreamHead = MessagesStreamHead;
|
||||
|
||||
fn detached_drain(head: &MessagesStreamHead) -> Option<litellm_host::protocol::DetachedDrain> {
|
||||
<Messages as litellm_host::protocol::Protocol>::detached_drain(head)
|
||||
}
|
||||
|
||||
fn unobserved_tail(
|
||||
head: &MessagesStreamHead,
|
||||
) -> Option<litellm_host::protocol::UnobservedTail> {
|
||||
<Messages as litellm_host::protocol::Protocol>::unobserved_tail(head)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) enum MessagesHostCall {
|
||||
|
|
@ -325,6 +394,7 @@ pub(super) struct MessagesPythonHost {
|
|||
pending_short_circuit: Option<(bool, Reply<bool>)>,
|
||||
answer: Option<Py<PyAny>>,
|
||||
answered: Option<Answered>,
|
||||
stream_opened: bool,
|
||||
}
|
||||
|
||||
impl MessagesPythonHost {
|
||||
|
|
@ -354,6 +424,7 @@ impl MessagesPythonHost {
|
|||
pending_short_circuit: None,
|
||||
answer: None,
|
||||
answered: None,
|
||||
stream_opened: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -550,32 +621,41 @@ impl MessagesPythonHost {
|
|||
.call1((&model, api_base.as_deref()))?
|
||||
.extract()?,
|
||||
};
|
||||
let filters_beta: bool = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("filters_beta_headers")?
|
||||
.call1((&model, custom_llm_provider.as_deref(), api_base.as_deref()))?
|
||||
.extract()?;
|
||||
let (extra_headers, mut unencodable_request_header) = match settings.extra_headers {
|
||||
Some(headers) => (Some(headers), None),
|
||||
None => self.merged_headers(
|
||||
py,
|
||||
arguments,
|
||||
custom_llm_provider.as_deref() == Some(AZURE_AI_PROVIDER),
|
||||
)?,
|
||||
None => self.merged_headers(py, arguments, filters_beta)?,
|
||||
};
|
||||
let provider_specific_header = self.provider_specific_header(
|
||||
py,
|
||||
arguments,
|
||||
custom_llm_provider.as_deref(),
|
||||
filters_beta,
|
||||
&mut unencodable_request_header,
|
||||
)?;
|
||||
let azure = match custom_llm_provider.as_deref() {
|
||||
Some(AZURE_AI_PROVIDER) => Some(azure_module_settings(|name| module_global(py, name))?),
|
||||
_ => None,
|
||||
};
|
||||
let bedrock = custom_llm_provider.as_deref() == Some(BEDROCK_PROVIDER);
|
||||
let mut capabilities = self.capabilities(py, &model, custom_llm_provider.as_deref())?;
|
||||
if bedrock {
|
||||
capabilities.bedrock =
|
||||
super::model_catalog::bedrock_capabilities(py, &model, argument("model_info")?)?;
|
||||
}
|
||||
let shaping = MessagesShaping {
|
||||
capabilities: self.capabilities(py, &model, custom_llm_provider.as_deref())?,
|
||||
capabilities,
|
||||
settings: MessagesSettings {
|
||||
whole_sse_frames: settings.stream_cost_model.is_some(),
|
||||
presanitized,
|
||||
intercepted: self.may_intercept(py)?,
|
||||
unencodable_request_header,
|
||||
module_api_key: azure.as_ref().and_then(|azure| azure.api_key.clone()),
|
||||
global_drop_params: global_drop_params(py)?,
|
||||
..settings.settings
|
||||
},
|
||||
};
|
||||
|
|
@ -587,47 +667,42 @@ impl MessagesPythonHost {
|
|||
Some(AZURE_AI_PROVIDER) => ENTRA_ID_PARAMS.to_vec(),
|
||||
_ => Vec::new(),
|
||||
};
|
||||
let settled = match azure {
|
||||
Some(azure) => azure_settled_params(azure, argument(CALLER_TOKEN_PROVIDER_PARAM)?)?,
|
||||
None => Map::new(),
|
||||
let settled = match (azure, bedrock) {
|
||||
(Some(azure), _) => {
|
||||
azure_settled_params(azure, argument(CALLER_TOKEN_PROVIDER_PARAM)?)?
|
||||
}
|
||||
(None, true) => bedrock_litellm_params(py, argument)?,
|
||||
(None, false) => Map::new(),
|
||||
};
|
||||
let litellm_params = project_litellm_params(provider_fields, settled, argument, |name| {
|
||||
module_global(py, name)
|
||||
})?;
|
||||
Ok(
|
||||
messages_body(body, custom_llm_provider.as_deref()).and_then(|body| {
|
||||
Ok(MessagesCall {
|
||||
body,
|
||||
api_key,
|
||||
api_base,
|
||||
extra_headers,
|
||||
provider_specific_header,
|
||||
custom_llm_provider,
|
||||
litellm_params: litellm_params?,
|
||||
timeout: optional_timeout(timeout),
|
||||
shaping,
|
||||
})
|
||||
}),
|
||||
)
|
||||
Ok(messages_body(body).and_then(|body| {
|
||||
Ok(MessagesCall {
|
||||
body,
|
||||
api_key,
|
||||
api_base,
|
||||
extra_headers,
|
||||
provider_specific_header,
|
||||
custom_llm_provider,
|
||||
litellm_params: litellm_params?,
|
||||
timeout: optional_timeout(timeout),
|
||||
shaping,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
fn merged_headers(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
arguments: &Bound<'_, PyDict>,
|
||||
provider_filters_beta: bool,
|
||||
filters_beta: bool,
|
||||
) -> PyResult<MergedHeaders> {
|
||||
let mut unencodable = None;
|
||||
let mut mapping = |name: &str| -> PyResult<Option<Map<String, Value>>> {
|
||||
self.view
|
||||
.argument(py, arguments, name)?
|
||||
.map(|value| {
|
||||
from_py(&ascii_headers(
|
||||
&value,
|
||||
&mut unencodable,
|
||||
provider_filters_beta,
|
||||
)?)
|
||||
})
|
||||
.map(|value| from_py(&ascii_headers(&value, &mut unencodable, filters_beta)?))
|
||||
.transpose()
|
||||
};
|
||||
let headers = mapping("headers")?;
|
||||
|
|
@ -640,6 +715,7 @@ impl MessagesPythonHost {
|
|||
py: Python<'_>,
|
||||
arguments: &Bound<'_, PyDict>,
|
||||
provider: Option<&str>,
|
||||
filters_beta: bool,
|
||||
unencodable: &mut Option<Vec<u8>>,
|
||||
) -> PyResult<Option<ProviderSpecificHeaders>> {
|
||||
let Some(value) = self
|
||||
|
|
@ -655,13 +731,9 @@ impl MessagesPythonHost {
|
|||
.getattr("ProviderSpecificHeaderUtils")?
|
||||
.call_method1("get_provider_specific_headers", (&value, provider, true))
|
||||
{
|
||||
ascii_headers(&scoped, unencodable, provider == AZURE_AI_PROVIDER)?;
|
||||
ascii_headers(&scoped, unencodable, filters_beta)?;
|
||||
}
|
||||
from_py(&ascii_scoped_headers(
|
||||
&value,
|
||||
provider == Some(AZURE_AI_PROVIDER),
|
||||
)?)
|
||||
.map(Some)
|
||||
from_py(&ascii_scoped_headers(&value, filters_beta)?).map(Some)
|
||||
}
|
||||
|
||||
fn capabilities(
|
||||
|
|
@ -723,6 +795,89 @@ impl MessagesPythonHost {
|
|||
Err(_) => error,
|
||||
}
|
||||
}
|
||||
|
||||
fn bedrock_stream_raise(&self, py: Python<'_>, raise: &BedrockStreamRaise) -> PyResult<PyErr> {
|
||||
let (kind, arguments) = match raise {
|
||||
BedrockStreamRaise::ChecksumMismatch {
|
||||
expected,
|
||||
calculated,
|
||||
} => (
|
||||
"ChecksumMismatch",
|
||||
serde_json::json!([expected, calculated]),
|
||||
),
|
||||
BedrockStreamRaise::InvalidHeadersLength(length) => {
|
||||
("InvalidHeadersLength", serde_json::json!([length]))
|
||||
}
|
||||
BedrockStreamRaise::InvalidPayloadLength(length) => {
|
||||
("InvalidPayloadLength", serde_json::json!([length]))
|
||||
}
|
||||
BedrockStreamRaise::DuplicateHeader(name) => {
|
||||
("DuplicateHeader", serde_json::json!([name]))
|
||||
}
|
||||
BedrockStreamRaise::InvalidJson(text) => ("InvalidJson", serde_json::json!([text])),
|
||||
BedrockStreamRaise::NotAnObject(chunk) => ("NotAnObject", serde_json::json!([chunk])),
|
||||
};
|
||||
let raised = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("bedrock_stream_raise")?
|
||||
.call1((kind, to_py(py, &arguments)?))?;
|
||||
Ok(PyErr::from_value(raised))
|
||||
}
|
||||
|
||||
fn mapped_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
|
||||
if let Some(raise) = litellm_error::find::<BedrockStreamRaise>(&error) {
|
||||
return self.bedrock_stream_raise(py, raise);
|
||||
}
|
||||
if let Some((raised, raw)) = python_failure(py, &error) {
|
||||
return if raw {
|
||||
Ok(raised)
|
||||
} else {
|
||||
self.map_transport_failure(py, raised)
|
||||
};
|
||||
}
|
||||
if let Some(raised) = super::workload_identity::python_error(py, &error)
|
||||
.or_else(|| super::workload_identity::entra_token_error(py, &error))
|
||||
{
|
||||
let mapped = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("map_token_failure")?
|
||||
.call1((
|
||||
raised.value(py),
|
||||
self.view.request(py)?,
|
||||
self.provider(py),
|
||||
self.asynchronous,
|
||||
))?;
|
||||
return Ok(PyErr::from_value(mapped));
|
||||
}
|
||||
if let Some(unencodable) = litellm_error::find::<UnencodableHeader>(&error) {
|
||||
let (value, received) = match unencodable {
|
||||
UnencodableHeader::Sent(value) => (value, false),
|
||||
UnencodableHeader::Received(value) => (value, true),
|
||||
};
|
||||
let raised = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("unencodable_header")?
|
||||
.call1((PyBytes::new(py, value), received))?;
|
||||
return self.map_transport_failure(py, PyErr::from_value(raised));
|
||||
}
|
||||
if let Some(undecodable) = litellm_error::find::<DecodingError>(&error) {
|
||||
let raised = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("decoding_failure")?
|
||||
.call1((
|
||||
undecodable.to_string(),
|
||||
self.view.request(py)?,
|
||||
self.provider(py),
|
||||
))?;
|
||||
return Ok(PyErr::from_value(raised));
|
||||
}
|
||||
route_failure(py, error, RequestMarker::Messages, |native| {
|
||||
if self.stream_opened {
|
||||
native.value(py).setattr(MESSAGES_STREAM_FAILURE, true)?;
|
||||
}
|
||||
Ok(self.map_failure(py, native))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl MessagesPythonHost {
|
||||
|
|
@ -839,6 +994,9 @@ impl PythonBinding for MessagesPythonHost {
|
|||
return Ok(answer);
|
||||
}
|
||||
let public = public_json_response(py, ROUTE_HOST_MODULE, &response)?;
|
||||
py.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("record_guardrail")?
|
||||
.call1((&public, self.view.request(py)?))?;
|
||||
self.safeguards.published(py, public)
|
||||
}
|
||||
|
||||
|
|
@ -859,15 +1017,12 @@ impl PythonBinding for MessagesPythonHost {
|
|||
py: Python<'_>,
|
||||
head: MessagesStreamHead,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
self.stream_opened = true;
|
||||
let module = py.import(ROUTE_HOST_MODULE)?;
|
||||
let head = module
|
||||
.getattr("stream_hidden_params")?
|
||||
.call1((to_py(py, &head.headers)?,))?;
|
||||
if let Some(arguments) = self
|
||||
.arguments
|
||||
.as_ref()
|
||||
.filter(|_| self.asynchronous && self.provider(py) == AZURE_AI_PROVIDER)
|
||||
{
|
||||
.call1((to_py(py, &head.headers)?, head.unobserved_tail.is_some()))?;
|
||||
if let Some(arguments) = self.arguments.as_ref().filter(|_| self.asynchronous) {
|
||||
module
|
||||
.getattr("python_stream_types")?
|
||||
.call1((&head, arguments))?;
|
||||
|
|
@ -896,52 +1051,15 @@ impl PythonBinding for MessagesPythonHost {
|
|||
}
|
||||
|
||||
fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
|
||||
if let Some((raised, raw)) = python_failure(py, &error) {
|
||||
return if raw {
|
||||
Ok(raised)
|
||||
} else {
|
||||
self.map_transport_failure(py, raised)
|
||||
};
|
||||
let mapped = self.mapped_error(py, error)?;
|
||||
if !self.stream_opened {
|
||||
return Ok(mapped);
|
||||
}
|
||||
if let Some(raised) = super::workload_identity::python_error(py, &error)
|
||||
.or_else(|| super::workload_identity::entra_token_error(py, &error))
|
||||
{
|
||||
let mapped = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("map_token_failure")?
|
||||
.call1((
|
||||
raised.value(py),
|
||||
self.view.request(py)?,
|
||||
self.provider(py),
|
||||
self.asynchronous,
|
||||
))?;
|
||||
return Ok(PyErr::from_value(mapped));
|
||||
}
|
||||
if let Some(unencodable) = litellm_error::find::<UnencodableHeader>(&error) {
|
||||
let (value, received) = match unencodable {
|
||||
UnencodableHeader::Sent(value) => (value, false),
|
||||
UnencodableHeader::Received(value) => (value, true),
|
||||
};
|
||||
let raised = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("unencodable_header")?
|
||||
.call1((PyBytes::new(py, value), received))?;
|
||||
return self.map_transport_failure(py, PyErr::from_value(raised));
|
||||
}
|
||||
if let Some(undecodable) = litellm_error::find::<DecodingError>(&error) {
|
||||
let raised = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("decoding_failure")?
|
||||
.call1((
|
||||
undecodable.to_string(),
|
||||
self.view.request(py)?,
|
||||
self.provider(py),
|
||||
))?;
|
||||
return Ok(PyErr::from_value(raised));
|
||||
}
|
||||
route_failure(py, error, RequestMarker::Messages, |native| {
|
||||
Ok(self.map_failure(py, native))
|
||||
})
|
||||
let described = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("stream_failure")?
|
||||
.call1((mapped.value(py), self.view.request(py)?, self.provider(py)))?;
|
||||
Ok(PyErr::from_value(described))
|
||||
}
|
||||
|
||||
fn host_error(error: &PyErr) -> Error {
|
||||
|
|
@ -1115,7 +1233,7 @@ mod tests {
|
|||
)]
|
||||
fn a_header_python_cannot_encode_is_held_back_unless_the_provider_filters_it(
|
||||
#[case] headers: &std::ffi::CStr,
|
||||
#[case] provider_filters_beta: bool,
|
||||
#[case] filters_beta: bool,
|
||||
#[case] kept: Value,
|
||||
#[case] unencodable: Option<Vec<u8>>,
|
||||
) {
|
||||
|
|
@ -1124,8 +1242,7 @@ mod tests {
|
|||
let headers = py.eval(headers, None, None).unwrap();
|
||||
let mut held_back = None;
|
||||
let sent: Value =
|
||||
from_py(&ascii_headers(&headers, &mut held_back, provider_filters_beta).unwrap())
|
||||
.unwrap();
|
||||
from_py(&ascii_headers(&headers, &mut held_back, filters_beta).unwrap()).unwrap();
|
||||
assert_eq!((sent, held_back), (kept, unencodable));
|
||||
});
|
||||
}
|
||||
|
|
@ -1143,6 +1260,14 @@ mod tests {
|
|||
json!({"model": "m", "vertex_project": "p", "vertex_ai_location": "us-east5", "vertex_credentials": {"type": "service_account"}}),
|
||||
Ok(json!({"model": "m", "vertex_project": "p", "vertex_ai_location": "us-east5", "vertex_credentials": {"type": "service_account"}})),
|
||||
)]
|
||||
#[case::aws_session_tags_are_projected(
|
||||
json!({"model": "m", "aws_role_name": "r", "aws_session_tags": [{"Key": "team", "Value": "genai"}]}),
|
||||
Ok(json!({"model": "m", "aws_role_name": "r", "aws_session_tags": [{"Key": "team", "Value": "genai"}]})),
|
||||
)]
|
||||
#[case::malformed_aws_session_tags_are_a_request_error(
|
||||
json!({"model": "m", "aws_session_tags": [{"Key": "team"}]}),
|
||||
Err(()),
|
||||
)]
|
||||
#[case::an_explicit_none_is_absent(json!({"model": "m", "aws_region_name": null}), Ok(json!({"model": "m"})))]
|
||||
#[case::a_wrong_type_is_a_request_error(json!({"model": "m", "aws_region_name": 7}), Err(()))]
|
||||
#[case::a_missing_model_is_a_request_error(json!({"aws_region_name": "eu-central-1"}), Err(()))]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
mod agentic;
|
||||
mod answer;
|
||||
mod host;
|
||||
mod model_catalog;
|
||||
mod modify_params;
|
||||
mod pre_request;
|
||||
mod requests_reason;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
use litellm_host_python::json_from_py;
|
||||
use litellm_llms::{
|
||||
anthropic::model_capabilities::bedrock_model_capabilities,
|
||||
base_llm::messages::context::BedrockCapabilities,
|
||||
};
|
||||
use litellm_model_catalog::Catalog;
|
||||
use pyo3::{
|
||||
exceptions::PyValueError,
|
||||
prelude::*,
|
||||
types::{PyBool, PyMapping},
|
||||
};
|
||||
|
||||
use super::host::MODEL_CAPABILITIES_MODULE;
|
||||
|
||||
type Version = (usize, usize, u64);
|
||||
|
||||
static LIVE: Mutex<Option<(Version, Arc<Catalog>)>> = Mutex::new(None);
|
||||
|
||||
fn live_catalog(py: Python<'_>) -> PyResult<Arc<Catalog>> {
|
||||
let known = LIVE.lock().unwrap_or_else(PoisonError::into_inner).clone();
|
||||
let (version, document): (Version, Option<Bound<'_, PyAny>>) = py
|
||||
.import(MODEL_CAPABILITIES_MODULE)?
|
||||
.getattr("model_cost_document")?
|
||||
.call1((known.as_ref().map(|(version, _)| *version),))?
|
||||
.extract()?;
|
||||
let Some(document) = document else {
|
||||
return known
|
||||
.map(|(_, catalog)| catalog)
|
||||
.ok_or_else(|| PyValueError::new_err("the model map came back without a document"));
|
||||
};
|
||||
let body = serde_json::to_vec(&json_from_py(&document)?)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
let catalog = Arc::new(
|
||||
py.detach(|| Catalog::parse(&body))
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?
|
||||
.value,
|
||||
);
|
||||
*LIVE.lock().unwrap_or_else(PoisonError::into_inner) = Some((version, Arc::clone(&catalog)));
|
||||
Ok(catalog)
|
||||
}
|
||||
|
||||
pub(super) fn bedrock_capabilities(
|
||||
py: Python<'_>,
|
||||
model: &str,
|
||||
model_info: Option<Bound<'_, PyAny>>,
|
||||
) -> PyResult<BedrockCapabilities> {
|
||||
let model_info = model_info.and_then(|info| info.cast_into::<PyMapping>().ok());
|
||||
let declared = |name: &str| {
|
||||
model_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.get_item(name).ok())
|
||||
.and_then(|value| value.cast_into::<PyBool>().ok())
|
||||
.map(|flag| flag.is_true())
|
||||
};
|
||||
Ok(bedrock_model_capabilities(
|
||||
&*live_catalog(py)?,
|
||||
model,
|
||||
&declared,
|
||||
))
|
||||
}
|
||||
238
litellm/rust/crates/python-bridge/src/secrets/local.rs
Normal file
238
litellm/rust/crates/python-bridge/src/secrets/local.rs
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
use std::{
|
||||
borrow::Cow,
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use futures_util::future::{BoxFuture, FutureExt, try_join_all};
|
||||
use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup};
|
||||
use litellm_secret_value::SecretValue;
|
||||
use litellm_secrets::{
|
||||
Error,
|
||||
source::{SecretSource, Secrets},
|
||||
};
|
||||
|
||||
const ENVIRONMENT_PREFIX: &str = "os.environ/";
|
||||
const OIDC_PREFIX: &str = "oidc/";
|
||||
|
||||
/// Python's `get_secret_str` while no secret manager is readable: names come from the process
|
||||
/// environment, read in Rust when the route looks them up, and `oidc/` references go to `python`.
|
||||
pub(super) struct LocalSecrets {
|
||||
python: Arc<dyn SecretSource>,
|
||||
}
|
||||
|
||||
impl LocalSecrets {
|
||||
pub(super) fn new(python: Arc<dyn SecretSource>) -> Self {
|
||||
Self { python }
|
||||
}
|
||||
}
|
||||
|
||||
fn python_name(name: &str) -> Cow<'_, str> {
|
||||
if name.starts_with(ENVIRONMENT_PREFIX) {
|
||||
Cow::Owned(name.replace(ENVIRONMENT_PREFIX, ""))
|
||||
} else {
|
||||
Cow::Borrowed(name)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_oidc(name: &str) -> bool {
|
||||
python_name(name).starts_with(OIDC_PREFIX)
|
||||
}
|
||||
|
||||
fn environment_str(name: &str) -> Option<SecretValue> {
|
||||
let name = python_name(name);
|
||||
if name.contains(['=', '\0']) {
|
||||
return None;
|
||||
}
|
||||
std::env::var(name.as_ref())
|
||||
.ok()
|
||||
.filter(|value| parse_str_bool(value).is_none())
|
||||
.map(SecretValue::new)
|
||||
}
|
||||
|
||||
impl SecretSource for LocalSecrets {
|
||||
fn get_secret_str<'a>(
|
||||
&'a self,
|
||||
name: &'a str,
|
||||
) -> BoxFuture<'a, Result<Option<SecretValue>, Error>> {
|
||||
if is_oidc(name) {
|
||||
return self.python.get_secret_str(name);
|
||||
}
|
||||
futures_util::future::ready(Ok(environment_str(name))).boxed()
|
||||
}
|
||||
|
||||
fn resolve<'a>(&'a self, names: &'a [&str]) -> BoxFuture<'a, Result<Secrets, Error>> {
|
||||
Box::pin(async move {
|
||||
let (oidc, environment): (Vec<&str>, Vec<&str>) =
|
||||
names.iter().copied().partition(|name| is_oidc(name));
|
||||
let oidc = try_join_all(oidc.into_iter().map(|name| async move {
|
||||
self.python
|
||||
.get_secret_str(name)
|
||||
.await
|
||||
.map(|value| (name.to_owned(), value))
|
||||
}))
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect();
|
||||
Ok(Arc::new(LocalLookup {
|
||||
environment: environment.into_iter().map(str::to_owned).collect(),
|
||||
oidc,
|
||||
}) as Secrets)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct LocalLookup {
|
||||
environment: HashSet<String>,
|
||||
oidc: HashMap<String, Option<SecretValue>>,
|
||||
}
|
||||
|
||||
impl Lookup for LocalLookup {
|
||||
fn get(&self, name: &str) -> Option<String> {
|
||||
let value = if self.environment.contains(name) {
|
||||
environment_str(name)
|
||||
} else {
|
||||
self.oidc.get(name).cloned().flatten()
|
||||
};
|
||||
value.map(|value| value.expose().to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use futures_util::future::{BoxFuture, FutureExt};
|
||||
use litellm_secret_value::SecretValue;
|
||||
use litellm_secrets::{Error, source::SecretSource};
|
||||
use rstest::rstest;
|
||||
|
||||
use super::LocalSecrets;
|
||||
|
||||
#[derive(Default)]
|
||||
struct Recording(Mutex<Vec<String>>);
|
||||
|
||||
impl Recording {
|
||||
fn reads(&self) -> Vec<String> {
|
||||
self.0.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl SecretSource for Recording {
|
||||
fn get_secret_str<'a>(
|
||||
&'a self,
|
||||
name: &'a str,
|
||||
) -> BoxFuture<'a, Result<Option<SecretValue>, Error>> {
|
||||
self.0.lock().unwrap().push(name.to_owned());
|
||||
futures_util::future::ready(Ok(Some(SecretValue::new(format!("python-{name}")))))
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
fn local() -> (LocalSecrets, Arc<Recording>) {
|
||||
let python = Arc::new(Recording::default());
|
||||
(
|
||||
LocalSecrets::new(Arc::clone(&python) as Arc<dyn SecretSource>),
|
||||
python,
|
||||
)
|
||||
}
|
||||
|
||||
fn set(name: &str, value: Option<&str>) {
|
||||
match value {
|
||||
Some(value) => unsafe { std::env::set_var(name, value) },
|
||||
None => unsafe { std::env::remove_var(name) },
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::plain(
|
||||
"LITELLM_LOCAL_PLAIN",
|
||||
"LITELLM_LOCAL_PLAIN",
|
||||
Some("value"),
|
||||
Some("value")
|
||||
)]
|
||||
#[case::empty("LITELLM_LOCAL_EMPTY", "LITELLM_LOCAL_EMPTY", Some(""), Some(""))]
|
||||
#[case::unset("LITELLM_LOCAL_UNSET", "LITELLM_LOCAL_UNSET", None, None)]
|
||||
#[case::reference(
|
||||
"LITELLM_LOCAL_REFERENCE",
|
||||
"os.environ/LITELLM_LOCAL_REFERENCE",
|
||||
Some("value"),
|
||||
Some("value")
|
||||
)]
|
||||
#[case::every_marker_removed(
|
||||
"LITELLM_LOCAL_TWICE",
|
||||
"os.environ/os.environ/LITELLM_LOCAL_TWICE",
|
||||
Some("value"),
|
||||
Some("value")
|
||||
)]
|
||||
#[case::marker_inside_only_when_prefixed(
|
||||
"LITELLM_LOCAL_INNER",
|
||||
"LITELLM_LOCAL_os.environ/INNER",
|
||||
Some("value"),
|
||||
None
|
||||
)]
|
||||
#[case::boolean_true("LITELLM_LOCAL_TRUE", "LITELLM_LOCAL_TRUE", Some(" TRUE\n"), None)]
|
||||
#[case::boolean_false("LITELLM_LOCAL_FALSE", "LITELLM_LOCAL_FALSE", Some("False"), None)]
|
||||
#[case::boolean_like("LITELLM_LOCAL_ONE", "LITELLM_LOCAL_ONE", Some("1"), Some("1"))]
|
||||
#[case::equals_in_name("LITELLM_LOCAL_EQ", "LITELLM_LOCAL_EQ=x", Some("x=y"), None)]
|
||||
#[tokio::test]
|
||||
async fn reads_what_get_secret_str_reads_from_the_environment(
|
||||
#[case] variable: &str,
|
||||
#[case] name: &str,
|
||||
#[case] value: Option<&str>,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
set(variable, value);
|
||||
let (secrets, python) = local();
|
||||
|
||||
let read = secrets.get_secret_str(name).await.unwrap();
|
||||
let resolved = secrets.resolve(&[name]).await.unwrap().get(name);
|
||||
set(variable, None);
|
||||
|
||||
assert_eq!(read.as_ref().map(SecretValue::expose), expected);
|
||||
assert_eq!(resolved.as_deref(), expected);
|
||||
assert!(python.reads().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::direct("oidc/env/LITELLM_LOCAL_TOKEN")]
|
||||
#[case::referenced("os.environ/oidc/env/LITELLM_LOCAL_TOKEN")]
|
||||
#[tokio::test]
|
||||
async fn oidc_references_are_left_to_python(#[case] name: &str) {
|
||||
let (secrets, python) = local();
|
||||
|
||||
let read = secrets.get_secret_str(name).await.unwrap();
|
||||
let resolved = secrets
|
||||
.resolve(&[name, "LITELLM_LOCAL_BESIDE"])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let expected = format!("python-{name}");
|
||||
assert_eq!(
|
||||
read.as_ref().map(SecretValue::expose),
|
||||
Some(expected.as_str())
|
||||
);
|
||||
assert_eq!(resolved.get(name).as_deref(), Some(expected.as_str()));
|
||||
assert_eq!(python.reads(), vec![name.to_owned(), name.to_owned()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolved_names_are_read_when_looked_up_and_others_are_absent() {
|
||||
let name = "LITELLM_LOCAL_LAZY";
|
||||
let other = "LITELLM_LOCAL_NOT_RESOLVED";
|
||||
set(name, None);
|
||||
set(other, Some("present"));
|
||||
let (secrets, python) = local();
|
||||
|
||||
let resolved = secrets.resolve(&[name]).await.unwrap();
|
||||
set(name, Some("set-after-resolve"));
|
||||
let looked_up = resolved.get(name);
|
||||
let unrequested = resolved.get(other);
|
||||
set(name, None);
|
||||
set(other, None);
|
||||
|
||||
assert_eq!(looked_up.as_deref(), Some("set-after-resolve"));
|
||||
assert_eq!(unrequested, None);
|
||||
assert!(python.reads().is_empty());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
pub(crate) mod callback;
|
||||
pub(crate) mod config;
|
||||
mod error;
|
||||
mod local;
|
||||
mod mutation;
|
||||
mod operations;
|
||||
mod provider;
|
||||
|
|
@ -9,6 +10,7 @@ pub(crate) mod resolved;
|
|||
pub(crate) mod runtime;
|
||||
pub(crate) mod seeded;
|
||||
mod vault;
|
||||
mod web_identity;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
|
|
@ -16,33 +18,98 @@ use std::sync::Arc;
|
|||
pub(crate) use error::read_error;
|
||||
pub(crate) use error::{ManagerError, python_error};
|
||||
use litellm_secrets::source::{EnvironmentSecrets, SecretSource};
|
||||
use local::LocalSecrets;
|
||||
use pyo3::prelude::*;
|
||||
use python::PythonSecrets;
|
||||
use resolved::ResolvedSecrets;
|
||||
pub(crate) use web_identity::HostWebIdentityTokens;
|
||||
|
||||
use crate::{coercion::FieldSpec, python_settings::PythonSettings};
|
||||
use crate::{
|
||||
coercion::FieldSpec,
|
||||
python_settings::{PythonSettings, Snapshot},
|
||||
};
|
||||
|
||||
const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager";
|
||||
const READABLE: FieldSpec<bool> = FieldSpec::new("readable", |field| field.schema_bool());
|
||||
const NATIVE: FieldSpec<bool> = FieldSpec::new("native", |field| field.schema_bool());
|
||||
|
||||
/// Where a Rust route reads provider secrets from. Python's `get_secret_str` when the settings
|
||||
/// projection reports no configured secret manager, otherwise the resolved manager: the native
|
||||
/// backend when the configured client captures one, the Python callback when it does not. A bare
|
||||
/// extension module without the litellm package reads the process environment.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum Selection {
|
||||
Local,
|
||||
Native,
|
||||
Python,
|
||||
}
|
||||
|
||||
fn select(snapshot: &Snapshot<'_>) -> PyResult<Selection> {
|
||||
Ok(if !snapshot.read(&READABLE)? {
|
||||
Selection::Local
|
||||
} else if snapshot.read(&NATIVE)? {
|
||||
Selection::Native
|
||||
} else {
|
||||
Selection::Python
|
||||
})
|
||||
}
|
||||
|
||||
/// Where a Rust route reads provider secrets from, decided per call from the settings projection.
|
||||
/// With no readable secret manager Python's `get_secret_str` only reads the environment, so the
|
||||
/// route reads it in Rust. A readable manager is the native backend when the configured client
|
||||
/// captures one, otherwise every read goes through Python's `get_secret_str`. A bare extension
|
||||
/// module without the litellm package reads the process environment.
|
||||
pub(crate) fn source(py: Python<'_>) -> PyResult<Arc<dyn SecretSource>> {
|
||||
let Some(snapshot) = PythonSettings::SecretManager.read_or_unset(py)? else {
|
||||
return Ok(Arc::new(EnvironmentSecrets::python_compatible(
|
||||
crate::http::host_client(py, litellm_http::ClientVariant::Provider)?,
|
||||
)));
|
||||
};
|
||||
if snapshot.read(&NATIVE)? {
|
||||
let context = litellm_host_python::PythonContext::capture(py)?;
|
||||
let client = crate::http::host_client(py, litellm_http::ClientVariant::Provider)?;
|
||||
return Ok(Arc::new(ResolvedSecrets::new(
|
||||
config::read(py)?,
|
||||
context,
|
||||
client,
|
||||
)));
|
||||
match select(&snapshot)? {
|
||||
Selection::Local => Ok(Arc::new(LocalSecrets::new(Arc::new(PythonSecrets::new(
|
||||
py,
|
||||
)?)))),
|
||||
Selection::Native => {
|
||||
let context = litellm_host_python::PythonContext::capture(py)?;
|
||||
let client = crate::http::host_client(py, litellm_http::ClientVariant::Provider)?;
|
||||
Ok(Arc::new(ResolvedSecrets::new(
|
||||
config::read(py)?,
|
||||
context,
|
||||
client,
|
||||
)))
|
||||
}
|
||||
Selection::Python => Ok(Arc::new(PythonSecrets::new(py)?)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
use rstest::rstest;
|
||||
|
||||
use super::{Selection, select};
|
||||
use crate::python_settings::PythonSettings;
|
||||
|
||||
#[rstest]
|
||||
#[case::no_manager(false, false, Selection::Local)]
|
||||
#[case::unreadable_native(false, true, Selection::Local)]
|
||||
#[case::python_manager(true, false, Selection::Python)]
|
||||
#[case::native_manager(true, true, Selection::Native)]
|
||||
fn the_projection_selects_where_secrets_are_read(
|
||||
#[case] readable: bool,
|
||||
#[case] native: bool,
|
||||
#[case] expected: Selection,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("readable", readable).unwrap();
|
||||
locals.set_item("native", native).unwrap();
|
||||
let value = py
|
||||
.eval(
|
||||
c"__import__('types').SimpleNamespace(readable=readable, native=native)",
|
||||
None,
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let selected = select(&PythonSettings::SecretManager.snapshot(value)).unwrap();
|
||||
assert_eq!(selected, expected);
|
||||
});
|
||||
}
|
||||
Ok(Arc::new(PythonSecrets::new(py)?))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,12 @@ use std::{
|
|||
};
|
||||
|
||||
use futures_util::future::{BoxFuture, FutureExt};
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_secret_value::SecretValue;
|
||||
use litellm_secrets::{Error, source::SecretSource};
|
||||
use litellm_secrets::{
|
||||
Error,
|
||||
source::{SecretSource, Secrets},
|
||||
};
|
||||
|
||||
/// A call's secret source that answers the secrets the host already read for the call with the
|
||||
/// values it read, so a secret Python reads once is not read a second time.
|
||||
|
|
@ -49,6 +53,39 @@ impl SecretSource for SeededSecrets {
|
|||
None => self.source.get_secret_str(name),
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve<'a>(&'a self, names: &'a [&str]) -> BoxFuture<'a, Result<Secrets, Error>> {
|
||||
let (seeded, unseeded) = {
|
||||
let read = self.read.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let (seeded, unseeded): (Vec<&str>, Vec<&str>) = names
|
||||
.iter()
|
||||
.copied()
|
||||
.partition(|name| read.contains_key(*name));
|
||||
let seeded: HashMap<String, Option<SecretValue>> = seeded
|
||||
.into_iter()
|
||||
.map(|name| (name.to_owned(), read.get(name).cloned().flatten()))
|
||||
.collect();
|
||||
(seeded, unseeded)
|
||||
};
|
||||
Box::pin(async move {
|
||||
let source = self.source.resolve(&unseeded).await?;
|
||||
Ok(Arc::new(SeededLookup { seeded, source }) as Secrets)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct SeededLookup {
|
||||
seeded: HashMap<String, Option<SecretValue>>,
|
||||
source: Secrets,
|
||||
}
|
||||
|
||||
impl Lookup for SeededLookup {
|
||||
fn get(&self, name: &str) -> Option<String> {
|
||||
match self.seeded.get(name) {
|
||||
Some(value) => value.as_ref().map(|value| value.expose().to_owned()),
|
||||
None => self.source.get(name),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -98,4 +135,49 @@ mod tests {
|
|||
);
|
||||
assert_eq!(source.0.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolve_answers_seeded_names_and_leaves_the_rest_to_the_source() {
|
||||
let source = Arc::new(Counting::default());
|
||||
let secrets = SeededSecrets::new(Arc::clone(&source) as Arc<dyn SecretSource>);
|
||||
secrets.seed([
|
||||
(
|
||||
"AZURE_API_BASE".to_owned(),
|
||||
Some("http://seeded".to_owned()),
|
||||
),
|
||||
("UNSET".to_owned(), None),
|
||||
]);
|
||||
|
||||
let resolved = secrets
|
||||
.resolve(&["AZURE_API_BASE", "UNSET", "OTHER"])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.get("AZURE_API_BASE").as_deref(),
|
||||
Some("http://seeded")
|
||||
);
|
||||
assert_eq!(resolved.get("UNSET"), None);
|
||||
assert_eq!(resolved.get("OTHER").as_deref(), Some("source-OTHER"));
|
||||
assert_eq!(resolved.get("NOT_REQUESTED"), None);
|
||||
assert_eq!(source.0.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolve_keeps_a_lazy_source_lazy() {
|
||||
let name = "LITELLM_SEEDED_LAZY";
|
||||
unsafe { std::env::remove_var(name) };
|
||||
let python = Arc::new(Counting::default());
|
||||
let secrets = SeededSecrets::new(Arc::new(crate::secrets::local::LocalSecrets::new(
|
||||
Arc::clone(&python) as Arc<dyn SecretSource>,
|
||||
)));
|
||||
|
||||
let resolved = secrets.resolve(&[name]).await.unwrap();
|
||||
unsafe { std::env::set_var(name, "set-after-resolve") };
|
||||
let looked_up = resolved.get(name);
|
||||
unsafe { std::env::remove_var(name) };
|
||||
|
||||
assert_eq!(looked_up.as_deref(), Some("set-after-resolve"));
|
||||
assert_eq!(python.0.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
use litellm_auth_aws::{WebIdentityTokenFuture, WebIdentityTokenSource};
|
||||
use litellm_error::Source;
|
||||
use litellm_host_python::{PythonContext, attach_blocking};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use super::error::external_error;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct HostWebIdentityTokens;
|
||||
|
||||
impl WebIdentityTokenSource for HostWebIdentityTokens {
|
||||
fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a> {
|
||||
Box::pin(async move {
|
||||
let context = Python::attach(|py| {
|
||||
PythonContext::empty(py).map_err(|error| external_error(py, error))
|
||||
})
|
||||
.map_err(Source::new)?;
|
||||
let source = match attach_blocking(context, |py| {
|
||||
super::source(py).map_err(|error| external_error(py, error))
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(source) => source,
|
||||
Err(error) => Err(Python::attach(|py| external_error(py, error))),
|
||||
}
|
||||
.map_err(Source::new)?;
|
||||
source.get_secret_str(reference).await.map_err(Source::new)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -126,7 +126,10 @@ fn a_param_of_the_wrong_type_is_rejected(#[case] value: Value) {
|
|||
#[rstest]
|
||||
fn fields_fill_the_model_the_credentials_and_every_group() {
|
||||
let filled: Value = LitellmParams::fields()
|
||||
.map(|name| (name.to_string(), Value::from(name)))
|
||||
.map(|name| match name {
|
||||
AwsParams::SESSION_TAGS => (name.to_string(), json!([{"Key": "k", "Value": "v"}])),
|
||||
_ => (name.to_string(), Value::from(name)),
|
||||
})
|
||||
.collect::<Map<_, _>>()
|
||||
.into();
|
||||
let typed = params(filled);
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ impl Credentials {
|
|||
profile_name: settings.aws_profile_name.clone(),
|
||||
web_identity_token: settings.aws_web_identity_token.as_ref().cloned(),
|
||||
sts_endpoint: settings.aws_sts_endpoint.clone(),
|
||||
session_tags: None,
|
||||
session_policy: None,
|
||||
},
|
||||
environment,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,9 +30,14 @@ class Target(str, Enum):
|
|||
MESSAGES = "messages"
|
||||
MESSAGES_PASSTHROUGH = "messages_passthrough"
|
||||
"""A deployment's own /v1/messages, which Python's OpenAI-compatible config forwards untranslated."""
|
||||
BEDROCK_INVOKE = "bedrock_invoke"
|
||||
"""Claude on Bedrock InvokeModel, an Anthropic Messages body behind the InvokeModel API."""
|
||||
RESPONSES = "responses"
|
||||
|
||||
|
||||
NATIVE_MESSAGES_TARGETS: Final = frozenset({Target.MESSAGES, Target.MESSAGES_PASSTHROUGH, Target.BEDROCK_INVOKE})
|
||||
|
||||
|
||||
class Feature(str, Enum):
|
||||
MOCK_RESPONSE = "mock_response"
|
||||
"""mock_response short-circuits before any provider call."""
|
||||
|
|
@ -202,6 +207,12 @@ RULES: Final[Rules] = (
|
|||
),
|
||||
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=MESSAGES_PYTHON_FEATURES),
|
||||
RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED, providers=frozenset({"anthropic", "azure_ai"})),
|
||||
RouteRule(
|
||||
Route.MESSAGES,
|
||||
Rollout.RUST_REQUIRED,
|
||||
providers=frozenset({"bedrock"}),
|
||||
targets=frozenset({Target.BEDROCK_INVOKE}),
|
||||
),
|
||||
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY),
|
||||
RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY, features=RESPONSES_PYTHON_FEATURES),
|
||||
RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY),
|
||||
|
|
|
|||
|
|
@ -707,18 +707,26 @@ def _log_superseded_stream_failure(
|
|||
verbose_proxy_logger.error("Error scheduling stream failure logging: %s", e)
|
||||
|
||||
|
||||
def _relayed(head: object) -> bool:
|
||||
from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead
|
||||
|
||||
return isinstance(head, MessagesStreamHead) and head.relayed
|
||||
|
||||
|
||||
def stream_failure(
|
||||
logger: Logging,
|
||||
endpoint_type: str,
|
||||
request_body: dict[str, object],
|
||||
chunks: list[bytes],
|
||||
error: Exception,
|
||||
head: object,
|
||||
) -> Coroutine[object, object, None]:
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
|
||||
|
||||
verbose_proxy_logger.error("Error in chunk_processor: %s", error)
|
||||
if not _relayed(head):
|
||||
verbose_proxy_logger.error("Error in chunk_processor: %s", error)
|
||||
# Python's stream fails before its success handler records `cache_hit`; only a cache hit
|
||||
# stamps it earlier.
|
||||
if logger.model_call_details.get("cache_hit") is not True:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from functools import lru_cache
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils import fallback_generalizations
|
||||
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping
|
||||
from litellm.litellm_core_utils.get_model_cost_map import FALLBACK_GENERALIZATIONS_KEY
|
||||
from litellm.litellm_core_utils.model_cost_table import ModelCostVersion, model_cost_version
|
||||
|
||||
|
||||
|
|
@ -22,6 +25,7 @@ class _Held:
|
|||
return isinstance(other, _Held) and other.value is self.value
|
||||
|
||||
|
||||
_CatalogVersion: TypeAlias = tuple[int, int, int]
|
||||
_Fingerprint: TypeAlias = tuple[str, str | None, _Held, ModelCostVersion, int, tuple[_Held, ...]]
|
||||
|
||||
|
||||
|
|
@ -95,3 +99,18 @@ def _capabilities(fingerprint: _Fingerprint, resolved_model: str, provider: str)
|
|||
"supports_mid_conversation_system": supports("supports_mid_conversation_system"),
|
||||
"effort_tiers": {level: tier(level) for level in ("minimal", "low", "medium", "high", "xhigh", "max")},
|
||||
}
|
||||
|
||||
|
||||
def model_cost_document(known: _CatalogVersion | None) -> tuple[_CatalogVersion, Mapping[str, object] | None]:
|
||||
"""The live model map and its active fallback rules as one document, which the native catalog reads, or
|
||||
``None`` when the map is still at version ``known``."""
|
||||
version: Final = model_cost_version()
|
||||
current: Final = (version.table, version.size, version.writes)
|
||||
if current == known:
|
||||
return current, None
|
||||
rows: Final = as_str_mapping(litellm.model_cost) or {} # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the model map is an untyped dict of rows
|
||||
rules: Final = as_object_list(fallback_generalizations.get_fallback_generalization_rules()) or [] # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the rule registry keeps the raw untyped list
|
||||
document: Final = json.loads(
|
||||
json.dumps({**rows, FALLBACK_GENERALIZATIONS_KEY: {"rules": list(rules)}}, default=str)
|
||||
)
|
||||
return current, as_str_mapping(document)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import http
|
|||
import http.client
|
||||
import inspect
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import ssl
|
||||
|
|
@ -18,12 +19,17 @@ from typing import Final, TypeAlias, TypeVar, cast # noqa: TID251 # narrows na
|
|||
from urllib.parse import SplitResult, urlsplit
|
||||
|
||||
import httpx
|
||||
from aiohttp import ClientPayloadError
|
||||
from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.caching_handler import LLMCachingHandler
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS,
|
||||
ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils import exception_mapping_utils
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
|
|
@ -42,7 +48,7 @@ from litellm.litellm_core_utils.model_latest_aliases import is_model_latest_alia
|
|||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
|
|
@ -70,7 +76,13 @@ from litellm.llms.anthropic.messages.native.handler import (
|
|||
_with_deployment_output_effort as _deployment_output_effort, # pyright: ignore[reportPrivateUsage] # the handler's own effort rounding
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.interceptors import MessagesInterceptor, get_messages_interceptors
|
||||
from litellm.llms.anthropic.messages.native.utils import normalize_native_messages, prepare_native_messages
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
continues_message_thread,
|
||||
message_thread_not_found_body,
|
||||
message_thread_unsupported_body,
|
||||
normalize_native_messages,
|
||||
prepare_native_messages,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.safeguards import (
|
||||
SafeguardsEvaluator,
|
||||
StreamedSafeguardResults,
|
||||
|
|
@ -81,6 +93,11 @@ from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enab
|
|||
from litellm.llms.anthropic.wif import get_anthropic_wif_token
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
ensure_bedrock_anthropic_messages_tool_names, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict
|
||||
normalize_custom_field_on_tools, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # its params dict is untyped
|
||||
|
|
@ -152,6 +169,8 @@ class MessagesSettings:
|
|||
ascii_response_headers: bool = False
|
||||
joined_reads: bool = False
|
||||
ascii_request_headers: bool = True
|
||||
detached_drain_timeout: float | None = None
|
||||
detached_drain_limit: int | None = None
|
||||
|
||||
|
||||
def response(value: Mapping[str, object]) -> AnthropicMessagesResponse:
|
||||
|
|
@ -191,15 +210,100 @@ def _anthropic_metadata(params: Mapping[str, object]) -> Mapping[str, object]:
|
|||
|
||||
def _body_fields(request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
params: Final = _anthropic_metadata(
|
||||
known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request)), request)
|
||||
known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request), _dropped_thread(request)), request)
|
||||
)
|
||||
if deployment_reasoning_effort_levels(request.get("model_info")) is None:
|
||||
return params
|
||||
return _with_deployment_effort(params, request)
|
||||
|
||||
|
||||
def _thread_provider(request: Mapping[str, object]) -> str | None:
|
||||
"""The provider of a call carrying a thread its Messages config cannot serve, as Python's handler
|
||||
resolves it, or None when the call carries no thread or its config serves threads."""
|
||||
if request.get("thread") is None:
|
||||
return None
|
||||
model, provider, _, _ = _requested_provider(
|
||||
str(request["model"]), optional_str(request.get("custom_llm_provider")), _handler_params(request)
|
||||
)
|
||||
config: Final = native_messages_config(
|
||||
model, provider, request.get("model_info"), optional_str(request.get("api_base"))
|
||||
)
|
||||
if config is not None and config.supports_message_threads():
|
||||
return None
|
||||
return str(provider)
|
||||
|
||||
|
||||
def _dropped_thread(request: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return () if _thread_provider(request) is None else ("thread",)
|
||||
|
||||
|
||||
def _thread_refusal(thread: object, provider: str, drop_params: object) -> AnthropicError | None:
|
||||
if continues_message_thread(thread):
|
||||
return AnthropicError(
|
||||
status_code=404,
|
||||
message=message_thread_not_found_body(
|
||||
f"message threads are not supported for provider {provider}, "
|
||||
"so this thread cannot be continued. Send the full conversation to start over"
|
||||
),
|
||||
)
|
||||
if not drops_unsupported_params(drop_params):
|
||||
return AnthropicError(
|
||||
status_code=400,
|
||||
message=message_thread_unsupported_body(
|
||||
f"message threads are not supported for provider {provider}. "
|
||||
"Send the request without thread, or set drop_params to serve it without a server-side thread"
|
||||
),
|
||||
)
|
||||
verbose_logger.warning(
|
||||
"Dropping thread for provider %s: message threads are not supported there, "
|
||||
"so the request is served without a server-side thread",
|
||||
provider,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _refuse_unsupported_thread(request: Mapping[str, object], asynchronous: bool) -> None:
|
||||
"""Python's handler refuses a thread its provider cannot serve, or drops it under drop_params."""
|
||||
provider: Final = _thread_provider(request)
|
||||
if provider is None:
|
||||
return
|
||||
refusal: Final = _thread_refusal(request.get("thread"), provider, request.get("drop_params"))
|
||||
if refusal is None:
|
||||
return
|
||||
_flag_messages_call(request.get("litellm_logging_obj"), asynchronous)
|
||||
try:
|
||||
raise refusal
|
||||
except AnthropicError as provider_error:
|
||||
_exception_type(
|
||||
model=str(request["model"]),
|
||||
custom_llm_provider=provider,
|
||||
original_exception=provider_error,
|
||||
extra_kwargs={name: value for name, value in request.items() if name not in _ENTRY_PARAMS},
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def _shape_bedrock_tools(tools: object, model: str) -> None:
|
||||
"""Python's Bedrock invoke transform shapes the caller's own tool dicts in place, so its logs show them shaped."""
|
||||
if not isinstance(tools, list):
|
||||
return
|
||||
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import (
|
||||
AmazonAnthropicClaudeMessagesConfig,
|
||||
)
|
||||
|
||||
body: Final[dict[str, object]] = {"tools": tools} # mutable-ok: the Bedrock transform helpers shape it in place
|
||||
AmazonAnthropicClaudeMessagesConfig()._remove_ttl_from_cache_control( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # the transform's own in-place step
|
||||
anthropic_messages_request=body, model=model
|
||||
)
|
||||
normalize_custom_field_on_tools(body)
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke(body)
|
||||
ensure_bedrock_anthropic_messages_tool_names(body)
|
||||
|
||||
|
||||
def body_params(request: Mapping[str, object], asynchronous: bool, hooked: bool = False) -> Mapping[str, object]:
|
||||
fields: Final = _body_fields(request)
|
||||
if _resolved_provider(request) == "bedrock":
|
||||
_shape_bedrock_tools(fields.get("tools"), str(request["model"]).removeprefix("bedrock/"))
|
||||
if not detectors.cache_control_injection(NativeCall(args=(), kwargs=request, base={})):
|
||||
return {**fields, "messages": request["messages"]} if hooked else fields
|
||||
messages, system, tools = _prompt_cached(request, asynchronous, hooked)
|
||||
|
|
@ -493,12 +597,18 @@ def answer(owner: MessagesInterceptor, request: Mapping[str, object], stream: bo
|
|||
)
|
||||
|
||||
|
||||
def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]:
|
||||
def stream_hidden_params(headers: Sequence[tuple[str, str]], relayed: bool = False) -> Mapping[str, object]:
|
||||
"""The stream's head. `relayed` says the stream runs through Python's native Messages relay,
|
||||
`BaseAnthropicMessagesStreamingIterator.async_sse_wrapper`, rather than its pass-through."""
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import (
|
||||
anthropic_messages_stream_hidden_params,
|
||||
)
|
||||
|
||||
return MessagesStreamHead(anthropic_messages_stream_hidden_params(callbacks_legacy_python.wire_headers(headers)))
|
||||
head: Final = MessagesStreamHead(
|
||||
anthropic_messages_stream_hidden_params(callbacks_legacy_python.wire_headers(headers))
|
||||
)
|
||||
head.relayed = relayed
|
||||
return head
|
||||
|
||||
|
||||
async def _unrequested_stream_delivered(
|
||||
|
|
@ -557,6 +667,7 @@ def unrequested_stream(
|
|||
|
||||
_UPSTREAM_ARGS: Final = TypeAdapter(tuple[int, str])
|
||||
_UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]])
|
||||
_INVOCATION_METRICS: Final = "amazon-bedrock-invocationMetrics"
|
||||
_BODY_CUT: Final = TypeAdapter(tuple[int, int | None, bool])
|
||||
_ERROR_CLASS: TypeAlias = Callable[..., Exception]
|
||||
_validate_metadata: Final = cast( # cast-ok: the handler's untyped metadata validator
|
||||
|
|
@ -601,12 +712,18 @@ def _rejection(error: Exception, request: Mapping[str, object], request_provider
|
|||
provider_rejection: Final = getattr(error, "messages_provider_rejection", None)
|
||||
if isinstance(provider_rejection, str):
|
||||
_log_resolved_call(request, request_provider)
|
||||
_flag_azure_ai_call(request, request_provider, asynchronous)
|
||||
return _public_provider_error(
|
||||
_config_error(400, provider_rejection, (), request, request_provider), error, request, request_provider
|
||||
_flag_messages_call(request.get("litellm_logging_obj"), asynchronous)
|
||||
rejection: Final = _provider_rejection(
|
||||
provider_rejection,
|
||||
getattr(error, "messages_provider_bad_request", False) is True,
|
||||
request,
|
||||
request_provider,
|
||||
)
|
||||
if isinstance(rejection, litellm.BadRequestError):
|
||||
return rejection
|
||||
return _public_provider_error(rejection, error, request, request_provider)
|
||||
if getattr(error, "messages_unsupported_param", False):
|
||||
_flag_azure_ai_call(request, request_provider, asynchronous)
|
||||
_flag_messages_call(request.get("litellm_logging_obj"), asynchronous)
|
||||
return litellm.UnsupportedParamsError(message=str(error))
|
||||
return litellm.BadRequestError(
|
||||
message=str(error),
|
||||
|
|
@ -615,6 +732,23 @@ def _rejection(error: Exception, request: Mapping[str, object], request_provider
|
|||
)
|
||||
|
||||
|
||||
def _provider_rejection(
|
||||
text: str, bad_request: bool, request: Mapping[str, object], request_provider: str
|
||||
) -> Exception:
|
||||
"""The Bedrock invoke config raises its own `BadRequestError`, which other configs' HTTP handler
|
||||
maps into their error class by its text."""
|
||||
if not bad_request:
|
||||
return _config_error(400, text, (), request, request_provider)
|
||||
raised: Final = litellm.BadRequestError(
|
||||
message=text,
|
||||
model=str(request["model"]).removeprefix(f"{request_provider}/"),
|
||||
llm_provider=request_provider,
|
||||
)
|
||||
if request_provider == "bedrock":
|
||||
return raised
|
||||
return _config_error(400, str(raised), (), request, request_provider)
|
||||
|
||||
|
||||
def _timeout_text(phase: str, request: Mapping[str, object]) -> str:
|
||||
"""`litellm.Timeout`'s text as Python's HTTP client raises it for the same timeout."""
|
||||
seconds: Final = request.get("timeout")
|
||||
|
|
@ -652,21 +786,115 @@ def _body_cut(received: int, expected: int | None, reset: bool) -> httpx.Transpo
|
|||
f"Not enough data to satisfy content length header (received {received} of {expected} bytes)."
|
||||
)
|
||||
)
|
||||
text: Final = f"Response payload is not completed: {payload!r}"
|
||||
if not reset:
|
||||
return httpx.ReadError(text)
|
||||
return httpx.ReadError(f"{text}. {ConnectionResetError(errno.ECONNRESET, os.strerror(errno.ECONNRESET))!r}")
|
||||
completed: Final = f"Response payload is not completed: {payload!r}"
|
||||
text: Final = (
|
||||
f"{completed}. {ConnectionResetError(errno.ECONNRESET, os.strerror(errno.ECONNRESET))!r}"
|
||||
if reset
|
||||
else completed
|
||||
)
|
||||
payload_error: Final = ClientPayloadError(text)
|
||||
payload_error.__cause__ = payload
|
||||
read_error: Final = httpx.ReadError(text)
|
||||
read_error.__cause__ = payload_error
|
||||
read_error.__context__ = payload_error
|
||||
return read_error
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ProviderCall:
|
||||
config: BaseAnthropicMessagesConfig
|
||||
model: str
|
||||
params: dict[str, object] # mutable-ok: the config hooks take a params dict
|
||||
|
||||
|
||||
def _provider_call(request: Mapping[str, object]) -> _ProviderCall | None:
|
||||
"""The config, model and params Python's handler hands its provider config for the call."""
|
||||
model, provider, _, _ = _requested_provider(
|
||||
str(request["model"]), optional_str(request.get("custom_llm_provider")), _handler_params(request)
|
||||
)
|
||||
config: Final = native_messages_config(
|
||||
model, provider, request.get("model_info"), optional_str(request.get("api_base"))
|
||||
)
|
||||
if config is None:
|
||||
return None
|
||||
return _ProviderCall(config=config, model=model, params=dict(_python_handler_params(request, provider)))
|
||||
|
||||
|
||||
def _complete_url(call: _ProviderCall, request: Mapping[str, object]) -> str:
|
||||
return call.config.get_complete_url( # pyright: ignore[reportUnknownMemberType] # the config's untyped params dicts
|
||||
api_base=optional_str(request.get("api_base")),
|
||||
api_key=optional_str(request.get("api_key")),
|
||||
model=call.model,
|
||||
optional_params=call.params,
|
||||
litellm_params=call.params,
|
||||
stream=request.get("stream") is True,
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_url(request: Mapping[str, object]) -> str | None:
|
||||
"""The URL Python's bedrock Messages config builds for the call."""
|
||||
call: Final = _provider_call(request)
|
||||
return None if call is None else _complete_url(call, request)
|
||||
|
||||
|
||||
def _signing_failure(kind: str) -> Exception | None:
|
||||
"""The exception Python's AWS signing raises for a credential failure whose outcome needs no STS,
|
||||
IMDS or secret manager call. botocore's environment provider raises `PartialCredentialsError`
|
||||
for an access key without its secret, where Rust's default chain finds nothing."""
|
||||
from botocore.exceptions import NoCredentialsError
|
||||
|
||||
from litellm.llms.bedrock.base_aws_llm import AwsAuthError
|
||||
|
||||
if kind == "web_identity_reference":
|
||||
return AwsAuthError(status_code=400, message="Invalid web identity token reference.")
|
||||
if kind == "missing_web_identity_token":
|
||||
return AwsAuthError(status_code=401, message="OIDC token could not be retrieved from secret manager.")
|
||||
if kind == "no_credentials" and not os.environ.get("AWS_ACCESS_KEY_ID"):
|
||||
return NoCredentialsError()
|
||||
return None
|
||||
|
||||
|
||||
def _bedrock_credential_failure(error: Exception, request: Mapping[str, object]) -> Exception | None:
|
||||
"""The exception Python's bedrock config raises while signing a call whose credentials Rust
|
||||
could not acquire, from botocore's `NoCredentialsError` to an STS or `AwsAuthError` failure."""
|
||||
kind: Final = getattr(error, "messages_credential_failure", None)
|
||||
if not isinstance(kind, str):
|
||||
return None
|
||||
known: Final = _signing_failure(kind)
|
||||
if known is not None:
|
||||
return _raised(known)
|
||||
call: Final = _provider_call(request)
|
||||
if call is None:
|
||||
return None
|
||||
try:
|
||||
call.config.sign_request( # pyright: ignore[reportUnknownMemberType] # the config's untyped params dicts
|
||||
headers={},
|
||||
optional_params=call.params,
|
||||
request_data={},
|
||||
api_base=_complete_url(call, request),
|
||||
api_key=optional_str(request.get("api_key")),
|
||||
model=call.model,
|
||||
stream=request.get("stream") is True,
|
||||
fake_stream=False,
|
||||
)
|
||||
except Exception as python_failure: # noqa: BLE001 # the exception Python's signing raises is the answer
|
||||
return python_failure
|
||||
return None
|
||||
|
||||
|
||||
def _sent_base(request: Mapping[str, object], request_provider: str, secrets_read: _SecretsRead) -> str | None:
|
||||
"""The base the call was sent to: azure_ai's falls back to the `AZURE_API_BASE` the host read
|
||||
to send it, and anthropic's to its default."""
|
||||
to send it, anthropic's to its default and bedrock's to the URL its config builds."""
|
||||
api_base: Final = request.get("api_base")
|
||||
if isinstance(api_base, str) and (api_base or request_provider != "azure_ai"):
|
||||
return api_base
|
||||
if request_provider == "anthropic":
|
||||
return api_base if isinstance(api_base, str) else anthropic_pass_through_api_base()
|
||||
if request_provider == "azure_ai" and not api_base:
|
||||
return anthropic_pass_through_api_base()
|
||||
if request_provider == "azure_ai":
|
||||
return dict(secrets_read).get("AZURE_API_BASE")
|
||||
return api_base if isinstance(api_base, str) else None
|
||||
if request_provider == "bedrock":
|
||||
return _bedrock_url(request)
|
||||
return None
|
||||
|
||||
|
||||
def _connect_failure(
|
||||
|
|
@ -869,6 +1097,58 @@ def _raised_like_python(error: _RaisedT, request_provider: str) -> _RaisedT:
|
|||
return error if request_provider in _MAIN_NATIVE_ERROR_PROVIDERS else _raised(error)
|
||||
|
||||
|
||||
def _chunk_parser_failure(text: str) -> Exception:
|
||||
try:
|
||||
json.loads(text).pop(_INVOCATION_METRICS, {})
|
||||
except Exception as error: # noqa: BLE001 # the chunk parser's own failure is what Python raised
|
||||
return error
|
||||
return ValueError(text)
|
||||
|
||||
|
||||
def _json_failure(text: str) -> Exception:
|
||||
try:
|
||||
json.loads(text)
|
||||
except json.JSONDecodeError as error:
|
||||
return error
|
||||
return ValueError(text)
|
||||
|
||||
|
||||
def bedrock_stream_raise(kind: str, arguments: Sequence[object]) -> Exception:
|
||||
"""The exception Python's Bedrock stream decoder raised unmapped for a body it could not parse."""
|
||||
from botocore import eventstream
|
||||
|
||||
raises: Final[Mapping[str, Callable[..., Exception]]] = {
|
||||
"ChecksumMismatch": eventstream.ChecksumMismatch,
|
||||
"InvalidHeadersLength": eventstream.InvalidHeadersLength,
|
||||
"InvalidPayloadLength": eventstream.InvalidPayloadLength,
|
||||
"DuplicateHeader": eventstream.DuplicateHeader,
|
||||
"InvalidJson": _json_failure,
|
||||
"NotAnObject": _chunk_parser_failure,
|
||||
}
|
||||
return _raise_with(raises[kind], arguments)
|
||||
|
||||
|
||||
def _raise_with(factory: Callable[..., Exception], arguments: Sequence[object]) -> Exception:
|
||||
return factory(*arguments)
|
||||
|
||||
|
||||
def stream_failure(error: BaseException, request: Mapping[str, object], request_provider: str) -> BaseException:
|
||||
"""Python's Bedrock Messages stream raises a failure after its head with the deployment's debug info."""
|
||||
if request_provider in _MAIN_NATIVE_ERROR_PROVIDERS or not isinstance(error, Exception):
|
||||
return error
|
||||
return _with_deployment_debug_info(error, request, request_provider)
|
||||
|
||||
|
||||
def _bedrock_stream_error(error: Exception, request: Mapping[str, object]) -> Exception | None:
|
||||
"""Python's Bedrock stream decoder raises its `BedrockError` as built, unmapped."""
|
||||
try:
|
||||
status, text = _UPSTREAM_ARGS.validate_python(error.args)
|
||||
headers: Final = _UPSTREAM_HEADERS.validate_python(getattr(error, "headers", None))
|
||||
except ValidationError:
|
||||
return None
|
||||
return _config_error(status, text, headers, request, "bedrock")
|
||||
|
||||
|
||||
def map_failure(
|
||||
error: Exception,
|
||||
request: Mapping[str, object],
|
||||
|
|
@ -878,6 +1158,10 @@ def map_failure(
|
|||
) -> Exception:
|
||||
"""Raises what Python's Messages handler raises for the same failure. `secrets_read` are the
|
||||
secrets the host read to send the call, such as azure_ai's `AZURE_API_BASE`."""
|
||||
if request_provider == "bedrock" and getattr(error, "messages_stream_failure", False):
|
||||
stream_error: Final = _bedrock_stream_error(error, request)
|
||||
if stream_error is not None:
|
||||
return stream_error
|
||||
if getattr(error, "messages_timeout", None) == "Body":
|
||||
# Python reads a stream's body outside its HTTP client's error mapping, so the transport's
|
||||
# own timeout reaches the caller unmapped.
|
||||
|
|
@ -915,6 +1199,10 @@ def map_failure(
|
|||
),
|
||||
request_provider,
|
||||
)
|
||||
credential_failure: Final = _bedrock_credential_failure(error, request) if request_provider == "bedrock" else None
|
||||
if credential_failure is not None:
|
||||
_log_resolved_call(request, request_provider)
|
||||
return map_token_failure(credential_failure, request, request_provider, asynchronous)
|
||||
provider_error: Final = _provider_error(error, request, request_provider, secrets_read or [])
|
||||
if provider_error is None:
|
||||
return failures.map_native_failure(
|
||||
|
|
@ -950,6 +1238,20 @@ def failure_provider(request: Mapping[str, object]) -> str:
|
|||
return _resolved_provider(request) or "anthropic"
|
||||
|
||||
|
||||
def filters_beta_headers(model: str, custom_llm_provider: str | None, api_base: str | None) -> bool:
|
||||
"""Whether the call's provider config filters the caller's `anthropic-beta` before Python's client encodes it."""
|
||||
config: Final = native_messages_config(
|
||||
model.removeprefix(f"{custom_llm_provider}/"), custom_llm_provider, None, api_base
|
||||
)
|
||||
return config is not None and config.should_filter_anthropic_beta_headers()
|
||||
|
||||
|
||||
def record_guardrail(public: Mapping[str, object], request: Mapping[str, object]) -> None:
|
||||
logger: Final = _logger(request)
|
||||
if logger is not None:
|
||||
logger.record_native_bedrock_guardrail(public)
|
||||
|
||||
|
||||
def _scope_provider_specific_header(request: Mapping[str, object]) -> None:
|
||||
scoped: Final = request.get("provider_specific_header")
|
||||
if scoped is None:
|
||||
|
|
@ -1009,10 +1311,15 @@ def settings(
|
|||
drops_unsupported_params=drops_unsupported_params(kwargs.get("drop_params")),
|
||||
ascii_response_headers=aiohttp,
|
||||
joined_reads=aiohttp,
|
||||
detached_drain_timeout=ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS if asynchronous else None,
|
||||
detached_drain_limit=ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS if asynchronous else None,
|
||||
)
|
||||
return asdict(
|
||||
configured if request is None else _with_call_settings(configured, request, kwargs.get("litellm_logging_obj"))
|
||||
)
|
||||
if request is None:
|
||||
return asdict(configured)
|
||||
call_settings: Final = _with_call_settings(configured, request, kwargs.get("litellm_logging_obj"))
|
||||
if call_settings.mock_response is None and not call_settings.mcp_gateway:
|
||||
_refuse_unsupported_thread(request, asynchronous)
|
||||
return asdict(call_settings)
|
||||
|
||||
|
||||
def _flag_messages_call(logger: object, asynchronous: bool) -> None:
|
||||
|
|
@ -1021,13 +1328,6 @@ def _flag_messages_call(logger: object, asynchronous: bool) -> None:
|
|||
logger.model_call_details.setdefault("litellm_params", {})[CallTypes.aanthropic_messages.value] = asynchronous
|
||||
|
||||
|
||||
def _flag_azure_ai_call(request: Mapping[str, object], request_provider: str, asynchronous: bool) -> None:
|
||||
"""Python's handler flags the call before it gets the credential or refuses a param. By the owner's ruling
|
||||
anthropic keeps main's unflagged failure."""
|
||||
if request_provider == "azure_ai":
|
||||
_flag_messages_call(request.get("litellm_logging_obj"), asynchronous)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Answered:
|
||||
value: object
|
||||
|
|
@ -1296,7 +1596,7 @@ def map_token_failure(
|
|||
) -> Exception:
|
||||
"""Python's handler maps a litellm exception raised while it gets the call's credential, and lets any other
|
||||
exception through as raised."""
|
||||
_flag_azure_ai_call(request, request_provider, asynchronous)
|
||||
_flag_messages_call(request.get("litellm_logging_obj"), asynchronous)
|
||||
if not isinstance(error, BaseLLMException):
|
||||
return error
|
||||
return _public_provider_error(error, error, request, request_provider)
|
||||
|
|
@ -1518,13 +1818,14 @@ def unencodable_header(value: bytes, received: bool) -> Exception:
|
|||
def _sent_url(request: Mapping[str, object], url: str) -> str:
|
||||
"""The native URL builder trims the caller's api_base; Python sends it as given."""
|
||||
api_base: Final = request.get("api_base")
|
||||
if (
|
||||
not isinstance(api_base, str)
|
||||
or api_base == api_base.strip()
|
||||
or not _unsendable_api_base(api_base)
|
||||
or _resolved_provider(request) != "anthropic"
|
||||
):
|
||||
if not isinstance(api_base, str) or api_base == api_base.strip() or not _unsendable_api_base(api_base):
|
||||
return url
|
||||
provider: Final = _resolved_provider(request)
|
||||
if provider not in ("anthropic", "bedrock"):
|
||||
return url
|
||||
if provider == "bedrock":
|
||||
trimmed: Final = f"{api_base.strip().rstrip('/')}/"
|
||||
return join_api_base(api_base, url.removeprefix(trimmed)) if url.startswith(trimmed) else url
|
||||
try:
|
||||
return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url
|
||||
except ValueError: # a base urlsplit cannot split keeps the URL the native builder made
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ class ProviderDefaults:
|
|||
enable_azure_ad_token_refresh: object
|
||||
api_key: object
|
||||
azure_key: object
|
||||
drop_params: object
|
||||
bedrock_request_metadata_fields: object
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -145,6 +147,8 @@ def provider_defaults() -> ProviderDefaults:
|
|||
enable_azure_ad_token_refresh=litellm.enable_azure_ad_token_refresh,
|
||||
api_key=litellm.api_key,
|
||||
azure_key=litellm.azure_key,
|
||||
drop_params=litellm.drop_params,
|
||||
bedrock_request_metadata_fields=litellm.bedrock_request_metadata_fields,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ class MessagesStreamHead(dict[str, object]):
|
|||
agentic: Callable[[execution.AsyncStream], object] | None = None
|
||||
deliver: Callable[[object], Awaitable[object]] | None = None
|
||||
python_types: bool = False
|
||||
relayed: bool = False
|
||||
cached: object | None = None
|
||||
cache_writer: Callable[[object], object] | None = None
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue