From 8b0bb6ae424cf0518a5e8d155d91876153eda270 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 16 Nov 2026 19:58:53 -0800 Subject: [PATCH] fix(bedrock): route bedrock// to that region on /v1/messages (#52136) * fix(bedrock): route bedrock// to that region on the Invoke path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep LIT010 flat and update the invoke region-path integration test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(bedrock): drop the comment that repeats the helper name Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/base_aws_llm.py | 18 ++++-- ...odel_prices_and_context_window_backup.json | 56 ------------------- .../tests/bedrock_native/main.rs | 6 +- .../crates/llms/src/bedrock/common_utils.rs | 8 +++ .../anthropic_claude_transformation.rs | 43 ++++++++------ model_prices_and_context_window.json | 56 ------------------- tests/unit/llms/bedrock/test_base_aws_llm.py | 29 ++++++++++ tests/unit/whitelisted_bedrock_models.txt | 2 - 8 files changed, 79 insertions(+), 139 deletions(-) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index e21e8822b41..35a84c202b1 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -616,11 +616,11 @@ class BaseAWSLLM(SignsRequestsWithAWS): # a malformed URL and cause botocore's EventStreamBuffer to receive # a JSON error body instead of a binary event-stream — surfaced as a # misleading ChecksumMismatch (0x223a7b22 == ':{"'). - # Use strip_bedrock_routing_prefix (no break) so compound prefixes + # Use split_bedrock_region_path (no break) so compound prefixes # like "bedrock/invoke/arn:..." are fully stripped in one call. - from litellm.llms.bedrock.common_utils import strip_bedrock_routing_prefix + from litellm.llms.bedrock.common_utils import split_bedrock_region_path - model_id = strip_bedrock_routing_prefix(model_id) + model_id = split_bedrock_region_path(model_id)[1] # URL-encode ARNs so colons and slashes are safe in the URL path. if model_id.startswith("arn:"): model_id = BaseAWSLLM.encode_model_id(model_id=model_id) @@ -713,6 +713,14 @@ class BaseAWSLLM(SignsRequestsWithAWS): return None + @staticmethod + def _get_aws_region_from_model_path(model: str | None) -> str | None: + if model is None: + return None + from litellm.llms.bedrock.common_utils import split_bedrock_region_path + + return split_bedrock_region_path(model)[0] + def _get_aws_region_name( self, optional_params: dict, @@ -738,7 +746,9 @@ class BaseAWSLLM(SignsRequestsWithAWS): if model_id is not None: aws_region_name = self._get_aws_region_from_model_arn(model_id) else: - aws_region_name = self._get_aws_region_from_model_arn(model) + aws_region_name = self._get_aws_region_from_model_arn(model) or self._get_aws_region_from_model_path( + model + ) # check env # litellm_aws_region_name: Final = get_secret("AWS_REGION_NAME", None) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 44a7da6568e..7334d83e6ad 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15826,34 +15826,6 @@ "litellm_provider": "bedrock" } }, - "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.5e-06, - "cache_creation_input_token_cost_above_1hr": 7.2e-06, - "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-27", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "supports_tool_search": true, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", - "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} - }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", @@ -16019,34 +15991,6 @@ "litellm_provider": "bedrock" } }, - "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.5e-06, - "cache_creation_input_token_cost_above_1hr": 7.2e-06, - "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-27", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "supports_tool_search": true, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", - "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} - }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-05-09", "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", diff --git a/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs b/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs index 3b0956a860a..fb2e7ff5c0c 100644 --- a/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs +++ b/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs @@ -229,7 +229,7 @@ async fn managed_aws_credentials_sign_the_exact_native_request( #[rstest] #[tokio::test] -async fn bedrock_invoke_keeps_the_region_path_and_signs_caller_headers_as_python_does( +async fn bedrock_invoke_routes_by_the_region_path_and_signs_caller_headers_as_python_does( call: MessagesCall, ) { let upstream = upstream([message_response()]).await; @@ -263,11 +263,11 @@ async fn bedrock_invoke_keeps_the_region_path_and_signs_caller_headers_as_python let auth = request.header("authorization").unwrap(); assert_eq!( request.url.path(), - "/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke" + "/model/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke" ); assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=test-access/")); assert!(auth.contains( - "/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;date;host;x-amz-security-token, " + "/us-west-2/bedrock/aws4_request, SignedHeaders=content-type;date;host;x-amz-security-token, " )); assert_eq!(request.header("x-amz-security-token"), Some("caller-token")); assert_eq!(request.header("x-amz-date"), None); diff --git a/litellm/rust/crates/llms/src/bedrock/common_utils.rs b/litellm/rust/crates/llms/src/bedrock/common_utils.rs index 1850f11128a..abb798d10cf 100644 --- a/litellm/rust/crates/llms/src/bedrock/common_utils.rs +++ b/litellm/rust/crates/llms/src/bedrock/common_utils.rs @@ -53,6 +53,14 @@ pub fn strip_bedrock_routing_prefix(model: &str) -> &str { }) } +pub fn split_bedrock_region_path(model: &str) -> (Option<&str>, &str) { + let stripped = strip_bedrock_routing_prefix(model); + match stripped.split_once('/') { + Some((region, model_id)) if is_bedrock_region(region) => (Some(region), model_id), + _ => (None, stripped), + } +} + fn model_name_from_arn(model: &str) -> &str { if !model.to_lowercase().contains("arn") { return model; diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs index 94b06b70f88..8a09d0a4d6d 100644 --- a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs @@ -40,7 +40,7 @@ use crate::bedrock::{ GUARDRAIL_CONFIG_PARAM, bedrock_invoke_guardrail_headers, }, common_utils::{ - strip_bedrock_routing_prefix, + split_bedrock_region_path, structured_output::{ NATIVE_OUTPUT_UNSUPPORTED, bedrock_native_output_format, bedrock_native_output_retry_requested, is_bedrock_native_output_rejection, @@ -100,7 +100,7 @@ fn get_bedrock_model_id(model: &str, model_id: Option<&str>) -> String { if let Some(model_id) = model_id { return encode_model_id(model_id).replacen(INVOKE_MODEL_PREFIX, "", 1); } - let stripped = strip_bedrock_routing_prefix(model); + let (_, stripped) = split_bedrock_region_path(model); if stripped.starts_with("arn:") { return encode_model_id(stripped); } @@ -146,6 +146,11 @@ fn get_aws_region_from_model_arn(model: &str) -> Option { } /// Port of Python's `AmazonInvokeConfig.get_complete_url`. +fn region_from_model(model: &str) -> Option { + get_aws_region_from_model_arn(model) + .or_else(|| split_bedrock_region_path(model).0.map(str::to_string)) +} + fn invoke_url( api_base: Option<&str>, model: &str, @@ -161,11 +166,7 @@ fn invoke_url( .get(MODEL_ID_PARAM) .and_then(Value::as_str), ); - let region = resolve_bedrock_region( - get_aws_region_from_model_arn(model).as_deref(), - aws, - env_lookup, - ); + let region = resolve_bedrock_region(region_from_model(model).as_deref(), aws, env_lookup); let path = if stream { INVOKE_STREAM_PATH } else { @@ -347,7 +348,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { headers, auth: AuthScheme::AwsSigV4 { region: resolve_bedrock_region( - get_aws_region_from_model_arn(model).as_deref(), + region_from_model(model).as_deref(), params, env_lookup, ), @@ -621,28 +622,34 @@ mod tests { } #[rstest] - #[case::region_segment_stays_in_the_path( + #[case::region_segment_leaves_the_path( "bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0", None, - "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", + "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", "us-west-2" )] - #[case::region_segment_does_not_pick_the_region( + #[case::region_segment_picks_the_region( "invoke/eu-central-1/anthropic.claude-3-5-sonnet-20240620-v1:0", None, - "https://bedrock-runtime.us-west-2.amazonaws.com/model/eu-central-1/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", - "us-west-2" + "https://bedrock-runtime.eu-central-1.amazonaws.com/model/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", + "eu-central-1" )] - #[case::params_region_hosts_a_region_segment( + #[case::gov_cloud_region_segment_without_a_route( + "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + None, + "https://bedrock-runtime.us-gov-west-1.amazonaws.com/model/anthropic.claude-sonnet-4-5-20250929-v1:0/invoke", + "us-gov-west-1" + )] + #[case::params_region_beats_a_region_segment( "invoke/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0", Some("us-east-1"), - "https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke", + "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke", "us-east-1" )] #[case::arn_after_a_region_segment_picks_the_region( "invoke/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", None, - "https://bedrock-runtime.eu-west-1.amazonaws.com/model/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke", + "https://bedrock-runtime.eu-west-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aeu-west-1%3A123456789012%3Ainference-profile%2Fus.anthropic.claude-sonnet-4-5-20250929-v1%3A0/invoke", "eu-west-1" )] #[case::arn_after_any_segment_picks_the_region( @@ -672,13 +679,13 @@ mod tests { #[case::nova_spec_is_removed_and_the_id_encoded( "invoke/us-west-2/nova/anthropic.claude-3", None, - "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2%2Fanthropic.claude-3/invoke", + "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3/invoke", "us-west-2" )] #[case::one_inner_invoke_segment_is_removed( "invoke/us-west-2/invoke/anthropic.claude-3", None, - "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2/anthropic.claude-3/invoke", + "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3/invoke", "us-west-2" )] fn url_and_signing_region_match_python_for_every_model_shape( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 44a7da6568e..7334d83e6ad 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15826,34 +15826,6 @@ "litellm_provider": "bedrock" } }, - "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.5e-06, - "cache_creation_input_token_cost_above_1hr": 7.2e-06, - "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-27", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "supports_tool_search": true, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", - "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} - }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", @@ -16019,34 +15991,6 @@ "litellm_provider": "bedrock" } }, - "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.5e-06, - "cache_creation_input_token_cost_above_1hr": 7.2e-06, - "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-27", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "supports_tool_search": true, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", - "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} - }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-05-09", "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", diff --git a/tests/unit/llms/bedrock/test_base_aws_llm.py b/tests/unit/llms/bedrock/test_base_aws_llm.py index d137b9bbdef..e926a30228e 100644 --- a/tests/unit/llms/bedrock/test_base_aws_llm.py +++ b/tests/unit/llms/bedrock/test_base_aws_llm.py @@ -3255,6 +3255,35 @@ class TestGetBedrockModelIdArnHandling: assert "%3A" in model_id, f"ARN not URL-encoded; got: {model_id}" assert "%2F" in model_id, f"ARN slashes not URL-encoded; got: {model_id}" + @pytest.mark.parametrize( + "model", + [ + "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + "bedrock/invoke/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + "invoke/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + "us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0", + ], + ) + def test_region_path_segment_selects_the_region_and_leaves_the_model_id(self, model: str, monkeypatch): + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + for name in ("AWS_REGION_NAME", "AWS_REGION", "AWS_DEFAULT_REGION"): + monkeypatch.delenv(name, raising=False) + + assert (self._call(model), BaseAWSLLM()._get_aws_region_name(optional_params={}, model=model)) == ( + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "us-gov-west-1", + ) + + def test_explicit_region_beats_the_region_path_segment(self): + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + model: Final = "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0" + assert ( + BaseAWSLLM()._get_aws_region_name(optional_params={"aws_region_name": "us-gov-east-1"}, model=model) + == "us-gov-east-1" + ) + def test_bare_arn_is_encoded(self): """Direct ARN without routing prefix must also be URL-encoded.""" model_id = self._call(self.ARN) diff --git a/tests/unit/whitelisted_bedrock_models.txt b/tests/unit/whitelisted_bedrock_models.txt index da5800a4760..681f5095af2 100644 --- a/tests/unit/whitelisted_bedrock_models.txt +++ b/tests/unit/whitelisted_bedrock_models.txt @@ -145,7 +145,6 @@ bedrock/us-gov-east-1/amazon.titan-text-premier-v1:0 bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0 bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0 bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0 -bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0 bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0 bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0 bedrock/us-gov-west-1/amazon.nova-lite-v1:0 @@ -158,7 +157,6 @@ bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0 bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0 bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0 bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0 -bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0 bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0 bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0 bedrock/us-west-1/meta.llama3-70b-instruct-v1:0