From c7246adc1dd571069a2cb3c77b3a40b720eb0da6 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 16 Sep 2026 03:29:02 -0700 Subject: [PATCH] fix(e2e): route only the Bedrock models the runner role can invoke The edge re-signs with the run pod's identity, whose IAM policy is an explicit per-model allowlist. Matching on the `anthropic.` infix instead routed every Anthropic-on-Bedrock model, so a model outside the policy came back 403 from Bedrock with no fallback, taking the whole claude_code Bedrock matrix red. An unlisted model now keeps its direct path and loses only caching. --- .../test_provider_cache.py | 4 ++++ tests/e2e/PROVIDER_CACHE.md | 4 +++- tests/e2e/provider_cache_routing.py | 23 ++++++++++++------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/code_coverage_tests/test_provider_cache.py b/tests/code_coverage_tests/test_provider_cache.py index 3dafbadf508..ac1ccd5692b 100644 --- a/tests/code_coverage_tests/test_provider_cache.py +++ b/tests/code_coverage_tests/test_provider_cache.py @@ -809,6 +809,8 @@ def test_registration_preserves_unsupported_or_explicit_routes(params: LiteLLMPa ("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "us-east-1"), ("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "os.environ/AWS_REGION"), ("bedrock/invoke/us.anthropic.claude-sonnet-5", "os.environ/AWS_REGION"), + ("bedrock/us.anthropic.claude-opus-4-7", "us-east-1"), + ("bedrock/converse/us.anthropic.claude-opus-4-7", "us-east-1"), ]) def test_anthropic_on_bedrock_registers_the_edge_as_its_runtime_endpoint(model: str, region: str | None) -> None: """Almost every Bedrock deployment in the suite declares its region as @@ -839,6 +841,8 @@ def test_anthropic_on_bedrock_registers_the_edge_as_its_runtime_endpoint(model: LiteLLMParamsBody(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", aws_region_name="eu-west-1"), LiteLLMParamsBody(model="bedrock/anthropic.claude-sonnet-5", aws_region_name="os.environ/AWS_REGION"), LiteLLMParamsBody(model="bedrock/invoke/eu.anthropic.claude-sonnet-5", aws_region_name="os.environ/AWS_REGION"), + LiteLLMParamsBody(model="bedrock/us.anthropic.claude-opus-4-5", aws_region_name="us-east-1"), + LiteLLMParamsBody(model="bedrock/converse/us.anthropic.claude-haiku-9-9", aws_region_name="us-east-1"), ]) def test_bedrock_deployments_the_edge_must_not_touch_keep_their_direct_route(params: LiteLLMParamsBody) -> None: """Non-Anthropic models the runner role cannot invoke, deployments carrying diff --git a/tests/e2e/PROVIDER_CACHE.md b/tests/e2e/PROVIDER_CACHE.md index c530574cda2..3a29c7b6fe4 100644 --- a/tests/e2e/PROVIDER_CACHE.md +++ b/tests/e2e/PROVIDER_CACHE.md @@ -22,7 +22,9 @@ Bedrock could not be mounted before because SigV4 signs the `Host` header, so a Almost every Bedrock deployment in the suite declares its region as `os.environ/AWS_REGION`, which only the proxy can resolve, and the run pod does not share that environment. A `us.` inference profile fans out across the US regions and is reachable from any of them, so those route to the default mount whatever the proxy resolved. A model that is not cross-region and declares its region that way keeps its direct path rather than being sent to a region it may not exist in. -Only deployments that carry no AWS identity of their own route to the edge. A deployment with `aws_role_name`, `aws_access_key_id`, an `api_base` or an `aws_bedrock_runtime_endpoint` keeps its direct path, because re-signing it would quietly replace the very credential chain that test exists to prove. Only Anthropic models route, matching what the runner role is allowed to invoke and what the edge knows how to validate +Only deployments that carry no AWS identity of their own route to the edge. A deployment with `aws_role_name`, `aws_access_key_id`, an `api_base` or an `aws_bedrock_runtime_endpoint` keeps its direct path, because re-signing it would quietly replace the very credential chain that test exists to prove + +Which models route is an explicit allowlist in `provider_cache_routing.py`, mirroring the runner role's IAM policy, which names its models one by one. That coupling is deliberate: the edge re-signs with the run pod's identity, so a model the role cannot invoke comes back 403 from Bedrock rather than falling back. An unlisted model keeps its direct path and loses only caching, so adding a Bedrock model to the suite can never turn it red. Adding one to the edge is a policy edit in litellm-ops plus a line here Vertex and Gemini are not mounted. litellm's `_check_custom_proxy` rewrites a path-prefixed Vertex `api_base` into `{api_base}:{endpoint}`, dropping project, location and model, so a mount under a path prefix cannot work without either a root-mounted edge on its own port or a change in litellm diff --git a/tests/e2e/provider_cache_routing.py b/tests/e2e/provider_cache_routing.py index d2237bb49bc..97e05344423 100644 --- a/tests/e2e/provider_cache_routing.py +++ b/tests/e2e/provider_cache_routing.py @@ -9,8 +9,15 @@ from models import LiteLLMParamsBody, ModelMode LIVE_PROVIDER_REQUIRED: Final[ContextVar[bool]] = ContextVar("live_provider_required", default=False) DEFAULT_BEDROCK_REGION: Final = "us-east-1" -BEDROCK_ANTHROPIC_INFIX: Final = "anthropic." BEDROCK_CROSS_REGION_PREFIX: Final = "us." +BEDROCK_EDGE_MODELS: Final = frozenset( + { + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "us.anthropic.claude-sonnet-5", + "us.anthropic.claude-opus-4-7", + } +) ENV_REFERENCE_PREFIX: Final = "os.environ/" @@ -33,16 +40,16 @@ def bedrock_region(declared: str | None, model: str) -> str | None: def bedrock_mount(params: LiteLLMParamsBody) -> str | None: - """The edge mount an Anthropic-on-Bedrock deployment belongs to, or None. + """The edge mount a Bedrock deployment belongs to, or None. - Only the Anthropic models route. The edge validates converse and invoke - bodies by their Anthropic and Converse terminator fields, and the runner role - is allowed to invoke exactly those models, so Bedrock embeddings, image - generation, rerank and realtime keep their existing direct path rather than - reaching an edge that could neither sign nor validate for them.""" + The allowlist mirrors the runner role's IAM policy, which names its models + one by one. A model outside it would be re-signed with an identity that + cannot invoke it and come back 403 from Bedrock, so an unlisted model keeps + its direct path and loses only caching. Adding a model is a policy edit in + litellm-ops and a line here.""" route: Final = params.model.partition("/")[2] model: Final = route.partition("/")[2] or route - if BEDROCK_ANTHROPIC_INFIX not in model: + if model not in BEDROCK_EDGE_MODELS: return None region: Final = bedrock_region(params.aws_region_name, model) return None if region is None else f"bedrock/{region}"