fix(e2e): route only the Bedrock models the runner role can invoke

The edge re-signs with the run pod's identity, whose IAM policy is an explicit
per-model allowlist. Matching on the `anthropic.` infix instead routed every
Anthropic-on-Bedrock model, so a model outside the policy came back 403 from
Bedrock with no fallback, taking the whole claude_code Bedrock matrix red.
An unlisted model now keeps its direct path and loses only caching.
This commit is contained in:
Yuneng Jiang 2026-09-16 03:29:02 -07:00
parent 30c6241e3a
commit c7246adc1d
3 changed files with 22 additions and 9 deletions

View file

@ -809,6 +809,8 @@ def test_registration_preserves_unsupported_or_explicit_routes(params: LiteLLMPa
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "us-east-1"),
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "os.environ/AWS_REGION"),
("bedrock/invoke/us.anthropic.claude-sonnet-5", "os.environ/AWS_REGION"),
("bedrock/us.anthropic.claude-opus-4-7", "us-east-1"),
("bedrock/converse/us.anthropic.claude-opus-4-7", "us-east-1"),
])
def test_anthropic_on_bedrock_registers_the_edge_as_its_runtime_endpoint(model: str, region: str | None) -> None:
"""Almost every Bedrock deployment in the suite declares its region as
@ -839,6 +841,8 @@ def test_anthropic_on_bedrock_registers_the_edge_as_its_runtime_endpoint(model:
LiteLLMParamsBody(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", aws_region_name="eu-west-1"),
LiteLLMParamsBody(model="bedrock/anthropic.claude-sonnet-5", aws_region_name="os.environ/AWS_REGION"),
LiteLLMParamsBody(model="bedrock/invoke/eu.anthropic.claude-sonnet-5", aws_region_name="os.environ/AWS_REGION"),
LiteLLMParamsBody(model="bedrock/us.anthropic.claude-opus-4-5", aws_region_name="us-east-1"),
LiteLLMParamsBody(model="bedrock/converse/us.anthropic.claude-haiku-9-9", aws_region_name="us-east-1"),
])
def test_bedrock_deployments_the_edge_must_not_touch_keep_their_direct_route(params: LiteLLMParamsBody) -> None:
"""Non-Anthropic models the runner role cannot invoke, deployments carrying

View file

@ -22,7 +22,9 @@ Bedrock could not be mounted before because SigV4 signs the `Host` header, so a
Almost every Bedrock deployment in the suite declares its region as `os.environ/AWS_REGION`, which only the proxy can resolve, and the run pod does not share that environment. A `us.` inference profile fans out across the US regions and is reachable from any of them, so those route to the default mount whatever the proxy resolved. A model that is not cross-region and declares its region that way keeps its direct path rather than being sent to a region it may not exist in.
Only deployments that carry no AWS identity of their own route to the edge. A deployment with `aws_role_name`, `aws_access_key_id`, an `api_base` or an `aws_bedrock_runtime_endpoint` keeps its direct path, because re-signing it would quietly replace the very credential chain that test exists to prove. Only Anthropic models route, matching what the runner role is allowed to invoke and what the edge knows how to validate
Only deployments that carry no AWS identity of their own route to the edge. A deployment with `aws_role_name`, `aws_access_key_id`, an `api_base` or an `aws_bedrock_runtime_endpoint` keeps its direct path, because re-signing it would quietly replace the very credential chain that test exists to prove
Which models route is an explicit allowlist in `provider_cache_routing.py`, mirroring the runner role's IAM policy, which names its models one by one. That coupling is deliberate: the edge re-signs with the run pod's identity, so a model the role cannot invoke comes back 403 from Bedrock rather than falling back. An unlisted model keeps its direct path and loses only caching, so adding a Bedrock model to the suite can never turn it red. Adding one to the edge is a policy edit in litellm-ops plus a line here
Vertex and Gemini are not mounted. litellm's `_check_custom_proxy` rewrites a path-prefixed Vertex `api_base` into `{api_base}:{endpoint}`, dropping project, location and model, so a mount under a path prefix cannot work without either a root-mounted edge on its own port or a change in litellm

View file

@ -9,8 +9,15 @@ from models import LiteLLMParamsBody, ModelMode
LIVE_PROVIDER_REQUIRED: Final[ContextVar[bool]] = ContextVar("live_provider_required", default=False)
DEFAULT_BEDROCK_REGION: Final = "us-east-1"
BEDROCK_ANTHROPIC_INFIX: Final = "anthropic."
BEDROCK_CROSS_REGION_PREFIX: Final = "us."
BEDROCK_EDGE_MODELS: Final = frozenset(
{
"us.anthropic.claude-haiku-4-5-20251001-v1:0",
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
"us.anthropic.claude-sonnet-5",
"us.anthropic.claude-opus-4-7",
}
)
ENV_REFERENCE_PREFIX: Final = "os.environ/"
@ -33,16 +40,16 @@ def bedrock_region(declared: str | None, model: str) -> str | None:
def bedrock_mount(params: LiteLLMParamsBody) -> str | None:
"""The edge mount an Anthropic-on-Bedrock deployment belongs to, or None.
"""The edge mount a Bedrock deployment belongs to, or None.
Only the Anthropic models route. The edge validates converse and invoke
bodies by their Anthropic and Converse terminator fields, and the runner role
is allowed to invoke exactly those models, so Bedrock embeddings, image
generation, rerank and realtime keep their existing direct path rather than
reaching an edge that could neither sign nor validate for them."""
The allowlist mirrors the runner role's IAM policy, which names its models
one by one. A model outside it would be re-signed with an identity that
cannot invoke it and come back 403 from Bedrock, so an unlisted model keeps
its direct path and loses only caching. Adding a model is a policy edit in
litellm-ops and a line here."""
route: Final = params.model.partition("/")[2]
model: Final = route.partition("/")[2] or route
if BEDROCK_ANTHROPIC_INFIX not in model:
if model not in BEDROCK_EDGE_MODELS:
return None
region: Final = bedrock_region(params.aws_region_name, model)
return None if region is None else f"bedrock/{region}"