mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-05 02:41:56 +00:00
fix(e2e): route Bedrock deployments whose region only the proxy can resolve
Almost every Bedrock deployment in the suite declares aws_region_name="os.environ/AWS_REGION". The mount resolver treated that string as a region name, produced a mount nothing serves, and left the whole Anthropic-on-Bedrock surface on its direct path, which is the one thing mounting Bedrock was for. The run pod does not share the proxy's environment, so the harness genuinely cannot resolve that reference. A `us.` inference profile fans out across the US regions and is reachable from any of them, so those route to the default mount whatever the proxy resolved. A model that is not cross-region and declares its region that way keeps its direct path rather than being sent to a region it may not exist in.
This commit is contained in:
parent
b68e60f706
commit
aebfcf7da3
3 changed files with 43 additions and 9 deletions
|
|
@ -766,13 +766,20 @@ def test_registration_preserves_unsupported_or_explicit_routes(params: LiteLLMPa
|
|||
assert route_cache_model(params, unexpected_edge, enabled=True) is params
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [
|
||||
"bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"bedrock/converse/us.anthropic.claude-sonnet-5",
|
||||
"bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
@pytest.mark.parametrize("model,region", [
|
||||
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", None),
|
||||
("bedrock/converse/us.anthropic.claude-sonnet-5", None),
|
||||
("bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", None),
|
||||
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "us-east-1"),
|
||||
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "os.environ/AWS_REGION"),
|
||||
("bedrock/invoke/us.anthropic.claude-sonnet-5", "os.environ/AWS_REGION"),
|
||||
])
|
||||
def test_anthropic_on_bedrock_registers_the_edge_as_its_runtime_endpoint(model: str) -> None:
|
||||
params: Final = LiteLLMParamsBody(model=model)
|
||||
def test_anthropic_on_bedrock_registers_the_edge_as_its_runtime_endpoint(model: str, region: str | None) -> None:
|
||||
"""Almost every Bedrock deployment in the suite declares its region as
|
||||
`os.environ/AWS_REGION`, which only the proxy can resolve. Treating that
|
||||
string as a region name would leave the whole Anthropic-on-Bedrock surface
|
||||
off the edge, which is the point of mounting it at all."""
|
||||
params: Final = LiteLLMParamsBody(model=model, aws_region_name=region)
|
||||
routed: Final = route_cache_model(params, lambda mount: f"http://edge.invalid/{mount}", enabled=True)
|
||||
assert routed.aws_bedrock_runtime_endpoint == "http://edge.invalid/bedrock/us-east-1"
|
||||
assert routed.api_base is None
|
||||
|
|
@ -794,15 +801,19 @@ def test_anthropic_on_bedrock_registers_the_edge_as_its_runtime_endpoint(model:
|
|||
aws_bedrock_runtime_endpoint="https://custom.invalid",
|
||||
),
|
||||
LiteLLMParamsBody(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", aws_region_name="eu-west-1"),
|
||||
LiteLLMParamsBody(model="bedrock/anthropic.claude-sonnet-5", aws_region_name="os.environ/AWS_REGION"),
|
||||
LiteLLMParamsBody(model="bedrock/invoke/eu.anthropic.claude-sonnet-5", aws_region_name="os.environ/AWS_REGION"),
|
||||
])
|
||||
def test_bedrock_deployments_the_edge_must_not_touch_keep_their_direct_route(params: LiteLLMParamsBody) -> None:
|
||||
"""Non-Anthropic models the runner role cannot invoke, deployments carrying
|
||||
their own AWS identity (routing those would replace the assume-role chain the
|
||||
batch suite exists to prove), explicit endpoints, and unmounted regions."""
|
||||
batch suite exists to prove), explicit endpoints, unmounted regions, and a
|
||||
region only the proxy can resolve on a model that is not cross-region, whose
|
||||
real region the harness cannot know."""
|
||||
routed: Final = route_cache_model(
|
||||
params, lambda mount: None if mount not in EDGE_MOUNTS else f"http://edge.invalid/{mount}", enabled=True,
|
||||
)
|
||||
assert routed is params or routed.aws_bedrock_runtime_endpoint == params.aws_bedrock_runtime_endpoint
|
||||
assert routed is params
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["batch", "realtime", "image_generation"])
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ An eligible miss calls the provider. A complete successful response is stored im
|
|||
|
||||
Bedrock could not be mounted before because SigV4 signs the `Host` header, so a rewritten `api_base` failed signature verification at the provider. The edge now re-signs: it drops the proxy's signature headers, signs the upstream request with the run pod's own AWS identity from its EKS Pod Identity association, and forwards that. The signature headers are excluded from the key, since `x-amz-date` is a timestamp and keying on it would make every Bedrock call a permanent miss
|
||||
|
||||
Almost every Bedrock deployment in the suite declares its region as `os.environ/AWS_REGION`, which only the proxy can resolve, and the run pod does not share that environment. A `us.` inference profile fans out across the US regions and is reachable from any of them, so those route to the default mount whatever the proxy resolved. A model that is not cross-region and declares its region that way keeps its direct path rather than being sent to a region it may not exist in.
|
||||
|
||||
Only deployments that carry no AWS identity of their own route to the edge. A deployment with `aws_role_name`, `aws_access_key_id`, an `api_base` or an `aws_bedrock_runtime_endpoint` keeps its direct path, because re-signing it would quietly replace the very credential chain that test exists to prove. Only Anthropic models route, matching what the runner role is allowed to invoke and what the edge knows how to validate
|
||||
|
||||
Vertex and Gemini are not mounted. litellm's `_check_custom_proxy` rewrites a path-prefixed Vertex `api_base` into `{api_base}:{endpoint}`, dropping project, location and model, so a mount under a path prefix cannot work without either a root-mounted edge on its own port or a change in litellm
|
||||
|
|
|
|||
|
|
@ -10,6 +10,26 @@ LIVE_PROVIDER_REQUIRED: Final[ContextVar[bool]] = ContextVar("live_provider_requ
|
|||
|
||||
DEFAULT_BEDROCK_REGION: Final = "us-east-1"
|
||||
BEDROCK_ANTHROPIC_INFIX: Final = "anthropic."
|
||||
BEDROCK_CROSS_REGION_PREFIX: Final = "us."
|
||||
ENV_REFERENCE_PREFIX: Final = "os.environ/"
|
||||
|
||||
|
||||
def bedrock_region(declared: str | None, model: str) -> str | None:
|
||||
"""The region whose edge mount a deployment belongs to, or None when the
|
||||
harness cannot know it.
|
||||
|
||||
Most Bedrock deployments declare `os.environ/AWS_REGION`, which the proxy
|
||||
resolves from its own environment. The run pod does not share that
|
||||
environment, so the harness genuinely does not know the region. A `us.`
|
||||
inference profile fans out across the US regions and is reachable from any
|
||||
of them, so the default entry point is correct for those whatever the proxy
|
||||
resolved; anything else keeps its direct path rather than being sent to a
|
||||
region the model may not exist in."""
|
||||
if declared is None:
|
||||
return DEFAULT_BEDROCK_REGION
|
||||
if not declared.startswith(ENV_REFERENCE_PREFIX):
|
||||
return declared
|
||||
return DEFAULT_BEDROCK_REGION if model.startswith(BEDROCK_CROSS_REGION_PREFIX) else None
|
||||
|
||||
|
||||
def bedrock_mount(params: LiteLLMParamsBody) -> str | None:
|
||||
|
|
@ -24,7 +44,8 @@ def bedrock_mount(params: LiteLLMParamsBody) -> str | None:
|
|||
model: Final = route.partition("/")[2] or route
|
||||
if BEDROCK_ANTHROPIC_INFIX not in model:
|
||||
return None
|
||||
return f"bedrock/{params.aws_region_name or DEFAULT_BEDROCK_REGION}"
|
||||
region: Final = bedrock_region(params.aws_region_name, model)
|
||||
return None if region is None else f"bedrock/{region}"
|
||||
|
||||
|
||||
def route_bedrock(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue