mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(bedrock): close anthropic passthrough metadata to provider schema at egress
Anthropic-on-Bedrock accepts metadata.user_id and rejects every other metadata field with "metadata.<field>: Extra inputs are not permitted". The proxy enriches the request metadata with internal spend fields (tags, spend_logs_metadata, user_api_key_*) and the invoke/converse passthrough route forwards that body verbatim, so the enrichment reaches AWS and 400s the request (#30629) Project the forwarded metadata down to the provider's closed schema (AnthropicMetadata, user_id only) in BedrockPassthroughConfig.sign_request, right before the body is signed. This is a single typed egress choke point that drops any internal field instead of chasing each enrichment site route by route, which is the structural hardening #30301 asks for. A new body is built rather than the cached body mutated, so spend tracking still sees the stripped fields
This commit is contained in:
parent
8bc18388e3
commit
541b4baf85
2 changed files with 149 additions and 3 deletions
|
|
@ -1,10 +1,11 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple, cast
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast
|
||||
|
||||
from httpx import Response
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
from ..common_utils import BedrockEventStreamDecoderBase, BedrockModelInfo
|
||||
|
|
@ -16,6 +17,40 @@ if TYPE_CHECKING:
|
|||
from litellm.types.utils import CostResponseTypes
|
||||
|
||||
|
||||
def restrict_anthropic_bedrock_metadata(
|
||||
request_data: Dict[str, object],
|
||||
) -> Dict[str, object]:
|
||||
"""
|
||||
Project a forwarded Anthropic-on-Bedrock body's ``metadata`` down to the
|
||||
provider's closed schema (:class:`AnthropicMetadata`, ``user_id`` only) before
|
||||
signing.
|
||||
|
||||
AWS rejects every metadata field other than ``user_id`` with
|
||||
``metadata.<field>: Extra inputs are not permitted``. The proxy enriches the
|
||||
request metadata with internal spend fields (tags, spend_logs_metadata,
|
||||
user_api_key_*), and the passthrough route forwards that body verbatim, so any
|
||||
enrichment lands in the provider payload and 400s the request.
|
||||
|
||||
A new body is returned rather than the input mutated, so the internal request
|
||||
data that spend tracking reads keeps the stripped fields.
|
||||
"""
|
||||
metadata = request_data.get("metadata")
|
||||
if "anthropic_version" not in request_data or not isinstance(metadata, dict):
|
||||
return request_data
|
||||
|
||||
metadata_obj = cast(Dict[str, object], metadata)
|
||||
allowed = {
|
||||
k: v for k, v in metadata_obj.items() if k in AnthropicMetadata.model_fields
|
||||
}
|
||||
if allowed == metadata_obj:
|
||||
return request_data
|
||||
|
||||
return {
|
||||
**{k: v for k, v in request_data.items() if k != "metadata"},
|
||||
**({"metadata": allowed} if allowed else {}),
|
||||
}
|
||||
|
||||
|
||||
class BedrockPassthroughConfig(
|
||||
BaseAWSLLM, BedrockModelInfo, BedrockEventStreamDecoderBase, BasePassthroughConfig
|
||||
):
|
||||
|
|
@ -111,7 +146,7 @@ class BedrockPassthroughConfig(
|
|||
service_name="bedrock",
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
request_data=request_data or {},
|
||||
request_data=restrict_anthropic_bedrock_metadata(request_data or {}),
|
||||
api_base=api_base,
|
||||
model=model,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,16 @@ sys.path.insert(
|
|||
0, os.path.abspath("../../../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig
|
||||
from litellm.llms.bedrock.passthrough.transformation import (
|
||||
BedrockPassthroughConfig,
|
||||
restrict_anthropic_bedrock_metadata,
|
||||
)
|
||||
|
||||
_ANTHROPIC_INVOKE_BODY = {
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
|
||||
|
||||
def test_bedrock_passthrough_get_complete_url_default_endpoint():
|
||||
|
|
@ -505,3 +514,105 @@ def test_bedrock_passthrough_model_id_without_arn():
|
|||
f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse"
|
||||
)
|
||||
assert url_str == expected_url
|
||||
|
||||
|
||||
def test_restrict_metadata_strips_internal_tags_keeps_user_id():
|
||||
"""
|
||||
The reported leak (#30629): the proxy merges a key/team/header spend tag into
|
||||
metadata.tags; Anthropic-on-Bedrock 400s on any metadata field but user_id.
|
||||
"""
|
||||
body = {
|
||||
**_ANTHROPIC_INVOKE_BODY,
|
||||
"metadata": {"user_id": "user_abc123", "tags": ["CC:cost-center-123"]},
|
||||
}
|
||||
|
||||
out = restrict_anthropic_bedrock_metadata(body)
|
||||
|
||||
assert out["metadata"] == {"user_id": "user_abc123"}
|
||||
assert out["messages"] == _ANTHROPIC_INVOKE_BODY["messages"]
|
||||
assert out["anthropic_version"] == "bedrock-2023-05-31"
|
||||
|
||||
|
||||
def test_restrict_metadata_does_not_mutate_input_so_spend_tracking_keeps_tags():
|
||||
body = {
|
||||
**_ANTHROPIC_INVOKE_BODY,
|
||||
"metadata": {"user_id": "user_abc123", "tags": ["CC:cost-center-123"]},
|
||||
}
|
||||
|
||||
restrict_anthropic_bedrock_metadata(body)
|
||||
|
||||
assert body["metadata"] == {
|
||||
"user_id": "user_abc123",
|
||||
"tags": ["CC:cost-center-123"],
|
||||
}
|
||||
|
||||
|
||||
def test_restrict_metadata_strips_every_non_user_id_internal_field():
|
||||
body = {
|
||||
**_ANTHROPIC_INVOKE_BODY,
|
||||
"metadata": {
|
||||
"user_id": "user_abc123",
|
||||
"tags": ["a"],
|
||||
"spend_logs_metadata": {"owner": "team-x"},
|
||||
"user_api_key_user_id": "k1",
|
||||
},
|
||||
}
|
||||
|
||||
out = restrict_anthropic_bedrock_metadata(body)
|
||||
|
||||
assert out["metadata"] == {"user_id": "user_abc123"}
|
||||
|
||||
|
||||
def test_restrict_metadata_drops_metadata_when_only_internal_fields_remain():
|
||||
body = {**_ANTHROPIC_INVOKE_BODY, "metadata": {"tags": ["a"]}}
|
||||
|
||||
out = restrict_anthropic_bedrock_metadata(body)
|
||||
|
||||
assert "metadata" not in out
|
||||
|
||||
|
||||
def test_restrict_metadata_is_noop_when_already_closed_to_user_id():
|
||||
body = {**_ANTHROPIC_INVOKE_BODY, "metadata": {"user_id": "user_abc123"}}
|
||||
|
||||
out = restrict_anthropic_bedrock_metadata(body)
|
||||
|
||||
assert out is body
|
||||
|
||||
|
||||
def test_restrict_metadata_is_noop_for_non_anthropic_body():
|
||||
"""A non-Anthropic invoke body (no anthropic_version) is forwarded verbatim."""
|
||||
body = {"inputText": "hi", "metadata": {"tags": ["a"], "foo": "bar"}}
|
||||
|
||||
out = restrict_anthropic_bedrock_metadata(body)
|
||||
|
||||
assert out is body
|
||||
|
||||
|
||||
def test_restrict_metadata_is_noop_without_metadata():
|
||||
out = restrict_anthropic_bedrock_metadata(_ANTHROPIC_INVOKE_BODY)
|
||||
|
||||
assert out is _ANTHROPIC_INVOKE_BODY
|
||||
|
||||
|
||||
def test_sign_request_strips_internal_metadata_before_signing():
|
||||
"""
|
||||
Wiring: sign_request must feed the projected body to the AWS signer, so the bytes
|
||||
AWS signs and receives never carry the internal tag.
|
||||
"""
|
||||
config = BedrockPassthroughConfig()
|
||||
body = {
|
||||
**_ANTHROPIC_INVOKE_BODY,
|
||||
"metadata": {"user_id": "user_abc123", "tags": ["CC:cost-center-123"]},
|
||||
}
|
||||
|
||||
with patch.object(config, "_sign_request", return_value=({}, b"")) as mock_sign:
|
||||
config.sign_request(
|
||||
headers={},
|
||||
litellm_params={},
|
||||
request_data=body,
|
||||
api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/m/invoke",
|
||||
model="anthropic.claude-3-5-sonnet",
|
||||
)
|
||||
|
||||
signed_request_data = mock_sign.call_args.kwargs["request_data"]
|
||||
assert signed_request_data["metadata"] == {"user_id": "user_abc123"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue