mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
fix(bedrock embed): strip cache_control_injection_points from inference_params
`cache_control_injection_points` is a chat-completion-only param consumed by `AnthropicCacheControlHook`. The Bedrock embedding router was passing through every non-AWS-auth key of `optional_params` to the per-provider `_transform_request` helpers, which forward unknown keys verbatim into the outgoing JSON. AWS rejected the resulting embedding request with `Malformed input request: #: extraneous key [cache_control_injection_points] is not permitted` (closes #30314). Pop the key in `embedding.py` at the same routing boundary that already strips `user` and AWS-auth params, so the fix covers every provider (titan-text, titan-image, titan-v2, cohere, twelvelabs, nova) at once instead of needing the same line in each transformer. Add a regression test under `tests/test_litellm/llms/bedrock/embed` that asserts the field never reaches the request body when set on a `titan-embed-image-v1` embedding call.
This commit is contained in:
parent
0c3017e1de
commit
a1f341fcba
2 changed files with 48 additions and 1 deletions
|
|
@ -402,7 +402,17 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
inference_params = {
|
||||
k: v for k, v in inference_params.items() if k.lower() not in self.aws_authentication_params
|
||||
}
|
||||
inference_params.pop("user", None) # make sure user is not passed in for bedrock call
|
||||
inference_params.pop(
|
||||
"user", None
|
||||
) # make sure user is not passed in for bedrock call
|
||||
# `cache_control_injection_points` is a chat-completion-only param
|
||||
# consumed by `AnthropicCacheControlHook`. Bedrock embedding APIs
|
||||
# reject it (`extraneous key [cache_control_injection_points] is not
|
||||
# permitted`) and the per-provider _transform_request helpers below
|
||||
# forward any unknown `inference_params` keys verbatim into the
|
||||
# outgoing JSON, so strip it here at the routing boundary rather
|
||||
# than having to remember it in every transformer.
|
||||
inference_params.pop("cache_control_injection_points", None)
|
||||
|
||||
data: CohereEmbeddingRequest | None = None
|
||||
batch_data: list | None = None
|
||||
|
|
|
|||
|
|
@ -1004,3 +1004,40 @@ def test_bedrock_cohere_embedding_types_wrapped_as_list(
|
|||
assert "embedding_types" in request_body
|
||||
assert request_body["embedding_types"] == expected_embedding_types
|
||||
assert isinstance(request_body["embedding_types"], list)
|
||||
|
||||
|
||||
def test_bedrock_embedding_strips_cache_control_injection_points():
|
||||
"""
|
||||
Issue #30314: passing `cache_control_injection_points` (a chat-completion-only
|
||||
param consumed by AnthropicCacheControlHook) to a Bedrock embedding call was
|
||||
forwarded verbatim into the outgoing JSON, triggering
|
||||
`Malformed input request: #: extraneous key [cache_control_injection_points]
|
||||
is not permitted` from AWS. The routing layer in embedding.py should strip
|
||||
the key before the per-provider transformer touches the request body.
|
||||
"""
|
||||
litellm.set_verbose = True
|
||||
client = HTTPHandler()
|
||||
|
||||
with patch.object(client, "post") as mock_post:
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = json.dumps(titan_embedding_response)
|
||||
mock_response.json = lambda: json.loads(mock_response.text)
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
litellm.embedding(
|
||||
model="bedrock/amazon.titan-embed-image-v1",
|
||||
input=[test_input],
|
||||
cache_control_injection_points=[
|
||||
{"location": "message", "role": "user"}
|
||||
],
|
||||
client=client,
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
api_key="test-bearer-token-12345",
|
||||
)
|
||||
|
||||
request_body = json.loads(mock_post.call_args.kwargs.get("data", "{}"))
|
||||
assert (
|
||||
"cache_control_injection_points" not in request_body
|
||||
), "cache_control_injection_points must not leak into Bedrock embedding requests"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue