diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 0743bd6e7e7..039e6e24f96 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -79,6 +79,18 @@ def _without_region_path(model: str) -> str: return model if region is None else model_id +_ASYNC_INVOKE_REQUEST_FIELDS: Final = frozenset({"clientRequestToken", "tags"}) + + +def _split_async_invoke_extra_body( + extra_body: Mapping[str, object], +) -> tuple[Mapping[str, object], Mapping[str, object]]: + return ( + {key: value for key, value in extra_body.items() if key in _ASYNC_INVOKE_REQUEST_FIELDS}, + {key: value for key, value in extra_body.items() if key not in _ASYNC_INVOKE_REQUEST_FIELDS}, + ) + + class BedrockEmbedding(BaseAWSLLM): _web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = BEDROCK_EMBEDDING_SESSION_POLICY @@ -296,7 +308,13 @@ class BedrockEmbedding(BaseAWSLLM): extra_body: Mapping[str, object] = MappingProxyType({}), ): responses: Final[list[dict]] = [] - for data in ({**extra_body, **item} for item in batch_data): + request_fields, model_fields = _split_async_invoke_extra_body(extra_body) + for data in ( + {**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}} + if "modelInput" in item + else {**extra_body, **item} + for item in batch_data + ): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} @@ -366,7 +384,13 @@ class BedrockEmbedding(BaseAWSLLM): extra_body: Mapping[str, object] = MappingProxyType({}), ): responses: Final[list[dict]] = [] - for data in ({**extra_body, **item} for item in batch_data): + request_fields, model_fields = _split_async_invoke_extra_body(extra_body) + for data in ( + {**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}} + if "modelInput" in item + else {**extra_body, **item} + for item in batch_data + ): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} diff --git a/tests/integration/providers/test_bedrock_async_invoke_embedding_wire.py b/tests/integration/providers/test_bedrock_async_invoke_embedding_wire.py new file mode 100644 index 00000000000..25993d8fb8a --- /dev/null +++ b/tests/integration/providers/test_bedrock_async_invoke_embedding_wire.py @@ -0,0 +1,48 @@ +import json +from typing import Final + +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +MODEL_ID: Final = "amazon.nova-2-multimodal-embeddings-v1:0" +TOKEN: Final = "synthetic-bedrock-bearer" +INPUT: Final = "hello world" +S3_URI: Final = "s3://synthetic-bucket/out/" +SEGMENTATION: Final = {"durationSeconds": 10} +REQUEST_TOKEN: Final = "retry-1" +INVOCATION_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123" + + +def start_async_invoke_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/async-invoke", request.target + assert request.headers["authorization"] == f"Bearer {TOKEN}" + body: Final = json.loads(request.body) + assert body.keys() == {"modelId", "modelInput", "outputDataConfig", "clientRequestToken"}, request.body + assert body["clientRequestToken"] == REQUEST_TOKEN, request.body + assert "clientRequestToken" not in body["modelInput"], request.body + assert body["modelId"] == MODEL_ID, request.body + assert body["outputDataConfig"] == {"s3OutputDataConfig": {"s3Uri": S3_URI}}, request.body + assert body["modelInput"]["segmentationConfig"] == SEGMENTATION, request.body + return Reply(body=json.dumps({"invocationArn": INVOCATION_ARN}).encode()) + + +def test_async_invoke_embedding_extra_body_fields_reach_bedrock_inside_model_input(gateway: Gateway) -> None: + with wire_server(start_async_invoke_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/async_invoke/{MODEL_ID}", + api_key=TOKEN, + api_base=wire.url, + aws_region_name="us-east-1", + output_s3_uri=S3_URI, + ) + response: Final = gateway.request( + "POST", + "/v1/embeddings", + { + "model": model, + "input": INPUT, + "extra_body": {"segmentationConfig": SEGMENTATION, "clientRequestToken": REQUEST_TOKEN}, + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1, "the async-invoke request never reached Bedrock" diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index b128ad1a564..ff11414ca79 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -2221,6 +2221,39 @@ def test_bedrock_nova_embedding_sends_extra_body_fields_in_the_request_body(resp assert body["segmentationConfig"] == {"durationSeconds": 10} +@pytest.mark.parametrize( + ("model_id", "extra_body", "mapped_field"), + [ + ("amazon.nova-2-multimodal-embeddings-v1:0", {"segmentationConfig": {"durationSeconds": 10}}, "taskType"), + ("twelvelabs.marengo-embed-2-7-v1:0", {"minClipSec": 4}, "inputText"), + ], +) +def test_bedrock_async_invoke_embedding_sends_extra_body_fields_inside_model_input( + respx_mock: respx.MockRouter, model_id: str, extra_body: dict[str, object], mapped_field: str +) -> None: + route: Final = respx_mock.post("https://bedrock-runtime.us-east-1.amazonaws.com/async-invoke").mock( + return_value=httpx.Response( + 200, json={"invocationArn": "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"} + ) + ) + request_fields: Final = {"clientRequestToken": "retry-1", "tags": [{"key": "team", "value": "search"}]} + request: Final = { + "model": f"bedrock/async_invoke/{model_id}", + "input": [test_input], + "api_key": "test-bearer-token", + "aws_region_name": "us-east-1", + "output_s3_uri": "s3://my-bucket/out/", + } + + litellm.embedding(**request) + plain: Final = json.loads(route.calls.last.request.read()) + litellm.embedding(**request, extra_body={**extra_body, **request_fields, mapped_field: "from extra_body"}) + body: Final = json.loads(route.calls.last.request.read()) + + assert plain.keys() == {"modelId", "modelInput", "outputDataConfig"} + assert body == {**plain, **request_fields, "modelInput": {**plain["modelInput"], **extra_body}} + + @pytest.mark.parametrize( ("extra_body", "field", "expected"), [