fix(bedrock): put extra_body fields inside modelInput for async-invoke embeddings (#51676)

* fix(bedrock): put extra_body fields inside modelInput for async-invoke embeddings

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(bedrock): keep clientRequestToken and tags at the top level of async-invoke requests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(bedrock): cover clientRequestToken staying top-level in the async-invoke wire test

---------

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-11-15 11:30:25 -08:00 • committed by GitHub
parent 2d487abfc1
commit 45c05c5bea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 107 additions and 2 deletions

View file

@ -79,6 +79,18 @@ def _without_region_path(model: str) -> str:
return model if region is None else model_id
_ASYNC_INVOKE_REQUEST_FIELDS: Final = frozenset({"clientRequestToken", "tags"})
def _split_async_invoke_extra_body(
extra_body: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
return (
{key: value for key, value in extra_body.items() if key in _ASYNC_INVOKE_REQUEST_FIELDS},
{key: value for key, value in extra_body.items() if key not in _ASYNC_INVOKE_REQUEST_FIELDS},
)
class BedrockEmbedding(BaseAWSLLM):
_web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = BEDROCK_EMBEDDING_SESSION_POLICY
@ -296,7 +308,13 @@ class BedrockEmbedding(BaseAWSLLM):
extra_body: Mapping[str, object] = MappingProxyType({}),
):
responses: Final[list[dict]] = []
for data in ({**extra_body, **item} for item in batch_data):
request_fields, model_fields = _split_async_invoke_extra_body(extra_body)
for data in (
{**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}}
if "modelInput" in item
else {**extra_body, **item}
for item in batch_data
):
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
@ -366,7 +384,13 @@ class BedrockEmbedding(BaseAWSLLM):
extra_body: Mapping[str, object] = MappingProxyType({}),
):
responses: Final[list[dict]] = []
for data in ({**extra_body, **item} for item in batch_data):
request_fields, model_fields = _split_async_invoke_extra_body(extra_body)
for data in (
{**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}}
if "modelInput" in item
else {**extra_body, **item}
for item in batch_data
):
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}

View file

@ -0,0 +1,48 @@
import json
from typing import Final
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
MODEL_ID: Final = "amazon.nova-2-multimodal-embeddings-v1:0"
TOKEN: Final = "synthetic-bedrock-bearer"
INPUT: Final = "hello world"
S3_URI: Final = "s3://synthetic-bucket/out/"
SEGMENTATION: Final = {"durationSeconds": 10}
REQUEST_TOKEN: Final = "retry-1"
INVOCATION_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"
def start_async_invoke_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/async-invoke", request.target
assert request.headers["authorization"] == f"Bearer {TOKEN}"
body: Final = json.loads(request.body)
assert body.keys() == {"modelId", "modelInput", "outputDataConfig", "clientRequestToken"}, request.body
assert body["clientRequestToken"] == REQUEST_TOKEN, request.body
assert "clientRequestToken" not in body["modelInput"], request.body
assert body["modelId"] == MODEL_ID, request.body
assert body["outputDataConfig"] == {"s3OutputDataConfig": {"s3Uri": S3_URI}}, request.body
assert body["modelInput"]["segmentationConfig"] == SEGMENTATION, request.body
return Reply(body=json.dumps({"invocationArn": INVOCATION_ARN}).encode())
def test_async_invoke_embedding_extra_body_fields_reach_bedrock_inside_model_input(gateway: Gateway) -> None:
with wire_server(start_async_invoke_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/async_invoke/{MODEL_ID}",
api_key=TOKEN,
api_base=wire.url,
aws_region_name="us-east-1",
output_s3_uri=S3_URI,
)
response: Final = gateway.request(
"POST",
"/v1/embeddings",
{
"model": model,
"input": INPUT,
"extra_body": {"segmentationConfig": SEGMENTATION, "clientRequestToken": REQUEST_TOKEN},
},
)
assert response.status_code == 200, response.text
assert len(wire.drain()) == 1, "the async-invoke request never reached Bedrock"

View file

@ -2221,6 +2221,39 @@ def test_bedrock_nova_embedding_sends_extra_body_fields_in_the_request_body(resp
assert body["segmentationConfig"] == {"durationSeconds": 10}
@pytest.mark.parametrize(
("model_id", "extra_body", "mapped_field"),
[
("amazon.nova-2-multimodal-embeddings-v1:0", {"segmentationConfig": {"durationSeconds": 10}}, "taskType"),
("twelvelabs.marengo-embed-2-7-v1:0", {"minClipSec": 4}, "inputText"),
],
)
def test_bedrock_async_invoke_embedding_sends_extra_body_fields_inside_model_input(
respx_mock: respx.MockRouter, model_id: str, extra_body: dict[str, object], mapped_field: str
) -> None:
route: Final = respx_mock.post("https://bedrock-runtime.us-east-1.amazonaws.com/async-invoke").mock(
return_value=httpx.Response(
200, json={"invocationArn": "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"}
)
)
request_fields: Final = {"clientRequestToken": "retry-1", "tags": [{"key": "team", "value": "search"}]}
request: Final = {
"model": f"bedrock/async_invoke/{model_id}",
"input": [test_input],
"api_key": "test-bearer-token",
"aws_region_name": "us-east-1",
"output_s3_uri": "s3://my-bucket/out/",
}
litellm.embedding(**request)
plain: Final = json.loads(route.calls.last.request.read())
litellm.embedding(**request, extra_body={**extra_body, **request_fields, mapped_field: "from extra_body"})
body: Final = json.loads(route.calls.last.request.read())
assert plain.keys() == {"modelId", "modelInput", "outputDataConfig"}
assert body == {**plain, **request_fields, "modelInput": {**plain["modelInput"], **extra_body}}
@pytest.mark.parametrize(
("extra_body", "field", "expected"),
[