mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(bedrock): put extra_body fields inside modelInput for async-invoke embeddings (#51676)
* fix(bedrock): put extra_body fields inside modelInput for async-invoke embeddings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep clientRequestToken and tags at the top level of async-invoke requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): cover clientRequestToken staying top-level in the async-invoke wire test --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
2d487abfc1
commit
45c05c5bea
3 changed files with 107 additions and 2 deletions
|
|
@ -79,6 +79,18 @@ def _without_region_path(model: str) -> str:
|
|||
return model if region is None else model_id
|
||||
|
||||
|
||||
_ASYNC_INVOKE_REQUEST_FIELDS: Final = frozenset({"clientRequestToken", "tags"})
|
||||
|
||||
|
||||
def _split_async_invoke_extra_body(
|
||||
extra_body: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
return (
|
||||
{key: value for key, value in extra_body.items() if key in _ASYNC_INVOKE_REQUEST_FIELDS},
|
||||
{key: value for key, value in extra_body.items() if key not in _ASYNC_INVOKE_REQUEST_FIELDS},
|
||||
)
|
||||
|
||||
|
||||
class BedrockEmbedding(BaseAWSLLM):
|
||||
_web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = BEDROCK_EMBEDDING_SESSION_POLICY
|
||||
|
||||
|
|
@ -296,7 +308,13 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
):
|
||||
responses: Final[list[dict]] = []
|
||||
for data in ({**extra_body, **item} for item in batch_data):
|
||||
request_fields, model_fields = _split_async_invoke_extra_body(extra_body)
|
||||
for data in (
|
||||
{**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}}
|
||||
if "modelInput" in item
|
||||
else {**extra_body, **item}
|
||||
for item in batch_data
|
||||
):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if extra_headers is not None:
|
||||
headers = {"Content-Type": "application/json", **extra_headers}
|
||||
|
|
@ -366,7 +384,13 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
):
|
||||
responses: Final[list[dict]] = []
|
||||
for data in ({**extra_body, **item} for item in batch_data):
|
||||
request_fields, model_fields = _split_async_invoke_extra_body(extra_body)
|
||||
for data in (
|
||||
{**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}}
|
||||
if "modelInput" in item
|
||||
else {**extra_body, **item}
|
||||
for item in batch_data
|
||||
):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if extra_headers is not None:
|
||||
headers = {"Content-Type": "application/json", **extra_headers}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL_ID: Final = "amazon.nova-2-multimodal-embeddings-v1:0"
|
||||
TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
INPUT: Final = "hello world"
|
||||
S3_URI: Final = "s3://synthetic-bucket/out/"
|
||||
SEGMENTATION: Final = {"durationSeconds": 10}
|
||||
REQUEST_TOKEN: Final = "retry-1"
|
||||
INVOCATION_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"
|
||||
|
||||
|
||||
def start_async_invoke_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/async-invoke", request.target
|
||||
assert request.headers["authorization"] == f"Bearer {TOKEN}"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body.keys() == {"modelId", "modelInput", "outputDataConfig", "clientRequestToken"}, request.body
|
||||
assert body["clientRequestToken"] == REQUEST_TOKEN, request.body
|
||||
assert "clientRequestToken" not in body["modelInput"], request.body
|
||||
assert body["modelId"] == MODEL_ID, request.body
|
||||
assert body["outputDataConfig"] == {"s3OutputDataConfig": {"s3Uri": S3_URI}}, request.body
|
||||
assert body["modelInput"]["segmentationConfig"] == SEGMENTATION, request.body
|
||||
return Reply(body=json.dumps({"invocationArn": INVOCATION_ARN}).encode())
|
||||
|
||||
|
||||
def test_async_invoke_embedding_extra_body_fields_reach_bedrock_inside_model_input(gateway: Gateway) -> None:
|
||||
with wire_server(start_async_invoke_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock/async_invoke/{MODEL_ID}",
|
||||
api_key=TOKEN,
|
||||
api_base=wire.url,
|
||||
aws_region_name="us-east-1",
|
||||
output_s3_uri=S3_URI,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/embeddings",
|
||||
{
|
||||
"model": model,
|
||||
"input": INPUT,
|
||||
"extra_body": {"segmentationConfig": SEGMENTATION, "clientRequestToken": REQUEST_TOKEN},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(wire.drain()) == 1, "the async-invoke request never reached Bedrock"
|
||||
|
|
@ -2221,6 +2221,39 @@ def test_bedrock_nova_embedding_sends_extra_body_fields_in_the_request_body(resp
|
|||
assert body["segmentationConfig"] == {"durationSeconds": 10}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "extra_body", "mapped_field"),
|
||||
[
|
||||
("amazon.nova-2-multimodal-embeddings-v1:0", {"segmentationConfig": {"durationSeconds": 10}}, "taskType"),
|
||||
("twelvelabs.marengo-embed-2-7-v1:0", {"minClipSec": 4}, "inputText"),
|
||||
],
|
||||
)
|
||||
def test_bedrock_async_invoke_embedding_sends_extra_body_fields_inside_model_input(
|
||||
respx_mock: respx.MockRouter, model_id: str, extra_body: dict[str, object], mapped_field: str
|
||||
) -> None:
|
||||
route: Final = respx_mock.post("https://bedrock-runtime.us-east-1.amazonaws.com/async-invoke").mock(
|
||||
return_value=httpx.Response(
|
||||
200, json={"invocationArn": "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"}
|
||||
)
|
||||
)
|
||||
request_fields: Final = {"clientRequestToken": "retry-1", "tags": [{"key": "team", "value": "search"}]}
|
||||
request: Final = {
|
||||
"model": f"bedrock/async_invoke/{model_id}",
|
||||
"input": [test_input],
|
||||
"api_key": "test-bearer-token",
|
||||
"aws_region_name": "us-east-1",
|
||||
"output_s3_uri": "s3://my-bucket/out/",
|
||||
}
|
||||
|
||||
litellm.embedding(**request)
|
||||
plain: Final = json.loads(route.calls.last.request.read())
|
||||
litellm.embedding(**request, extra_body={**extra_body, **request_fields, mapped_field: "from extra_body"})
|
||||
body: Final = json.loads(route.calls.last.request.read())
|
||||
|
||||
assert plain.keys() == {"modelId", "modelInput", "outputDataConfig"}
|
||||
assert body == {**plain, **request_fields, "modelInput": {**plain["modelInput"], **extra_body}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extra_body", "field", "expected"),
|
||||
[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue