From 2d487abfc1c88e9f3c560df2bc00b4ebcb741821 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sun, 15 Nov 2026 11:30:21 -0800 Subject: [PATCH] fix(vertex_ai): send extra_body fields as predict parameters for BGE embedding models (#51662) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/vertex_ai/vertex_embeddings/bge.py | 10 +++++-- .../providers/test_vertex_embedding_wire.py | 29 +++++++++++++++++++ .../test_embedding_handler.py | 15 +++++++--- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_embeddings/bge.py b/litellm/llms/vertex_ai/vertex_embeddings/bge.py index 71135a66fa5..c13e31aa1c8 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/bge.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/bge.py @@ -10,8 +10,9 @@ Model name handling: - This module focuses on request/response transformation only """ -from typing import Final +from typing import Final, cast # noqa: TID251 # extra_body carries raw fields the TypedDict lacks +from litellm.llms.base_llm.embedding.transformation import split_extra_body from litellm.types.utils import EmbeddingResponse, Usage from .types import ( @@ -84,7 +85,12 @@ class VertexBGEConfig: vertex_text_embedding_input_list.append(embedding_input) vertex_request["instances"] = vertex_text_embedding_input_list - vertex_request["parameters"] = EmbeddingParameters(**optional_params) + body_params, extra_body = split_extra_body(optional_params) + vertex_request["parameters"] = ( + cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list + EmbeddingParameters, {**extra_body, **body_params} + ) + ) return vertex_request diff --git a/tests/integration/providers/test_vertex_embedding_wire.py b/tests/integration/providers/test_vertex_embedding_wire.py index d30b6e4e3ae..8ee0274d7f1 100644 --- a/tests/integration/providers/test_vertex_embedding_wire.py +++ b/tests/integration/providers/test_vertex_embedding_wire.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter _PROJECT: Final = "scripted-embedding-project" _LOCATION: Final = "us-central1" _VERTEX_MODEL: Final = "text-embedding-005" +_VERTEX_BGE_MODEL: Final = "bge-small-en-v1.5" _VERTEX_INPUTS: Final = ("vertex embedding alpha", "vertex embedding bravo") _GEMINI_INPUTS: Final = ("gemini embedding alpha", "gemini embedding bravo") _VECTOR_A: Final = (1.0,) + (0.0,) * 255 @@ -105,6 +106,34 @@ def test_vertex_text_embedding_sends_dimensionality_and_preserves_input_order( ] +def test_vertex_bge_embedding_sends_extra_body_fields_as_predict_parameters(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert _JSON_OBJECT.validate_json(request.body) == { + "instances": [{"prompt": _VERTEX_INPUTS[0]}], + "parameters": {"autoTruncate": False}, + } + return Reply(body=json.dumps({"predictions": [list(_VECTOR_A)]}).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"vertex_ai/{_VERTEX_BGE_MODEL}", + api_base=wire.url, + api_key=None, + vertex_project=_PROJECT, + vertex_location=_LOCATION, + vertex_credentials=service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")), + ) + response: Final = gateway.request( + "POST", + "/v1/embeddings", + {"model": model, "input": [_VERTEX_INPUTS[0]], "extra_body": {"autoTruncate": False}}, + ) + assert response.status_code == 200, response.text + payload: Final = _EmbeddingResponse.model_validate_json(response.content) + assert [item.embedding for item in payload.data] == [list(_VECTOR_A)], response.text + assert len(wire.drain()) == 1 + + @pytest.mark.parametrize( ("embedding_input", "expected_inputs"), ((list(_GEMINI_INPUTS), _GEMINI_INPUTS), (_GEMINI_INPUTS[0], (_GEMINI_INPUTS[0],))), diff --git a/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py index 0e8d49b01e5..e7643cc0049 100644 --- a/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py +++ b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py @@ -307,11 +307,18 @@ async def test_vertex_embedding_usage_carries_the_billable_character_count_googl ) -def test_vertex_embedding_call_sends_extra_body_fields_as_predict_parameters(predict_endpoint: respx.Route) -> None: - predict_endpoint.mock(return_value=_predict_response({"predictions": [_gecko_prediction([0.1, 2], 3)]})) +@pytest.mark.parametrize( + ("model", "prediction", "instance"), + [(GECKO, _gecko_prediction([0.1, 2], 3), {"content": "hello"}), (BGE, [0.1, 2], {"prompt": "hello"})], + ids=["gecko", "bge"], +) +def test_vertex_embedding_call_sends_extra_body_fields_as_predict_parameters( + predict_endpoint: respx.Route, model: str, prediction: object, instance: dict[str, str] +) -> None: + predict_endpoint.mock(return_value=_predict_response({"predictions": [prediction]})) litellm.embedding( - model=f"vertex_ai/{GECKO}", + model=f"vertex_ai/{model}", input=["hello"], vertex_project=PUBLIC_CALL_PROJECT, vertex_location="us-central1", @@ -319,6 +326,6 @@ def test_vertex_embedding_call_sends_extra_body_fields_as_predict_parameters(pre ) body: Final = json.loads(predict_endpoint.calls.last.request.content) - assert body["instances"] == [{"content": "hello"}] + assert body["instances"] == [instance] assert "extra_body" not in body["parameters"] assert body["parameters"]["autoTruncate"] is False