fix(vertex_ai): send extra_body fields as predict parameters for BGE embedding models (#51662)

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-11-15 11:30:21 -08:00 • committed by GitHub
parent 19d8e57485
commit 2d487abfc1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 48 additions and 6 deletions

View file

@ -10,8 +10,9 @@ Model name handling:
- This module focuses on request/response transformation only
"""
from typing import Final
from typing import Final, cast # noqa: TID251 # extra_body carries raw fields the TypedDict lacks
from litellm.llms.base_llm.embedding.transformation import split_extra_body
from litellm.types.utils import EmbeddingResponse, Usage
from .types import (
@ -84,7 +85,12 @@ class VertexBGEConfig:
vertex_text_embedding_input_list.append(embedding_input)
vertex_request["instances"] = vertex_text_embedding_input_list
vertex_request["parameters"] = EmbeddingParameters(**optional_params)
body_params, extra_body = split_extra_body(optional_params)
vertex_request["parameters"] = (
cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list
EmbeddingParameters, {**extra_body, **body_params}
)
)
return vertex_request

View file

@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
_PROJECT: Final = "scripted-embedding-project"
_LOCATION: Final = "us-central1"
_VERTEX_MODEL: Final = "text-embedding-005"
_VERTEX_BGE_MODEL: Final = "bge-small-en-v1.5"
_VERTEX_INPUTS: Final = ("vertex embedding alpha", "vertex embedding bravo")
_GEMINI_INPUTS: Final = ("gemini embedding alpha", "gemini embedding bravo")
_VECTOR_A: Final = (1.0,) + (0.0,) * 255
@ -105,6 +106,34 @@ def test_vertex_text_embedding_sends_dimensionality_and_preserves_input_order(
]
def test_vertex_bge_embedding_sends_extra_body_fields_as_predict_parameters(gateway: Gateway) -> None:
def respond(request: Request) -> Reply:
assert _JSON_OBJECT.validate_json(request.body) == {
"instances": [{"prompt": _VERTEX_INPUTS[0]}],
"parameters": {"autoTruncate": False},
}
return Reply(body=json.dumps({"predictions": [list(_VECTOR_A)]}).encode())
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"vertex_ai/{_VERTEX_BGE_MODEL}",
api_base=wire.url,
api_key=None,
vertex_project=_PROJECT,
vertex_location=_LOCATION,
vertex_credentials=service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")),
)
response: Final = gateway.request(
"POST",
"/v1/embeddings",
{"model": model, "input": [_VERTEX_INPUTS[0]], "extra_body": {"autoTruncate": False}},
)
assert response.status_code == 200, response.text
payload: Final = _EmbeddingResponse.model_validate_json(response.content)
assert [item.embedding for item in payload.data] == [list(_VECTOR_A)], response.text
assert len(wire.drain()) == 1
@pytest.mark.parametrize(
("embedding_input", "expected_inputs"),
((list(_GEMINI_INPUTS), _GEMINI_INPUTS), (_GEMINI_INPUTS[0], (_GEMINI_INPUTS[0],))),

View file

@ -307,11 +307,18 @@ async def test_vertex_embedding_usage_carries_the_billable_character_count_googl
)
def test_vertex_embedding_call_sends_extra_body_fields_as_predict_parameters(predict_endpoint: respx.Route) -> None:
predict_endpoint.mock(return_value=_predict_response({"predictions": [_gecko_prediction([0.1, 2], 3)]}))
@pytest.mark.parametrize(
("model", "prediction", "instance"),
[(GECKO, _gecko_prediction([0.1, 2], 3), {"content": "hello"}), (BGE, [0.1, 2], {"prompt": "hello"})],
ids=["gecko", "bge"],
)
def test_vertex_embedding_call_sends_extra_body_fields_as_predict_parameters(
predict_endpoint: respx.Route, model: str, prediction: object, instance: dict[str, str]
) -> None:
predict_endpoint.mock(return_value=_predict_response({"predictions": [prediction]}))
litellm.embedding(
model=f"vertex_ai/{GECKO}",
model=f"vertex_ai/{model}",
input=["hello"],
vertex_project=PUBLIC_CALL_PROJECT,
vertex_location="us-central1",
@ -319,6 +326,6 @@ def test_vertex_embedding_call_sends_extra_body_fields_as_predict_parameters(pre
)
body: Final = json.loads(predict_endpoint.calls.last.request.content)
assert body["instances"] == [{"content": "hello"}]
assert body["instances"] == [instance]
assert "extra_body" not in body["parameters"]
assert body["parameters"]["autoTruncate"] is False