fix(vertex_ai): gate a batch row's sampling knobs by the model the job runs on

A batch file row whose model kept a gemini/ prefix, copied from an AI Studio
call, was looked up under that prefix on the Vertex route. The AI Studio row
exists, so the provider check rejected it and the fallback rule never ran, the
lookup raised, and the reader fell open: temperature, top_p and top_k were
stored even under drop_params. The row runs on the deployment's Vertex model
either way, which the stored GCS path already binds it to, so the row is now
shaped and gated for that model, falling back to its own id only for a direct
SDK upload that names no deployment, with the Google namespace stripped the
way a live call strips it
This commit is contained in:
ryan-crabbe-berri 2026-10-24 18:37:13 -07:00
parent 75c9dd1c4b
commit c2c76fc7e4
2 changed files with 95 additions and 9 deletions

View file

@ -679,9 +679,28 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows(
)
_GOOGLE_MODEL_NAMESPACES: Final = ("vertex_ai/", "gemini/")
def _configured_deployment_model(litellm_params: Mapping[str, object]) -> str | None:
configured_model: Final = litellm_params.get("model")
return configured_model if isinstance(configured_model, str) else None
def _batch_row_model(deployment_model: str | None, row_model: object) -> str:
"""The Gemini id a batch row is shaped and gated for: the deployment's model, which the stored GCS path
binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google
namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied
from an AI Studio call lands on its Vertex row instead of past every row."""
chosen: Final = deployment_model if deployment_model else row_model if isinstance(row_model, str) else ""
namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "")
return chosen.removeprefix(namespace)
def _openai_batch_jsonl_entry_to_vertex_rows(
openai_entry: dict[str, Any],
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
deployment_model: str | None = None,
) -> tuple[Mapping[str, object], ...]:
"""
Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to.
@ -701,10 +720,11 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
if _is_responses_batch_entry(openai_entry)
else openai_request_body
)
row_body: Final = {**chat_request_body, "model": _batch_row_model(deployment_model, chat_request_body.get("model"))}
vertex_request_body: Final = transform_request_body(
messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
model=chat_request_body.get("model", ""),
optional_params=map_openai_to_vertex_params(chat_request_body),
messages=map_developer_role_to_system_role(row_body.get("messages", [])),
model=row_body["model"],
optional_params=map_openai_to_vertex_params(row_body),
custom_llm_provider="vertex_ai",
litellm_params={},
cached_content=None,
@ -854,15 +874,19 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
self,
openai_file_content: FileTypes,
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
deployment_model: str | None,
) -> None:
self._openai_file_content = openai_file_content
self._map_openai_to_vertex_params = map_openai_to_vertex_params
self._deployment_model = deployment_model
def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]:
first = True
for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content):
try:
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params)
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(
entry, self._map_openai_to_vertex_params, self._deployment_model
)
except Exception as e:
raise VertexAIError(
status_code=400,
@ -1047,7 +1071,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
api_key: str | None,
model: str,
optional_params: dict,
litellm_params: dict,
litellm_params: dict[str, object],
data: CreateFileRequest,
) -> str:
"""
@ -1071,11 +1095,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
raise ValueError("file is required")
if purpose is None:
raise ValueError("purpose is required")
configured_model: Final = litellm_params.get("model")
object_name = self.get_object_name(
file_data,
purpose,
deployment_model=configured_model if isinstance(configured_model, str) else None,
deployment_model=_configured_deployment_model(litellm_params),
passthrough=is_passthrough_batch_upload(data, litellm_params),
)
if object_prefix:
@ -1131,7 +1154,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
model: str,
create_file_data: CreateFileRequest,
optional_params: dict,
litellm_params: dict,
litellm_params: dict[str, object],
) -> bytes | str | dict:
"""
2 Cases:
@ -1162,6 +1185,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
body_stream=_OpenAIToVertexBatchUploadStream(
file_data,
self._map_openai_to_vertex_params,
_configured_deployment_model(litellm_params),
),
content_type="application/json",
)

View file

@ -5,7 +5,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format.
import json
import urllib.parse
from collections.abc import Mapping
from collections.abc import Iterable, Mapping
from types import MappingProxyType
from typing import Final
from unittest.mock import MagicMock
@ -17,6 +17,7 @@ from openai.types.file_deleted import FileDeleted
from pydantic import TypeAdapter
import litellm
from litellm.llms.vertex_ai.common_utils import VertexAIError
from litellm.llms.vertex_ai.files.transformation import (
VertexAIFilesConfig,
_get_litellm_batch_custom_id_from_labels,
@ -2069,3 +2070,64 @@ def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_
monkeypatch.setattr(litellm, "drop_params", True)
(row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
assert _generation_config(row) == {"max_output_tokens": 5}
def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_name: str) -> None:
monkeypatch.setitem(litellm.model_cost, f"gemini/{model_name}", {"litellm_provider": "gemini", "mode": "chat"})
monkeypatch.setitem(
litellm.model_cost,
f"vertex_ai/{model_name}",
{"litellm_provider": "vertex_ai", "mode": "chat", "supports_sampling_params": False},
)
def _upload_rows(entries: Iterable[Mapping[str, object]], deployment_model: str | None) -> list[object]:
"""The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's
`?model=<deployment>` branch runs it, with the deployment's model in litellm_params"""
jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode()
request: Final = VertexAIFilesConfig().transform_create_file_request(
model="",
create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"},
optional_params={},
litellm_params={"model": deployment_model} if deployment_model else {},
)
assert isinstance(request, dict)
body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes())
return [json.loads(line) for line in body.decode().splitlines()]
@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"])
def test_batch_row_keeping_a_gemini_prefix_is_judged_by_its_vertex_row(
deployment_model: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A row copied from an AI Studio call keeps its `gemini/` prefix but runs on Vertex, so the Vertex row's flag
decides. Looked up under that prefix on the Vertex route it found no row and fell open, so the knobs were
stored even though the model ignores them"""
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(VertexAIError, match=r"temperature, top_p, top_k.*drop_params"):
_upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model)
@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"])
def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_params(
deployment_model: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
monkeypatch.setattr(litellm, "drop_params", True)
(row,) = _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model)
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The stored GCS path binds the batch job to the deployment's model, so a row written for an older id that
still takes the knobs is gated for the model that actually executes it"""
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
monkeypatch.setitem(
litellm.model_cost, "vertex_ai/gemini-zeta-1", {"litellm_provider": "vertex_ai", "mode": "chat"}
)
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(VertexAIError, match=r"gemini-zeta-9 does not support temperature, top_p, top_k"):
_upload_rows([_chat_entry("gemini-zeta-1", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9")