From c2c76fc7e4eaad1d18721078b8a6c57643d6e210 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 24 Oct 2026 18:37:13 -0700 Subject: [PATCH] fix(vertex_ai): gate a batch row's sampling knobs by the model the job runs on A batch file row whose model kept a gemini/ prefix, copied from an AI Studio call, was looked up under that prefix on the Vertex route. The AI Studio row exists, so the provider check rejected it and the fallback rule never ran, the lookup raised, and the reader fell open: temperature, top_p and top_k were stored even under drop_params. The row runs on the deployment's Vertex model either way, which the stored GCS path already binds it to, so the row is now shaped and gated for that model, falling back to its own id only for a direct SDK upload that names no deployment, with the Google namespace stripped the way a live call strips it --- .../llms/vertex_ai/files/transformation.py | 40 +++++++++--- .../test_vertex_ai_files_transformation.py | 64 ++++++++++++++++++- 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 881c6d56b69..250aec1a14c 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -679,9 +679,28 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( ) +_GOOGLE_MODEL_NAMESPACES: Final = ("vertex_ai/", "gemini/") + + +def _configured_deployment_model(litellm_params: Mapping[str, object]) -> str | None: + configured_model: Final = litellm_params.get("model") + return configured_model if isinstance(configured_model, str) else None + + +def _batch_row_model(deployment_model: str | None, row_model: object) -> str: + """The Gemini id a batch row is shaped and gated for: the deployment's model, which the stored GCS path + binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google + namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied + from an AI Studio call lands on its Vertex row instead of past every row.""" + chosen: Final = deployment_model if deployment_model else row_model if isinstance(row_model, str) else "" + namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "") + return chosen.removeprefix(namespace) + + def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: dict[str, Any], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + deployment_model: str | None = None, ) -> tuple[Mapping[str, object], ...]: """ Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to. @@ -701,10 +720,11 @@ def _openai_batch_jsonl_entry_to_vertex_rows( if _is_responses_batch_entry(openai_entry) else openai_request_body ) + row_body: Final = {**chat_request_body, "model": _batch_row_model(deployment_model, chat_request_body.get("model"))} vertex_request_body: Final = transform_request_body( - messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])), - model=chat_request_body.get("model", ""), - optional_params=map_openai_to_vertex_params(chat_request_body), + messages=map_developer_role_to_system_role(row_body.get("messages", [])), + model=row_body["model"], + optional_params=map_openai_to_vertex_params(row_body), custom_llm_provider="vertex_ai", litellm_params={}, cached_content=None, @@ -854,15 +874,19 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + deployment_model: str | None, ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params + self._deployment_model = deployment_model def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]: first = True for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content): try: - vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params) + vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows( + entry, self._map_openai_to_vertex_params, self._deployment_model + ) except Exception as e: raise VertexAIError( status_code=400, @@ -1047,7 +1071,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): api_key: str | None, model: str, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], data: CreateFileRequest, ) -> str: """ @@ -1071,11 +1095,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): raise ValueError("file is required") if purpose is None: raise ValueError("purpose is required") - configured_model: Final = litellm_params.get("model") object_name = self.get_object_name( file_data, purpose, - deployment_model=configured_model if isinstance(configured_model, str) else None, + deployment_model=_configured_deployment_model(litellm_params), passthrough=is_passthrough_batch_upload(data, litellm_params), ) if object_prefix: @@ -1131,7 +1154,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], ) -> bytes | str | dict: """ 2 Cases: @@ -1162,6 +1185,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): body_stream=_OpenAIToVertexBatchUploadStream( file_data, self._map_openai_to_vertex_params, + _configured_deployment_model(litellm_params), ), content_type="application/json", ) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 64af8153169..b4482e4a413 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -5,7 +5,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from types import MappingProxyType from typing import Final from unittest.mock import MagicMock @@ -17,6 +17,7 @@ from openai.types.file_deleted import FileDeleted from pydantic import TypeAdapter import litellm +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, @@ -2069,3 +2070,64 @@ def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_ monkeypatch.setattr(litellm, "drop_params", True) (row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) assert _generation_config(row) == {"max_output_tokens": 5} + + +def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: + monkeypatch.setitem(litellm.model_cost, f"gemini/{model_name}", {"litellm_provider": "gemini", "mode": "chat"}) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{model_name}", + {"litellm_provider": "vertex_ai", "mode": "chat", "supports_sampling_params": False}, + ) + + +def _upload_rows(entries: Iterable[Mapping[str, object]], deployment_model: str | None) -> list[object]: + """The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's + `?model=` branch runs it, with the deployment's model in litellm_params""" + jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode() + request: Final = VertexAIFilesConfig().transform_create_file_request( + model="", + create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"}, + optional_params={}, + litellm_params={"model": deployment_model} if deployment_model else {}, + ) + assert isinstance(request, dict) + body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes()) + return [json.loads(line) for line in body.decode().splitlines()] + + +@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"]) +def test_batch_row_keeping_a_gemini_prefix_is_judged_by_its_vertex_row( + deployment_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A row copied from an AI Studio call keeps its `gemini/` prefix but runs on Vertex, so the Vertex row's flag + decides. Looked up under that prefix on the Vertex route it found no row and fell open, so the knobs were + stored even though the model ignores them""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(VertexAIError, match=r"temperature, top_p, top_k.*drop_params"): + _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model) + + +@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"]) +def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_params( + deployment_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", True) + (row,) = _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model) + assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} + + +def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The stored GCS path binds the batch job to the deployment's model, so a row written for an older id that + still takes the knobs is gated for the model that actually executes it""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setitem( + litellm.model_cost, "vertex_ai/gemini-zeta-1", {"litellm_provider": "vertex_ai", "mode": "chat"} + ) + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(VertexAIError, match=r"gemini-zeta-9 does not support temperature, top_p, top_k"): + _upload_rows([_chat_entry("gemini-zeta-1", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9")