diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 881c6d56b69..250aec1a14c 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -679,9 +679,28 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( ) +_GOOGLE_MODEL_NAMESPACES: Final = ("vertex_ai/", "gemini/") + + +def _configured_deployment_model(litellm_params: Mapping[str, object]) -> str | None: + configured_model: Final = litellm_params.get("model") + return configured_model if isinstance(configured_model, str) else None + + +def _batch_row_model(deployment_model: str | None, row_model: object) -> str: + """The Gemini id a batch row is shaped and gated for: the deployment's model, which the stored GCS path + binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google + namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied + from an AI Studio call lands on its Vertex row instead of past every row.""" + chosen: Final = deployment_model if deployment_model else row_model if isinstance(row_model, str) else "" + namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "") + return chosen.removeprefix(namespace) + + def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: dict[str, Any], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + deployment_model: str | None = None, ) -> tuple[Mapping[str, object], ...]: """ Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to. @@ -701,10 +720,11 @@ def _openai_batch_jsonl_entry_to_vertex_rows( if _is_responses_batch_entry(openai_entry) else openai_request_body ) + row_body: Final = {**chat_request_body, "model": _batch_row_model(deployment_model, chat_request_body.get("model"))} vertex_request_body: Final = transform_request_body( - messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])), - model=chat_request_body.get("model", ""), - optional_params=map_openai_to_vertex_params(chat_request_body), + messages=map_developer_role_to_system_role(row_body.get("messages", [])), + model=row_body["model"], + optional_params=map_openai_to_vertex_params(row_body), custom_llm_provider="vertex_ai", litellm_params={}, cached_content=None, @@ -854,15 +874,19 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + deployment_model: str | None, ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params + self._deployment_model = deployment_model def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]: first = True for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content): try: - vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params) + vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows( + entry, self._map_openai_to_vertex_params, self._deployment_model + ) except Exception as e: raise VertexAIError( status_code=400, @@ -1047,7 +1071,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): api_key: str | None, model: str, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], data: CreateFileRequest, ) -> str: """ @@ -1071,11 +1095,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): raise ValueError("file is required") if purpose is None: raise ValueError("purpose is required") - configured_model: Final = litellm_params.get("model") object_name = self.get_object_name( file_data, purpose, - deployment_model=configured_model if isinstance(configured_model, str) else None, + deployment_model=_configured_deployment_model(litellm_params), passthrough=is_passthrough_batch_upload(data, litellm_params), ) if object_prefix: @@ -1131,7 +1154,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], ) -> bytes | str | dict: """ 2 Cases: @@ -1162,6 +1185,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): body_stream=_OpenAIToVertexBatchUploadStream( file_data, self._map_openai_to_vertex_params, + _configured_deployment_model(litellm_params), ), content_type="application/json", ) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 64af8153169..b4482e4a413 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -5,7 +5,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from types import MappingProxyType from typing import Final from unittest.mock import MagicMock @@ -17,6 +17,7 @@ from openai.types.file_deleted import FileDeleted from pydantic import TypeAdapter import litellm +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, @@ -2069,3 +2070,64 @@ def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_ monkeypatch.setattr(litellm, "drop_params", True) (row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) assert _generation_config(row) == {"max_output_tokens": 5} + + +def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: + monkeypatch.setitem(litellm.model_cost, f"gemini/{model_name}", {"litellm_provider": "gemini", "mode": "chat"}) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{model_name}", + {"litellm_provider": "vertex_ai", "mode": "chat", "supports_sampling_params": False}, + ) + + +def _upload_rows(entries: Iterable[Mapping[str, object]], deployment_model: str | None) -> list[object]: + """The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's + `?model=` branch runs it, with the deployment's model in litellm_params""" + jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode() + request: Final = VertexAIFilesConfig().transform_create_file_request( + model="", + create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"}, + optional_params={}, + litellm_params={"model": deployment_model} if deployment_model else {}, + ) + assert isinstance(request, dict) + body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes()) + return [json.loads(line) for line in body.decode().splitlines()] + + +@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"]) +def test_batch_row_keeping_a_gemini_prefix_is_judged_by_its_vertex_row( + deployment_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A row copied from an AI Studio call keeps its `gemini/` prefix but runs on Vertex, so the Vertex row's flag + decides. Looked up under that prefix on the Vertex route it found no row and fell open, so the knobs were + stored even though the model ignores them""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(VertexAIError, match=r"temperature, top_p, top_k.*drop_params"): + _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model) + + +@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"]) +def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_params( + deployment_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", True) + (row,) = _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model) + assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} + + +def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The stored GCS path binds the batch job to the deployment's model, so a row written for an older id that + still takes the knobs is gated for the model that actually executes it""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setitem( + litellm.model_cost, "vertex_ai/gemini-zeta-1", {"litellm_provider": "vertex_ai", "mode": "chat"} + ) + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(VertexAIError, match=r"gemini-zeta-9 does not support temperature, top_p, top_k"): + _upload_rows([_chat_entry("gemini-zeta-1", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9")