From 62ff5d2bd93d43d819104e1bb9179da62c26c2e1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 26 Oct 2026 17:22:46 -0700 Subject: [PATCH] fix(vertex_ai): honor a batch deployment's drop_params on flagged sampling knobs A batch deployment with `litellm_params: drop_params: true` still rejected rows that carried temperature, top_p or top_k for a Gemini model flagged `supports_sampling_params: false`, because the batch mapper passed `drop_params=False` to the rejection check while the live chat path reads the deployment's drop_params. Both gates now read the same flag, so the knobs are dropped instead of the upload failing with a 400 The upload stream's `deployment_model` argument defaults to None again, so the two-argument callers (direct SDK uploads and the existing batch tests) keep working, and a row whose body.model is missing or not a string is still reported as a 400 naming the line when the deployment names no model --- .../llms/vertex_ai/files/transformation.py | 11 +++++--- .../test_vertex_ai_files_transformation.py | 25 ++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index d32341a498e..8ffaa5bf8e0 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -697,7 +697,9 @@ def _batch_row_model(deployment_model: str | None, row_model: object) -> str: binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied from an AI Studio call lands on its Vertex row instead of past every row.""" - chosen: Final = deployment_model if deployment_model else row_model if isinstance(row_model, str) else "" + chosen: Final = deployment_model if deployment_model else row_model + if not isinstance(chosen, str): + raise TypeError("`body.model` on batch requests must be a string, but was missing or not a string") namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "") return chosen.removeprefix(namespace) @@ -879,7 +881,7 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], - deployment_model: str | None, + deployment_model: str | None = None, ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params @@ -1143,15 +1145,16 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): sampling_params: Final = tuple( p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None ) + drop_params: Final = drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)) if sampling_params and not VertexGeminiConfig.supports_sampling_params(model, "vertex_ai"): VertexGeminiConfig.reject_unsupported_sampling_param( - model=model, param=", ".join(sampling_params), drop_params=False + model=model, param=", ".join(sampling_params), drop_params=drop_params ) vertex_params: Final = config.map_openai_params( model=model, non_default_params=openai_request_body, optional_params={}, - drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)), + drop_params=drop_params, allow_integer_enums=False, ) return vertex_params diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index b4482e4a413..25ede9f0d01 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -2081,15 +2081,21 @@ def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_n ) -def _upload_rows(entries: Iterable[Mapping[str, object]], deployment_model: str | None) -> list[object]: +def _upload_rows( + entries: Iterable[Mapping[str, object]], deployment_model: str | None, deployment_drop_params: bool | None = None +) -> list[object]: """The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's - `?model=` branch runs it, with the deployment's model in litellm_params""" + `?model=` branch runs it, with the deployment's model and drop_params in litellm_params""" jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode() + litellm_params: Final[dict[str, object]] = { + **({"model": deployment_model} if deployment_model else {}), + **({"drop_params": deployment_drop_params} if deployment_drop_params is not None else {}), + } request: Final = VertexAIFilesConfig().transform_create_file_request( model="", create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"}, optional_params={}, - litellm_params={"model": deployment_model} if deployment_model else {}, + litellm_params=litellm_params, ) assert isinstance(request, dict) body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes()) @@ -2119,6 +2125,19 @@ def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_par assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} +def test_batch_row_for_a_flagged_gemini_model_drops_the_knobs_under_the_deployment_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A batch deployment with `drop_params: true` in its litellm_params gets the knobs dropped the way a + real-time call on that deployment does, instead of a 400 that only the global switch could avoid""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + (row,) = _upload_rows( + [_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9", deployment_drop_params=True + ) + assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} + + def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with( monkeypatch: pytest.MonkeyPatch, ) -> None: