fix(vertex_ai): honor a batch deployment's drop_params on flagged sampling knobs

A batch deployment with `litellm_params: drop_params: true` still rejected rows that
carried temperature, top_p or top_k for a Gemini model flagged
`supports_sampling_params: false`, because the batch mapper passed `drop_params=False`
to the rejection check while the live chat path reads the deployment's drop_params.
Both gates now read the same flag, so the knobs are dropped instead of the upload
failing with a 400

The upload stream's `deployment_model` argument defaults to None again, so the
two-argument callers (direct SDK uploads and the existing batch tests) keep working,
and a row whose body.model is missing or not a string is still reported as a 400
naming the line when the deployment names no model
This commit is contained in:
ryan-crabbe-berri 2026-10-26 17:22:46 -07:00
parent b2dddd4012
commit 62ff5d2bd9
2 changed files with 29 additions and 7 deletions

View file

@ -697,7 +697,9 @@ def _batch_row_model(deployment_model: str | None, row_model: object) -> str:
binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google
namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied
from an AI Studio call lands on its Vertex row instead of past every row."""
chosen: Final = deployment_model if deployment_model else row_model if isinstance(row_model, str) else ""
chosen: Final = deployment_model if deployment_model else row_model
if not isinstance(chosen, str):
raise TypeError("`body.model` on batch requests must be a string, but was missing or not a string")
namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "")
return chosen.removeprefix(namespace)
@ -879,7 +881,7 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
self,
openai_file_content: FileTypes,
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
deployment_model: str | None,
deployment_model: str | None = None,
) -> None:
self._openai_file_content = openai_file_content
self._map_openai_to_vertex_params = map_openai_to_vertex_params
@ -1143,15 +1145,16 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
sampling_params: Final = tuple(
p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None
)
drop_params: Final = drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params))
if sampling_params and not VertexGeminiConfig.supports_sampling_params(model, "vertex_ai"):
VertexGeminiConfig.reject_unsupported_sampling_param(
model=model, param=", ".join(sampling_params), drop_params=False
model=model, param=", ".join(sampling_params), drop_params=drop_params
)
vertex_params: Final = config.map_openai_params(
model=model,
non_default_params=openai_request_body,
optional_params={},
drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)),
drop_params=drop_params,
allow_integer_enums=False,
)
return vertex_params

View file

@ -2081,15 +2081,21 @@ def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_n
)
def _upload_rows(entries: Iterable[Mapping[str, object]], deployment_model: str | None) -> list[object]:
def _upload_rows(
entries: Iterable[Mapping[str, object]], deployment_model: str | None, deployment_drop_params: bool | None = None
) -> list[object]:
"""The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's
`?model=<deployment>` branch runs it, with the deployment's model in litellm_params"""
`?model=<deployment>` branch runs it, with the deployment's model and drop_params in litellm_params"""
jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode()
litellm_params: Final[dict[str, object]] = {
**({"model": deployment_model} if deployment_model else {}),
**({"drop_params": deployment_drop_params} if deployment_drop_params is not None else {}),
}
request: Final = VertexAIFilesConfig().transform_create_file_request(
model="",
create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"},
optional_params={},
litellm_params={"model": deployment_model} if deployment_model else {},
litellm_params=litellm_params,
)
assert isinstance(request, dict)
body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes())
@ -2119,6 +2125,19 @@ def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_par
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
def test_batch_row_for_a_flagged_gemini_model_drops_the_knobs_under_the_deployment_drop_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A batch deployment with `drop_params: true` in its litellm_params gets the knobs dropped the way a
real-time call on that deployment does, instead of a 400 that only the global switch could avoid"""
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
monkeypatch.setattr(litellm, "drop_params", False)
(row,) = _upload_rows(
[_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9", deployment_drop_params=True
)
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with(
monkeypatch: pytest.MonkeyPatch,
) -> None: